aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2018-03-14 00:53:01 -0400
committerDave Airlie <airlied@redhat.com>2018-03-14 00:53:01 -0400
commit963976cfe9c54d4d9e725e61c90c47a4af6b5ea2 (patch)
tree20735b399c810390573a8416115978e734eef8a3
parent6fa7324ac5489ad43c4b6351355b869bc5458bef (diff)
parentcf07a60f03f06d6298f4e70b3865bd5faac21c3b (diff)
Merge tag 'drm-intel-next-2018-03-08' of git://anongit.freedesktop.org/drm/drm-intel into drm-next
UAPI Changes: - Query uAPI interface (used for GPU topology information currently) * Mesa: https://patchwork.freedesktop.org/series/38795/ Driver Changes: - Increase PSR2 size for CNL (DK) - Avoid retraining LSPCON link unnecessarily (Ville) - Decrease request signaling latency (Chris) - GuC error capture fix (Daniele) * tag 'drm-intel-next-2018-03-08' of git://anongit.freedesktop.org/drm/drm-intel: (127 commits) drm/i915: Update DRIVER_DATE to 20180308 drm/i915: add schedule out notification of preempted but completed request drm/i915: expose rcs topology through query uAPI drm/i915: add query uAPI drm/i915: add rcs topology to error state drm/i915/debugfs: add rcs topology entry drm/i915/debugfs: reuse max slice/subslices already stored in sseu drm/i915: store all subslice masks drm/i915/guc: work around gcc-4.4.4 union initializer issue drm/i915/cnl: Add Wa_2201832410 drm/i915/icl: Gen11 forcewake support drm/i915/icl: Add Indirect Context Offset for Gen11 drm/i915/icl: Enhanced execution list support drm/i915/icl: new context descriptor support drm/i915/icl: Correctly initialize the Gen11 engines drm/i915: Assert that the request is indeed complete when signaled from irq drm/i915: Handle changing enable_fbc parameter at runtime better. drm/i915: Track whether the DP link is trained or not drm/i915: Nuke intel_dp->channel_eq_status drm/i915: Move SST DP link retraining into the ->post_hotplug() hook ...
Diffstat
-rw-r--r--Documentation/gpu/todo.rst7
-rw-r--r--drivers/gpu/drm/drm_vblank.c22
-rw-r--r--drivers/gpu/drm/i915/Makefile6
-rw-r--r--drivers/gpu/drm/i915/gvt/Makefile2
-rw-r--r--drivers/gpu/drm/i915/gvt/dmabuf.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.c1457
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.h189
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.h21
-rw-r--r--drivers/gpu/drm/i915/gvt/handlers.c38
-rw-r--r--drivers/gpu/drm/i915/gvt/hypercall.h9
-rw-r--r--drivers/gpu/drm/i915/gvt/kvmgt.c313
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio.c9
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio_context.c210
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio_context.h5
-rw-r--r--drivers/gpu/drm/i915/gvt/mpt.h67
-rw-r--r--drivers/gpu/drm/i915/gvt/page_track.c184
-rw-r--r--drivers/gpu/drm/i915/gvt/page_track.h56
-rw-r--r--drivers/gpu/drm/i915/gvt/sched_policy.c5
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.c60
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.h2
-rw-r--r--drivers/gpu/drm/i915/gvt/trace.h10
-rw-r--r--drivers/gpu/drm/i915/gvt/vgpu.c1
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c214
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c16
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h41
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c119
-rw-r--r--drivers/gpu/drm/i915/i915_gem.h7
-rw-r--r--drivers/gpu/drm/i915/i915_gem_batch_pool.c2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_context.c29
-rw-r--r--drivers/gpu/drm/i915/i915_gem_context.h4
-rw-r--r--drivers/gpu/drm/i915/i915_gem_evict.c4
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c60
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c38
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.h5
-rw-r--r--drivers/gpu/drm/i915/i915_gem_object.h2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_render_state.c2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_render_state.h4
-rw-r--r--drivers/gpu/drm/i915/i915_gem_shrinker.c4
-rw-r--r--drivers/gpu/drm/i915/i915_gem_timeline.h4
-rw-r--r--drivers/gpu/drm/i915/i915_gpu_error.c124
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c244
-rw-r--r--drivers/gpu/drm/i915/i915_pci.c4
-rw-r--r--drivers/gpu/drm/i915/i915_perf.c68
-rw-r--r--drivers/gpu/drm/i915/i915_query.c125
-rw-r--r--drivers/gpu/drm/i915/i915_query.h15
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h39
-rw-r--r--drivers/gpu/drm/i915/i915_request.c (renamed from drivers/gpu/drm/i915/i915_gem_request.c)390
-rw-r--r--drivers/gpu/drm/i915/i915_request.h (renamed from drivers/gpu/drm/i915/i915_gem_request.h)222
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h128
-rw-r--r--drivers/gpu/drm/i915/i915_vma.c3
-rw-r--r--drivers/gpu/drm/i915/i915_vma.h2
-rw-r--r--drivers/gpu/drm/i915/intel_breadcrumbs.c292
-rw-r--r--drivers/gpu/drm/i915/intel_color.c97
-rw-r--r--drivers/gpu/drm/i915/intel_crt.c4
-rw-r--r--drivers/gpu/drm/i915/intel_ddi.c233
-rw-r--r--drivers/gpu/drm/i915/intel_device_info.c284
-rw-r--r--drivers/gpu/drm/i915/intel_device_info.h73
-rw-r--r--drivers/gpu/drm/i915/intel_display.c189
-rw-r--r--drivers/gpu/drm/i915/intel_display.h12
-rw-r--r--drivers/gpu/drm/i915/intel_dp.c747
-rw-r--r--drivers/gpu/drm/i915/intel_dp_link_training.c8
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h22
-rw-r--r--drivers/gpu/drm/i915/intel_engine_cs.c79
-rw-r--r--drivers/gpu/drm/i915/intel_fbc.c105
-rw-r--r--drivers/gpu/drm/i915/intel_fbdev.c2
-rw-r--r--drivers/gpu/drm/i915/intel_frontbuffer.c2
-rw-r--r--drivers/gpu/drm/i915/intel_guc.c44
-rw-r--r--drivers/gpu/drm/i915/intel_guc.h4
-rw-r--r--drivers/gpu/drm/i915/intel_guc_fw.c10
-rw-r--r--drivers/gpu/drm/i915/intel_guc_log.c6
-rw-r--r--drivers/gpu/drm/i915/intel_guc_submission.c140
-rw-r--r--drivers/gpu/drm/i915/intel_hdmi.c1
-rw-r--r--drivers/gpu/drm/i915/intel_hotplug.c25
-rw-r--r--drivers/gpu/drm/i915/intel_huc.c182
-rw-r--r--drivers/gpu/drm/i915/intel_huc.h2
-rw-r--r--drivers/gpu/drm/i915/intel_huc_fw.c166
-rw-r--r--drivers/gpu/drm/i915/intel_huc_fw.h15
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.c455
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.h3
-rw-r--r--drivers/gpu/drm/i915/intel_lrc_reg.h1
-rw-r--r--drivers/gpu/drm/i915/intel_mocs.c28
-rw-r--r--drivers/gpu/drm/i915/intel_mocs.h2
-rw-r--r--drivers/gpu/drm/i915/intel_overlay.c82
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c21
-rw-r--r--drivers/gpu/drm/i915/intel_psr.c181
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c196
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.h114
-rw-r--r--drivers/gpu/drm/i915/intel_runtime_pm.c3
-rw-r--r--drivers/gpu/drm/i915/intel_sdvo.c12
-rw-r--r--drivers/gpu/drm/i915/intel_uc.c47
-rw-r--r--drivers/gpu/drm/i915/intel_uc.h2
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.c157
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.h23
-rw-r--r--drivers/gpu/drm/i915/selftests/huge_pages.c6
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_coherency.c8
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_context.c8
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_evict.c6
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_gem_object.c6
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_live_selftests.h2
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_mock_selftests.h2
-rw-r--r--drivers/gpu/drm/i915/selftests/i915_request.c (renamed from drivers/gpu/drm/i915/selftests/i915_gem_request.c)125
-rw-r--r--drivers/gpu/drm/i915/selftests/intel_hangcheck.c119
-rw-r--r--drivers/gpu/drm/i915/selftests/intel_uncore.c31
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_engine.c10
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_gem_device.c2
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_request.c10
-rw-r--r--drivers/gpu/drm/i915/selftests/mock_request.h8
-rw-r--r--include/drm/i915_pciids.h12
-rw-r--r--include/uapi/drm/i915_drm.h108
110 files changed, 5510 insertions, 3620 deletions
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 1a0a413eeced..f4d0b3476d9c 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -450,5 +450,12 @@ See drivers/gpu/drm/amd/display/TODO for tasks.
450 450
451Contact: Harry Wentland, Alex Deucher 451Contact: Harry Wentland, Alex Deucher
452 452
453i915
454----
455
456- Our early/late pm callbacks could be removed in favour of using
457 device_link_add to model the dependency between i915 and snd_had. See
458 https://dri.freedesktop.org/docs/drm/driver-api/device_link.html
459
453Outside DRM 460Outside DRM
454=========== 461===========
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index e02ddb01917b..28cdcf76b6f9 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -1247,12 +1247,15 @@ void drm_crtc_vblank_on(struct drm_crtc *crtc)
1247EXPORT_SYMBOL(drm_crtc_vblank_on); 1247EXPORT_SYMBOL(drm_crtc_vblank_on);
1248 1248
1249/** 1249/**
1250 * drm_vblank_restore - estimated vblanks using timestamps and update it. 1250 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1251 * @dev: DRM device
1252 * @pipe: CRTC index
1251 * 1253 *
1252 * Power manamement features can cause frame counter resets between vblank 1254 * Power manamement features can cause frame counter resets between vblank
1253 * disable and enable. Drivers can then use this function in their 1255 * disable and enable. Drivers can use this function in their
1254 * &drm_crtc_funcs.enable_vblank implementation to estimate the vblanks since 1256 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1255 * the last &drm_crtc_funcs.disable_vblank. 1257 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1258 * vblank counter.
1256 * 1259 *
1257 * This function is the legacy version of drm_crtc_vblank_restore(). 1260 * This function is the legacy version of drm_crtc_vblank_restore().
1258 */ 1261 */
@@ -1293,11 +1296,14 @@ void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1293EXPORT_SYMBOL(drm_vblank_restore); 1296EXPORT_SYMBOL(drm_vblank_restore);
1294 1297
1295/** 1298/**
1296 * drm_crtc_vblank_restore - estimate vblanks using timestamps and update it. 1299 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1300 * @crtc: CRTC in question
1301 *
1297 * Power manamement features can cause frame counter resets between vblank 1302 * Power manamement features can cause frame counter resets between vblank
1298 * disable and enable. Drivers can then use this function in their 1303 * disable and enable. Drivers can use this function in their
1299 * &drm_crtc_funcs.enable_vblank implementation to estimate the vblanks since 1304 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1300 * the last &drm_crtc_funcs.disable_vblank. 1305 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1306 * vblank counter.
1301 */ 1307 */
1302void drm_crtc_vblank_restore(struct drm_crtc *crtc) 1308void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1303{ 1309{
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index f55cc028b2eb..4eee91a3a236 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -63,13 +63,14 @@ i915-y += i915_cmd_parser.o \
63 i915_gem.o \ 63 i915_gem.o \
64 i915_gem_object.o \ 64 i915_gem_object.o \
65 i915_gem_render_state.o \ 65 i915_gem_render_state.o \
66 i915_gem_request.o \
67 i915_gem_shrinker.o \ 66 i915_gem_shrinker.o \
68 i915_gem_stolen.o \ 67 i915_gem_stolen.o \
69 i915_gem_tiling.o \ 68 i915_gem_tiling.o \
70 i915_gem_timeline.o \ 69 i915_gem_timeline.o \
71 i915_gem_userptr.o \ 70 i915_gem_userptr.o \
72 i915_gemfs.o \ 71 i915_gemfs.o \
72 i915_query.o \
73 i915_request.o \
73 i915_trace_points.o \ 74 i915_trace_points.o \
74 i915_vma.o \ 75 i915_vma.o \
75 intel_breadcrumbs.o \ 76 intel_breadcrumbs.o \
@@ -89,7 +90,8 @@ i915-y += intel_uc.o \
89 intel_guc_fw.o \ 90 intel_guc_fw.o \
90 intel_guc_log.o \ 91 intel_guc_log.o \
91 intel_guc_submission.o \ 92 intel_guc_submission.o \
92 intel_huc.o 93 intel_huc.o \
94 intel_huc_fw.o
93 95
94# autogenerated null render state 96# autogenerated null render state
95i915-y += intel_renderstate_gen6.o \ 97i915-y += intel_renderstate_gen6.o \
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 347116faa558..b016dc753db9 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -3,7 +3,7 @@ GVT_DIR := gvt
3GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \ 3GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
4 interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \ 4 interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
5 execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \ 5 execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \
6 fb_decoder.o dmabuf.o 6 fb_decoder.o dmabuf.o page_track.o
7 7
8ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) 8ccflags-y += -I$(src) -I$(src)/$(GVT_DIR)
9i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) 9i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c
index 9a471b0afb15..b555eb26f9ce 100644
--- a/drivers/gpu/drm/i915/gvt/dmabuf.c
+++ b/drivers/gpu/drm/i915/gvt/dmabuf.c
@@ -459,7 +459,7 @@ int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id)
459 459
460 obj = vgpu_create_gem(dev, dmabuf_obj->info); 460 obj = vgpu_create_gem(dev, dmabuf_obj->info);
461 if (obj == NULL) { 461 if (obj == NULL) {
462 gvt_vgpu_err("create gvt gem obj failed:%d\n", vgpu->id); 462 gvt_vgpu_err("create gvt gem obj failed\n");
463 ret = -ENOMEM; 463 ret = -ENOMEM;
464 goto out; 464 goto out;
465 } 465 }
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 8d5317d0122d..0a100a288e6d 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -38,6 +38,12 @@
38#include "i915_pvinfo.h" 38#include "i915_pvinfo.h"
39#include "trace.h" 39#include "trace.h"
40 40
41#if defined(VERBOSE_DEBUG)
42#define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args)
43#else
44#define gvt_vdbg_mm(fmt, args...)
45#endif
46
41static bool enable_out_of_sync = false; 47static bool enable_out_of_sync = false;
42static int preallocated_oos_pages = 8192; 48static int preallocated_oos_pages = 8192;
43 49
@@ -264,7 +270,7 @@ static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
264 return readq(addr); 270 return readq(addr);
265} 271}
266 272
267static void gtt_invalidate(struct drm_i915_private *dev_priv) 273static void ggtt_invalidate(struct drm_i915_private *dev_priv)
268{ 274{
269 mmio_hw_access_pre(dev_priv); 275 mmio_hw_access_pre(dev_priv);
270 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 276 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
@@ -331,20 +337,20 @@ static inline int gtt_set_entry64(void *pt,
331 337
332#define GTT_HAW 46 338#define GTT_HAW 46
333 339
334#define ADDR_1G_MASK (((1UL << (GTT_HAW - 30)) - 1) << 30) 340#define ADDR_1G_MASK GENMASK_ULL(GTT_HAW - 1, 30)
335#define ADDR_2M_MASK (((1UL << (GTT_HAW - 21)) - 1) << 21) 341#define ADDR_2M_MASK GENMASK_ULL(GTT_HAW - 1, 21)
336#define ADDR_4K_MASK (((1UL << (GTT_HAW - 12)) - 1) << 12) 342#define ADDR_4K_MASK GENMASK_ULL(GTT_HAW - 1, 12)
337 343
338static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e) 344static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
339{ 345{
340 unsigned long pfn; 346 unsigned long pfn;
341 347
342 if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) 348 if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY)
343 pfn = (e->val64 & ADDR_1G_MASK) >> 12; 349 pfn = (e->val64 & ADDR_1G_MASK) >> PAGE_SHIFT;
344 else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) 350 else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY)
345 pfn = (e->val64 & ADDR_2M_MASK) >> 12; 351 pfn = (e->val64 & ADDR_2M_MASK) >> PAGE_SHIFT;
346 else 352 else
347 pfn = (e->val64 & ADDR_4K_MASK) >> 12; 353 pfn = (e->val64 & ADDR_4K_MASK) >> PAGE_SHIFT;
348 return pfn; 354 return pfn;
349} 355}
350 356
@@ -352,16 +358,16 @@ static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
352{ 358{
353 if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) { 359 if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) {
354 e->val64 &= ~ADDR_1G_MASK; 360 e->val64 &= ~ADDR_1G_MASK;
355 pfn &= (ADDR_1G_MASK >> 12); 361 pfn &= (ADDR_1G_MASK >> PAGE_SHIFT);
356 } else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) { 362 } else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) {
357 e->val64 &= ~ADDR_2M_MASK; 363 e->val64 &= ~ADDR_2M_MASK;
358 pfn &= (ADDR_2M_MASK >> 12); 364 pfn &= (ADDR_2M_MASK >> PAGE_SHIFT);
359 } else { 365 } else {
360 e->val64 &= ~ADDR_4K_MASK; 366 e->val64 &= ~ADDR_4K_MASK;
361 pfn &= (ADDR_4K_MASK >> 12); 367 pfn &= (ADDR_4K_MASK >> PAGE_SHIFT);
362 } 368 }
363 369
364 e->val64 |= (pfn << 12); 370 e->val64 |= (pfn << PAGE_SHIFT);
365} 371}
366 372
367static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e) 373static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
@@ -371,7 +377,7 @@ static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
371 return false; 377 return false;
372 378
373 e->type = get_entry_type(e->type); 379 e->type = get_entry_type(e->type);
374 if (!(e->val64 & BIT(7))) 380 if (!(e->val64 & _PAGE_PSE))
375 return false; 381 return false;
376 382
377 e->type = get_pse_type(e->type); 383 e->type = get_pse_type(e->type);
@@ -389,17 +395,17 @@ static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
389 || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) 395 || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
390 return (e->val64 != 0); 396 return (e->val64 != 0);
391 else 397 else
392 return (e->val64 & BIT(0)); 398 return (e->val64 & _PAGE_PRESENT);
393} 399}
394 400
395static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e) 401static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
396{ 402{
397 e->val64 &= ~BIT(0); 403 e->val64 &= ~_PAGE_PRESENT;
398} 404}
399 405
400static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e) 406static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e)
401{ 407{
402 e->val64 |= BIT(0); 408 e->val64 |= _PAGE_PRESENT;
403} 409}
404 410
405/* 411/*
@@ -447,58 +453,91 @@ static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
447 .gma_to_pml4_index = gen8_gma_to_pml4_index, 453 .gma_to_pml4_index = gen8_gma_to_pml4_index,
448}; 454};
449 455
450static int gtt_entry_p2m(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *p, 456/*
451 struct intel_gvt_gtt_entry *m) 457 * MM helpers.
458 */
459static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm,
460 struct intel_gvt_gtt_entry *entry, unsigned long index,
461 bool guest)
452{ 462{
453 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 463 struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
454 unsigned long gfn, mfn;
455 464
456 *m = *p; 465 GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT);
457 466
458 if (!ops->test_present(p)) 467 entry->type = mm->ppgtt_mm.root_entry_type;
459 return 0; 468 pte_ops->get_entry(guest ? mm->ppgtt_mm.guest_pdps :
469 mm->ppgtt_mm.shadow_pdps,
470 entry, index, false, 0, mm->vgpu);
460 471
461 gfn = ops->get_pfn(p); 472 pte_ops->test_pse(entry);
473}
462 474
463 mfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, gfn); 475static inline void ppgtt_get_guest_root_entry(struct intel_vgpu_mm *mm,
464 if (mfn == INTEL_GVT_INVALID_ADDR) { 476 struct intel_gvt_gtt_entry *entry, unsigned long index)
465 gvt_vgpu_err("fail to translate gfn: 0x%lx\n", gfn); 477{
466 return -ENXIO; 478 _ppgtt_get_root_entry(mm, entry, index, true);
467 } 479}
468 480
469 ops->set_pfn(m, mfn); 481static inline void ppgtt_get_shadow_root_entry(struct intel_vgpu_mm *mm,
470 return 0; 482 struct intel_gvt_gtt_entry *entry, unsigned long index)
483{
484 _ppgtt_get_root_entry(mm, entry, index, false);
471} 485}
472 486
473/* 487static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm,
474 * MM helpers. 488 struct intel_gvt_gtt_entry *entry, unsigned long index,
475 */ 489 bool guest)
476int intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
477 void *page_table, struct intel_gvt_gtt_entry *e,
478 unsigned long index)
479{ 490{
480 struct intel_gvt *gvt = mm->vgpu->gvt; 491 struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
481 struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
482 int ret;
483 492
484 e->type = mm->page_table_entry_type; 493 pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps :
494 mm->ppgtt_mm.shadow_pdps,
495 entry, index, false, 0, mm->vgpu);
496}
485 497
486 ret = ops->get_entry(page_table, e, index, false, 0, mm->vgpu); 498static inline void ppgtt_set_guest_root_entry(struct intel_vgpu_mm *mm,
487 if (ret) 499 struct intel_gvt_gtt_entry *entry, unsigned long index)
488 return ret; 500{
501 _ppgtt_set_root_entry(mm, entry, index, true);
502}
489 503
490 ops->test_pse(e); 504static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm,
491 return 0; 505 struct intel_gvt_gtt_entry *entry, unsigned long index)
506{
507 _ppgtt_set_root_entry(mm, entry, index, false);
492} 508}
493 509
494int intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm, 510static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm,
495 void *page_table, struct intel_gvt_gtt_entry *e, 511 struct intel_gvt_gtt_entry *entry, unsigned long index)
496 unsigned long index)
497{ 512{
498 struct intel_gvt *gvt = mm->vgpu->gvt; 513 struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
499 struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; 514
515 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
516
517 entry->type = GTT_TYPE_GGTT_PTE;
518 pte_ops->get_entry(mm->ggtt_mm.virtual_ggtt, entry, index,
519 false, 0, mm->vgpu);
520}
521
522static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm,
523 struct intel_gvt_gtt_entry *entry, unsigned long index)
524{
525 struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
526
527 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
528
529 pte_ops->set_entry(mm->ggtt_mm.virtual_ggtt, entry, index,
530 false, 0, mm->vgpu);
531}
532
533static void ggtt_set_host_entry(struct intel_vgpu_mm *mm,
534 struct intel_gvt_gtt_entry *entry, unsigned long index)
535{
536 struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
537
538 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
500 539
501 return ops->set_entry(page_table, e, index, false, 0, mm->vgpu); 540 pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu);
502} 541}
503 542
504/* 543/*
@@ -520,12 +559,15 @@ static inline int ppgtt_spt_get_entry(
520 return -EINVAL; 559 return -EINVAL;
521 560
522 ret = ops->get_entry(page_table, e, index, guest, 561 ret = ops->get_entry(page_table, e, index, guest,
523 spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT, 562 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
524 spt->vgpu); 563 spt->vgpu);
525 if (ret) 564 if (ret)
526 return ret; 565 return ret;
527 566
528 ops->test_pse(e); 567 ops->test_pse(e);
568
569 gvt_vdbg_mm("read ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n",
570 type, e->type, index, e->val64);
529 return 0; 571 return 0;
530} 572}
531 573
@@ -541,18 +583,21 @@ static inline int ppgtt_spt_set_entry(
541 if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n")) 583 if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
542 return -EINVAL; 584 return -EINVAL;
543 585
586 gvt_vdbg_mm("set ppgtt entry, spt type %d, entry type %d, index %lu, value %llx\n",
587 type, e->type, index, e->val64);
588
544 return ops->set_entry(page_table, e, index, guest, 589 return ops->set_entry(page_table, e, index, guest,
545 spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT, 590 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
546 spt->vgpu); 591 spt->vgpu);
547} 592}
548 593
549#define ppgtt_get_guest_entry(spt, e, index) \ 594#define ppgtt_get_guest_entry(spt, e, index) \
550 ppgtt_spt_get_entry(spt, NULL, \ 595 ppgtt_spt_get_entry(spt, NULL, \
551 spt->guest_page_type, e, index, true) 596 spt->guest_page.type, e, index, true)
552 597
553#define ppgtt_set_guest_entry(spt, e, index) \ 598#define ppgtt_set_guest_entry(spt, e, index) \
554 ppgtt_spt_set_entry(spt, NULL, \ 599 ppgtt_spt_set_entry(spt, NULL, \
555 spt->guest_page_type, e, index, true) 600 spt->guest_page.type, e, index, true)
556 601
557#define ppgtt_get_shadow_entry(spt, e, index) \ 602#define ppgtt_get_shadow_entry(spt, e, index) \
558 ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \ 603 ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \
@@ -562,159 +607,6 @@ static inline int ppgtt_spt_set_entry(
562 ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \ 607 ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \
563 spt->shadow_page.type, e, index, false) 608 spt->shadow_page.type, e, index, false)
564 609
565/**
566 * intel_vgpu_init_page_track - init a page track data structure
567 * @vgpu: a vGPU
568 * @t: a page track data structure
569 * @gfn: guest memory page frame number
570 * @handler: the function will be called when target guest memory page has
571 * been modified.
572 *
573 * This function is called when a user wants to prepare a page track data
574 * structure to track a guest memory page.
575 *
576 * Returns:
577 * Zero on success, negative error code if failed.
578 */
579int intel_vgpu_init_page_track(struct intel_vgpu *vgpu,
580 struct intel_vgpu_page_track *t,
581 unsigned long gfn,
582 int (*handler)(void *, u64, void *, int),
583 void *data)
584{
585 INIT_HLIST_NODE(&t->node);
586
587 t->tracked = false;
588 t->gfn = gfn;
589 t->handler = handler;
590 t->data = data;
591
592 hash_add(vgpu->gtt.tracked_guest_page_hash_table, &t->node, t->gfn);
593 return 0;
594}
595
596/**
597 * intel_vgpu_clean_page_track - release a page track data structure
598 * @vgpu: a vGPU
599 * @t: a page track data structure
600 *
601 * This function is called before a user frees a page track data structure.
602 */
603void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu,
604 struct intel_vgpu_page_track *t)
605{
606 if (!hlist_unhashed(&t->node))
607 hash_del(&t->node);
608
609 if (t->tracked)
610 intel_gvt_hypervisor_disable_page_track(vgpu, t);
611}
612
613/**
614 * intel_vgpu_find_tracked_page - find a tracked guest page
615 * @vgpu: a vGPU
616 * @gfn: guest memory page frame number
617 *
618 * This function is called when the emulation layer wants to figure out if a
619 * trapped GFN is a tracked guest page.
620 *
621 * Returns:
622 * Pointer to page track data structure, NULL if not found.
623 */
624struct intel_vgpu_page_track *intel_vgpu_find_tracked_page(
625 struct intel_vgpu *vgpu, unsigned long gfn)
626{
627 struct intel_vgpu_page_track *t;
628
629 hash_for_each_possible(vgpu->gtt.tracked_guest_page_hash_table,
630 t, node, gfn) {
631 if (t->gfn == gfn)
632 return t;
633 }
634 return NULL;
635}
636
637static int init_guest_page(struct intel_vgpu *vgpu,
638 struct intel_vgpu_guest_page *p,
639 unsigned long gfn,
640 int (*handler)(void *, u64, void *, int),
641 void *data)
642{
643 p->oos_page = NULL;
644 p->write_cnt = 0;
645
646 return intel_vgpu_init_page_track(vgpu, &p->track, gfn, handler, data);
647}
648
649static int detach_oos_page(struct intel_vgpu *vgpu,
650 struct intel_vgpu_oos_page *oos_page);
651
652static void clean_guest_page(struct intel_vgpu *vgpu,
653 struct intel_vgpu_guest_page *p)
654{
655 if (p->oos_page)
656 detach_oos_page(vgpu, p->oos_page);
657
658 intel_vgpu_clean_page_track(vgpu, &p->track);
659}
660
661static inline int init_shadow_page(struct intel_vgpu *vgpu,
662 struct intel_vgpu_shadow_page *p, int type, bool hash)
663{
664 struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
665 dma_addr_t daddr;
666
667 daddr = dma_map_page(kdev, p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
668 if (dma_mapping_error(kdev, daddr)) {
669 gvt_vgpu_err("fail to map dma addr\n");
670 return -EINVAL;
671 }
672
673 p->vaddr = page_address(p->page);
674 p->type = type;
675
676 INIT_HLIST_NODE(&p->node);
677
678 p->mfn = daddr >> I915_GTT_PAGE_SHIFT;
679 if (hash)
680 hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
681 return 0;
682}
683
684static inline void clean_shadow_page(struct intel_vgpu *vgpu,
685 struct intel_vgpu_shadow_page *p)
686{
687 struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
688
689 dma_unmap_page(kdev, p->mfn << I915_GTT_PAGE_SHIFT, 4096,
690 PCI_DMA_BIDIRECTIONAL);
691
692 if (!hlist_unhashed(&p->node))
693 hash_del(&p->node);
694}
695
696static inline struct intel_vgpu_shadow_page *find_shadow_page(
697 struct intel_vgpu *vgpu, unsigned long mfn)
698{
699 struct intel_vgpu_shadow_page *p;
700
701 hash_for_each_possible(vgpu->gtt.shadow_page_hash_table,
702 p, node, mfn) {
703 if (p->mfn == mfn)
704 return p;
705 }
706 return NULL;
707}
708
709#define page_track_to_guest_page(ptr) \
710 container_of(ptr, struct intel_vgpu_guest_page, track)
711
712#define guest_page_to_ppgtt_spt(ptr) \
713 container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
714
715#define shadow_page_to_ppgtt_spt(ptr) \
716 container_of(ptr, struct intel_vgpu_ppgtt_spt, shadow_page)
717
718static void *alloc_spt(gfp_t gfp_mask) 610static void *alloc_spt(gfp_t gfp_mask)
719{ 611{
720 struct intel_vgpu_ppgtt_spt *spt; 612 struct intel_vgpu_ppgtt_spt *spt;
@@ -737,63 +629,96 @@ static void free_spt(struct intel_vgpu_ppgtt_spt *spt)
737 kfree(spt); 629 kfree(spt);
738} 630}
739 631
740static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt) 632static int detach_oos_page(struct intel_vgpu *vgpu,
633 struct intel_vgpu_oos_page *oos_page);
634
635static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt)
741{ 636{
742 trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type); 637 struct device *kdev = &spt->vgpu->gvt->dev_priv->drm.pdev->dev;
743 638
744 clean_shadow_page(spt->vgpu, &spt->shadow_page); 639 trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type);
745 clean_guest_page(spt->vgpu, &spt->guest_page); 640
746 list_del_init(&spt->post_shadow_list); 641 dma_unmap_page(kdev, spt->shadow_page.mfn << I915_GTT_PAGE_SHIFT, 4096,
642 PCI_DMA_BIDIRECTIONAL);
747 643
644 radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);
645
646 if (spt->guest_page.oos_page)
647 detach_oos_page(spt->vgpu, spt->guest_page.oos_page);
648
649 intel_vgpu_unregister_page_track(spt->vgpu, spt->guest_page.gfn);
650
651 list_del_init(&spt->post_shadow_list);
748 free_spt(spt); 652 free_spt(spt);
749} 653}
750 654
751static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu) 655static void ppgtt_free_all_spt(struct intel_vgpu *vgpu)
752{ 656{
753 struct hlist_node *n; 657 struct intel_vgpu_ppgtt_spt *spt;
754 struct intel_vgpu_shadow_page *sp; 658 struct radix_tree_iter iter;
755 int i; 659 void **slot;
756 660
757 hash_for_each_safe(vgpu->gtt.shadow_page_hash_table, i, n, sp, node) 661 radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {
758 ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp)); 662 spt = radix_tree_deref_slot(slot);
663 ppgtt_free_spt(spt);
664 }
759} 665}
760 666
761static int ppgtt_handle_guest_write_page_table_bytes( 667static int ppgtt_handle_guest_write_page_table_bytes(
762 struct intel_vgpu_guest_page *gpt, 668 struct intel_vgpu_ppgtt_spt *spt,
763 u64 pa, void *p_data, int bytes); 669 u64 pa, void *p_data, int bytes);
764 670
765static int ppgtt_write_protection_handler(void *data, u64 pa, 671static int ppgtt_write_protection_handler(
766 void *p_data, int bytes) 672 struct intel_vgpu_page_track *page_track,
673 u64 gpa, void *data, int bytes)
767{ 674{
768 struct intel_vgpu_page_track *t = data; 675 struct intel_vgpu_ppgtt_spt *spt = page_track->priv_data;
769 struct intel_vgpu_guest_page *p = page_track_to_guest_page(t); 676
770 int ret; 677 int ret;
771 678
772 if (bytes != 4 && bytes != 8) 679 if (bytes != 4 && bytes != 8)
773 return -EINVAL; 680 return -EINVAL;
774 681
775 if (!t->tracked) 682 ret = ppgtt_handle_guest_write_page_table_bytes(spt, gpa, data, bytes);
776 return -EINVAL;
777
778 ret = ppgtt_handle_guest_write_page_table_bytes(p,
779 pa, p_data, bytes);
780 if (ret) 683 if (ret)
781 return ret; 684 return ret;
782 return ret; 685 return ret;
783} 686}
784 687
785static int reclaim_one_mm(struct intel_gvt *gvt); 688/* Find a spt by guest gfn. */
689static struct intel_vgpu_ppgtt_spt *intel_vgpu_find_spt_by_gfn(
690 struct intel_vgpu *vgpu, unsigned long gfn)
691{
692 struct intel_vgpu_page_track *track;
693
694 track = intel_vgpu_find_page_track(vgpu, gfn);
695 if (track && track->handler == ppgtt_write_protection_handler)
696 return track->priv_data;
786 697
787static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_shadow_page( 698 return NULL;
699}
700
701/* Find the spt by shadow page mfn. */
702static inline struct intel_vgpu_ppgtt_spt *intel_vgpu_find_spt_by_mfn(
703 struct intel_vgpu *vgpu, unsigned long mfn)
704{
705 return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);
706}
707
708static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt);
709
710static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt(
788 struct intel_vgpu *vgpu, int type, unsigned long gfn) 711 struct intel_vgpu *vgpu, int type, unsigned long gfn)
789{ 712{
713 struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
790 struct intel_vgpu_ppgtt_spt *spt = NULL; 714 struct intel_vgpu_ppgtt_spt *spt = NULL;
715 dma_addr_t daddr;
791 int ret; 716 int ret;
792 717
793retry: 718retry:
794 spt = alloc_spt(GFP_KERNEL | __GFP_ZERO); 719 spt = alloc_spt(GFP_KERNEL | __GFP_ZERO);
795 if (!spt) { 720 if (!spt) {
796 if (reclaim_one_mm(vgpu->gvt)) 721 if (reclaim_one_ppgtt_mm(vgpu->gvt))
797 goto retry; 722 goto retry;
798 723
799 gvt_vgpu_err("fail to allocate ppgtt shadow page\n"); 724 gvt_vgpu_err("fail to allocate ppgtt shadow page\n");
@@ -801,44 +726,48 @@ retry:
801 } 726 }
802 727
803 spt->vgpu = vgpu; 728 spt->vgpu = vgpu;
804 spt->guest_page_type = type;
805 atomic_set(&spt->refcount, 1); 729 atomic_set(&spt->refcount, 1);
806 INIT_LIST_HEAD(&spt->post_shadow_list); 730 INIT_LIST_HEAD(&spt->post_shadow_list);
807 731
808 /* 732 /*
809 * TODO: guest page type may be different with shadow page type, 733 * Init shadow_page.
810 * when we support PSE page in future.
811 */ 734 */
812 ret = init_shadow_page(vgpu, &spt->shadow_page, type, true); 735 spt->shadow_page.type = type;
813 if (ret) { 736 daddr = dma_map_page(kdev, spt->shadow_page.page,
814 gvt_vgpu_err("fail to initialize shadow page for spt\n"); 737 0, 4096, PCI_DMA_BIDIRECTIONAL);
815 goto err; 738 if (dma_mapping_error(kdev, daddr)) {
739 gvt_vgpu_err("fail to map dma addr\n");
740 ret = -EINVAL;
741 goto err_free_spt;
816 } 742 }
743 spt->shadow_page.vaddr = page_address(spt->shadow_page.page);
744 spt->shadow_page.mfn = daddr >> I915_GTT_PAGE_SHIFT;
817 745
818 ret = init_guest_page(vgpu, &spt->guest_page, 746 /*
819 gfn, ppgtt_write_protection_handler, NULL); 747 * Init guest_page.
820 if (ret) { 748 */
821 gvt_vgpu_err("fail to initialize guest page for spt\n"); 749 spt->guest_page.type = type;
822 goto err; 750 spt->guest_page.gfn = gfn;
823 }
824 751
825 trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn); 752 ret = intel_vgpu_register_page_track(vgpu, spt->guest_page.gfn,
826 return spt; 753 ppgtt_write_protection_handler, spt);
827err: 754 if (ret)
828 ppgtt_free_shadow_page(spt); 755 goto err_unmap_dma;
829 return ERR_PTR(ret);
830}
831 756
832static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page( 757 ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);
833 struct intel_vgpu *vgpu, unsigned long mfn) 758 if (ret)
834{ 759 goto err_unreg_page_track;
835 struct intel_vgpu_shadow_page *p = find_shadow_page(vgpu, mfn);
836 760
837 if (p) 761 trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
838 return shadow_page_to_ppgtt_spt(p); 762 return spt;
839 763
840 gvt_vgpu_err("fail to find ppgtt shadow page: 0x%lx\n", mfn); 764err_unreg_page_track:
841 return NULL; 765 intel_vgpu_unregister_page_track(vgpu, spt->guest_page.gfn);
766err_unmap_dma:
767 dma_unmap_page(kdev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
768err_free_spt:
769 free_spt(spt);
770 return ERR_PTR(ret);
842} 771}
843 772
844#define pt_entry_size_shift(spt) \ 773#define pt_entry_size_shift(spt) \
@@ -857,7 +786,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
857 if (!ppgtt_get_shadow_entry(spt, e, i) && \ 786 if (!ppgtt_get_shadow_entry(spt, e, i) && \
858 spt->vgpu->gvt->gtt.pte_ops->test_present(e)) 787 spt->vgpu->gvt->gtt.pte_ops->test_present(e))
859 788
860static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt) 789static void ppgtt_get_spt(struct intel_vgpu_ppgtt_spt *spt)
861{ 790{
862 int v = atomic_read(&spt->refcount); 791 int v = atomic_read(&spt->refcount);
863 792
@@ -866,17 +795,16 @@ static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
866 atomic_inc(&spt->refcount); 795 atomic_inc(&spt->refcount);
867} 796}
868 797
869static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt); 798static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt);
870 799
871static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu, 800static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu,
872 struct intel_gvt_gtt_entry *e) 801 struct intel_gvt_gtt_entry *e)
873{ 802{
874 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 803 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
875 struct intel_vgpu_ppgtt_spt *s; 804 struct intel_vgpu_ppgtt_spt *s;
876 intel_gvt_gtt_type_t cur_pt_type; 805 intel_gvt_gtt_type_t cur_pt_type;
877 806
878 if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type)))) 807 GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(e->type)));
879 return -EINVAL;
880 808
881 if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY 809 if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY
882 && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { 810 && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
@@ -885,16 +813,33 @@ static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu,
885 vgpu->gtt.scratch_pt[cur_pt_type].page_mfn) 813 vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)
886 return 0; 814 return 0;
887 } 815 }
888 s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e)); 816 s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e));
889 if (!s) { 817 if (!s) {
890 gvt_vgpu_err("fail to find shadow page: mfn: 0x%lx\n", 818 gvt_vgpu_err("fail to find shadow page: mfn: 0x%lx\n",
891 ops->get_pfn(e)); 819 ops->get_pfn(e));
892 return -ENXIO; 820 return -ENXIO;
893 } 821 }
894 return ppgtt_invalidate_shadow_page(s); 822 return ppgtt_invalidate_spt(s);
895} 823}
896 824
897static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt) 825static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt,
826 struct intel_gvt_gtt_entry *entry)
827{
828 struct intel_vgpu *vgpu = spt->vgpu;
829 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
830 unsigned long pfn;
831 int type;
832
833 pfn = ops->get_pfn(entry);
834 type = spt->shadow_page.type;
835
836 if (pfn == vgpu->gtt.scratch_pt[type].page_mfn)
837 return;
838
839 intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);
840}
841
842static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt)
898{ 843{
899 struct intel_vgpu *vgpu = spt->vgpu; 844 struct intel_vgpu *vgpu = spt->vgpu;
900 struct intel_gvt_gtt_entry e; 845 struct intel_gvt_gtt_entry e;
@@ -903,30 +848,40 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
903 int v = atomic_read(&spt->refcount); 848 int v = atomic_read(&spt->refcount);
904 849
905 trace_spt_change(spt->vgpu->id, "die", spt, 850 trace_spt_change(spt->vgpu->id, "die", spt,
906 spt->guest_page.track.gfn, spt->shadow_page.type); 851 spt->guest_page.gfn, spt->shadow_page.type);
907 852
908 trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1)); 853 trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
909 854
910 if (atomic_dec_return(&spt->refcount) > 0) 855 if (atomic_dec_return(&spt->refcount) > 0)
911 return 0; 856 return 0;
912 857
913 if (gtt_type_is_pte_pt(spt->shadow_page.type))
914 goto release;
915
916 for_each_present_shadow_entry(spt, &e, index) { 858 for_each_present_shadow_entry(spt, &e, index) {
917 if (!gtt_type_is_pt(get_next_pt_type(e.type))) { 859 switch (e.type) {
918 gvt_vgpu_err("GVT doesn't support pse bit for now\n"); 860 case GTT_TYPE_PPGTT_PTE_4K_ENTRY:
919 return -EINVAL; 861 gvt_vdbg_mm("invalidate 4K entry\n");
862 ppgtt_invalidate_pte(spt, &e);
863 break;
864 case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
865 case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
866 WARN(1, "GVT doesn't support 2M/1GB page\n");
867 continue;
868 case GTT_TYPE_PPGTT_PML4_ENTRY:
869 case GTT_TYPE_PPGTT_PDP_ENTRY:
870 case GTT_TYPE_PPGTT_PDE_ENTRY:
871 gvt_vdbg_mm("invalidate PMUL4/PDP/PDE entry\n");
872 ret = ppgtt_invalidate_spt_by_shadow_entry(
873 spt->vgpu, &e);
874 if (ret)
875 goto fail;
876 break;
877 default:
878 GEM_BUG_ON(1);
920 } 879 }
921 ret = ppgtt_invalidate_shadow_page_by_shadow_entry(
922 spt->vgpu, &e);
923 if (ret)
924 goto fail;
925 } 880 }
926release: 881
927 trace_spt_change(spt->vgpu->id, "release", spt, 882 trace_spt_change(spt->vgpu->id, "release", spt,
928 spt->guest_page.track.gfn, spt->shadow_page.type); 883 spt->guest_page.gfn, spt->shadow_page.type);
929 ppgtt_free_shadow_page(spt); 884 ppgtt_free_spt(spt);
930 return 0; 885 return 0;
931fail: 886fail:
932 gvt_vgpu_err("fail: shadow page %p shadow entry 0x%llx type %d\n", 887 gvt_vgpu_err("fail: shadow page %p shadow entry 0x%llx type %d\n",
@@ -934,52 +889,44 @@ fail:
934 return ret; 889 return ret;
935} 890}
936 891
937static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt); 892static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt);
938 893
939static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry( 894static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
940 struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we) 895 struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)
941{ 896{
942 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 897 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
943 struct intel_vgpu_ppgtt_spt *s = NULL; 898 struct intel_vgpu_ppgtt_spt *spt = NULL;
944 struct intel_vgpu_guest_page *g;
945 struct intel_vgpu_page_track *t;
946 int ret; 899 int ret;
947 900
948 if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) { 901 GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(we->type)));
949 ret = -EINVAL;
950 goto fail;
951 }
952 902
953 t = intel_vgpu_find_tracked_page(vgpu, ops->get_pfn(we)); 903 spt = intel_vgpu_find_spt_by_gfn(vgpu, ops->get_pfn(we));
954 if (t) { 904 if (spt)
955 g = page_track_to_guest_page(t); 905 ppgtt_get_spt(spt);
956 s = guest_page_to_ppgtt_spt(g); 906 else {
957 ppgtt_get_shadow_page(s);
958 } else {
959 int type = get_next_pt_type(we->type); 907 int type = get_next_pt_type(we->type);
960 908
961 s = ppgtt_alloc_shadow_page(vgpu, type, ops->get_pfn(we)); 909 spt = ppgtt_alloc_spt(vgpu, type, ops->get_pfn(we));
962 if (IS_ERR(s)) { 910 if (IS_ERR(spt)) {
963 ret = PTR_ERR(s); 911 ret = PTR_ERR(spt);
964 goto fail; 912 goto fail;
965 } 913 }
966 914
967 ret = intel_gvt_hypervisor_enable_page_track(vgpu, 915 ret = intel_vgpu_enable_page_track(vgpu, spt->guest_page.gfn);
968 &s->guest_page.track);
969 if (ret) 916 if (ret)
970 goto fail; 917 goto fail;
971 918
972 ret = ppgtt_populate_shadow_page(s); 919 ret = ppgtt_populate_spt(spt);
973 if (ret) 920 if (ret)
974 goto fail; 921 goto fail;
975 922
976 trace_spt_change(vgpu->id, "new", s, s->guest_page.track.gfn, 923 trace_spt_change(vgpu->id, "new", spt, spt->guest_page.gfn,
977 s->shadow_page.type); 924 spt->shadow_page.type);
978 } 925 }
979 return s; 926 return spt;
980fail: 927fail:
981 gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", 928 gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
982 s, we->val64, we->type); 929 spt, we->val64, we->type);
983 return ERR_PTR(ret); 930 return ERR_PTR(ret);
984} 931}
985 932
@@ -994,7 +941,44 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
994 ops->set_pfn(se, s->shadow_page.mfn); 941 ops->set_pfn(se, s->shadow_page.mfn);
995} 942}
996 943
997static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt) 944static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
945 struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
946 struct intel_gvt_gtt_entry *ge)
947{
948 struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
949 struct intel_gvt_gtt_entry se = *ge;
950 unsigned long gfn;
951 dma_addr_t dma_addr;
952 int ret;
953
954 if (!pte_ops->test_present(ge))
955 return 0;
956
957 gfn = pte_ops->get_pfn(ge);
958
959 switch (ge->type) {
960 case GTT_TYPE_PPGTT_PTE_4K_ENTRY:
961 gvt_vdbg_mm("shadow 4K gtt entry\n");
962 break;
963 case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
964 case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
965 gvt_vgpu_err("GVT doesn't support 2M/1GB entry\n");
966 return -EINVAL;
967 default:
968 GEM_BUG_ON(1);
969 };
970
971 /* direct shadow */
972 ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, &dma_addr);
973 if (ret)
974 return -ENXIO;
975
976 pte_ops->set_pfn(&se, dma_addr >> PAGE_SHIFT);
977 ppgtt_set_shadow_entry(spt, &se, index);
978 return 0;
979}
980
981static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt)
998{ 982{
999 struct intel_vgpu *vgpu = spt->vgpu; 983 struct intel_vgpu *vgpu = spt->vgpu;
1000 struct intel_gvt *gvt = vgpu->gvt; 984 struct intel_gvt *gvt = vgpu->gvt;
@@ -1005,34 +989,30 @@ static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
1005 int ret; 989 int ret;
1006 990
1007 trace_spt_change(spt->vgpu->id, "born", spt, 991 trace_spt_change(spt->vgpu->id, "born", spt,
1008 spt->guest_page.track.gfn, spt->shadow_page.type); 992 spt->guest_page.gfn, spt->shadow_page.type);
1009 993
1010 if (gtt_type_is_pte_pt(spt->shadow_page.type)) { 994 for_each_present_guest_entry(spt, &ge, i) {
1011 for_each_present_guest_entry(spt, &ge, i) { 995 if (gtt_type_is_pt(get_next_pt_type(ge.type))) {
996 s = ppgtt_populate_spt_by_guest_entry(vgpu, &ge);
997 if (IS_ERR(s)) {
998 ret = PTR_ERR(s);
999 goto fail;
1000 }
1001 ppgtt_get_shadow_entry(spt, &se, i);
1002 ppgtt_generate_shadow_entry(&se, s, &ge);
1003 ppgtt_set_shadow_entry(spt, &se, i);
1004 } else {
1012 gfn = ops->get_pfn(&ge); 1005 gfn = ops->get_pfn(&ge);
1013 if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn) || 1006 if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {
1014 gtt_entry_p2m(vgpu, &ge, &se))
1015 ops->set_pfn(&se, gvt->gtt.scratch_mfn); 1007 ops->set_pfn(&se, gvt->gtt.scratch_mfn);
1016 ppgtt_set_shadow_entry(spt, &se, i); 1008 ppgtt_set_shadow_entry(spt, &se, i);
1017 } 1009 continue;
1018 return 0; 1010 }
1019 }
1020
1021 for_each_present_guest_entry(spt, &ge, i) {
1022 if (!gtt_type_is_pt(get_next_pt_type(ge.type))) {
1023 gvt_vgpu_err("GVT doesn't support pse bit now\n");
1024 ret = -EINVAL;
1025 goto fail;
1026 }
1027 1011
1028 s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge); 1012 ret = ppgtt_populate_shadow_entry(vgpu, spt, i, &ge);
1029 if (IS_ERR(s)) { 1013 if (ret)
1030 ret = PTR_ERR(s); 1014 goto fail;
1031 goto fail;
1032 } 1015 }
1033 ppgtt_get_shadow_entry(spt, &se, i);
1034 ppgtt_generate_shadow_entry(&se, s, &ge);
1035 ppgtt_set_shadow_entry(spt, &se, i);
1036 } 1016 }
1037 return 0; 1017 return 0;
1038fail: 1018fail:
@@ -1041,36 +1021,40 @@ fail:
1041 return ret; 1021 return ret;
1042} 1022}
1043 1023
1044static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt, 1024static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt,
1045 struct intel_gvt_gtt_entry *se, unsigned long index) 1025 struct intel_gvt_gtt_entry *se, unsigned long index)
1046{ 1026{
1047 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1048 struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
1049 struct intel_vgpu *vgpu = spt->vgpu; 1027 struct intel_vgpu *vgpu = spt->vgpu;
1050 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 1028 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1051 int ret; 1029 int ret;
1052 1030
1053 trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, se->val64, 1031 trace_spt_guest_change(spt->vgpu->id, "remove", spt,
1054 index); 1032 spt->shadow_page.type, se->val64, index);
1033
1034 gvt_vdbg_mm("destroy old shadow entry, type %d, index %lu, value %llx\n",
1035 se->type, index, se->val64);
1055 1036
1056 if (!ops->test_present(se)) 1037 if (!ops->test_present(se))
1057 return 0; 1038 return 0;
1058 1039
1059 if (ops->get_pfn(se) == vgpu->gtt.scratch_pt[sp->type].page_mfn) 1040 if (ops->get_pfn(se) ==
1041 vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)
1060 return 0; 1042 return 0;
1061 1043
1062 if (gtt_type_is_pt(get_next_pt_type(se->type))) { 1044 if (gtt_type_is_pt(get_next_pt_type(se->type))) {
1063 struct intel_vgpu_ppgtt_spt *s = 1045 struct intel_vgpu_ppgtt_spt *s =
1064 ppgtt_find_shadow_page(vgpu, ops->get_pfn(se)); 1046 intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(se));
1065 if (!s) { 1047 if (!s) {
1066 gvt_vgpu_err("fail to find guest page\n"); 1048 gvt_vgpu_err("fail to find guest page\n");
1067 ret = -ENXIO; 1049 ret = -ENXIO;
1068 goto fail; 1050 goto fail;
1069 } 1051 }
1070 ret = ppgtt_invalidate_shadow_page(s); 1052 ret = ppgtt_invalidate_spt(s);
1071 if (ret) 1053 if (ret)
1072 goto fail; 1054 goto fail;
1073 } 1055 } else
1056 ppgtt_invalidate_pte(spt, se);
1057
1074 return 0; 1058 return 0;
1075fail: 1059fail:
1076 gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n", 1060 gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
@@ -1078,21 +1062,22 @@ fail:
1078 return ret; 1062 return ret;
1079} 1063}
1080 1064
1081static int ppgtt_handle_guest_entry_add(struct intel_vgpu_guest_page *gpt, 1065static int ppgtt_handle_guest_entry_add(struct intel_vgpu_ppgtt_spt *spt,
1082 struct intel_gvt_gtt_entry *we, unsigned long index) 1066 struct intel_gvt_gtt_entry *we, unsigned long index)
1083{ 1067{
1084 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1085 struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
1086 struct intel_vgpu *vgpu = spt->vgpu; 1068 struct intel_vgpu *vgpu = spt->vgpu;
1087 struct intel_gvt_gtt_entry m; 1069 struct intel_gvt_gtt_entry m;
1088 struct intel_vgpu_ppgtt_spt *s; 1070 struct intel_vgpu_ppgtt_spt *s;
1089 int ret; 1071 int ret;
1090 1072
1091 trace_gpt_change(spt->vgpu->id, "add", spt, sp->type, 1073 trace_spt_guest_change(spt->vgpu->id, "add", spt, spt->shadow_page.type,
1092 we->val64, index); 1074 we->val64, index);
1075
1076 gvt_vdbg_mm("add shadow entry: type %d, index %lu, value %llx\n",
1077 we->type, index, we->val64);
1093 1078
1094 if (gtt_type_is_pt(get_next_pt_type(we->type))) { 1079 if (gtt_type_is_pt(get_next_pt_type(we->type))) {
1095 s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, we); 1080 s = ppgtt_populate_spt_by_guest_entry(vgpu, we);
1096 if (IS_ERR(s)) { 1081 if (IS_ERR(s)) {
1097 ret = PTR_ERR(s); 1082 ret = PTR_ERR(s);
1098 goto fail; 1083 goto fail;
@@ -1101,10 +1086,9 @@ static int ppgtt_handle_guest_entry_add(struct intel_vgpu_guest_page *gpt,
1101 ppgtt_generate_shadow_entry(&m, s, we); 1086 ppgtt_generate_shadow_entry(&m, s, we);
1102 ppgtt_set_shadow_entry(spt, &m, index); 1087 ppgtt_set_shadow_entry(spt, &m, index);
1103 } else { 1088 } else {
1104 ret = gtt_entry_p2m(vgpu, we, &m); 1089 ret = ppgtt_populate_shadow_entry(vgpu, spt, index, we);
1105 if (ret) 1090 if (ret)
1106 goto fail; 1091 goto fail;
1107 ppgtt_set_shadow_entry(spt, &m, index);
1108 } 1092 }
1109 return 0; 1093 return 0;
1110fail: 1094fail:
@@ -1119,41 +1103,39 @@ static int sync_oos_page(struct intel_vgpu *vgpu,
1119 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; 1103 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1120 struct intel_gvt *gvt = vgpu->gvt; 1104 struct intel_gvt *gvt = vgpu->gvt;
1121 struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops; 1105 struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
1122 struct intel_vgpu_ppgtt_spt *spt = 1106 struct intel_vgpu_ppgtt_spt *spt = oos_page->spt;
1123 guest_page_to_ppgtt_spt(oos_page->guest_page); 1107 struct intel_gvt_gtt_entry old, new;
1124 struct intel_gvt_gtt_entry old, new, m;
1125 int index; 1108 int index;
1126 int ret; 1109 int ret;
1127 1110
1128 trace_oos_change(vgpu->id, "sync", oos_page->id, 1111 trace_oos_change(vgpu->id, "sync", oos_page->id,
1129 oos_page->guest_page, spt->guest_page_type); 1112 spt, spt->guest_page.type);
1130 1113
1131 old.type = new.type = get_entry_type(spt->guest_page_type); 1114 old.type = new.type = get_entry_type(spt->guest_page.type);
1132 old.val64 = new.val64 = 0; 1115 old.val64 = new.val64 = 0;
1133 1116
1134 for (index = 0; index < (I915_GTT_PAGE_SIZE >> 1117 for (index = 0; index < (I915_GTT_PAGE_SIZE >>
1135 info->gtt_entry_size_shift); index++) { 1118 info->gtt_entry_size_shift); index++) {
1136 ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu); 1119 ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
1137 ops->get_entry(NULL, &new, index, true, 1120 ops->get_entry(NULL, &new, index, true,
1138 oos_page->guest_page->track.gfn << PAGE_SHIFT, vgpu); 1121 spt->guest_page.gfn << PAGE_SHIFT, vgpu);
1139 1122
1140 if (old.val64 == new.val64 1123 if (old.val64 == new.val64
1141 && !test_and_clear_bit(index, spt->post_shadow_bitmap)) 1124 && !test_and_clear_bit(index, spt->post_shadow_bitmap))
1142 continue; 1125 continue;
1143 1126
1144 trace_oos_sync(vgpu->id, oos_page->id, 1127 trace_oos_sync(vgpu->id, oos_page->id,
1145 oos_page->guest_page, spt->guest_page_type, 1128 spt, spt->guest_page.type,
1146 new.val64, index); 1129 new.val64, index);
1147 1130
1148 ret = gtt_entry_p2m(vgpu, &new, &m); 1131 ret = ppgtt_populate_shadow_entry(vgpu, spt, index, &new);
1149 if (ret) 1132 if (ret)
1150 return ret; 1133 return ret;
1151 1134
1152 ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu); 1135 ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu);
1153 ppgtt_set_shadow_entry(spt, &m, index);
1154 } 1136 }
1155 1137
1156 oos_page->guest_page->write_cnt = 0; 1138 spt->guest_page.write_cnt = 0;
1157 list_del_init(&spt->post_shadow_list); 1139 list_del_init(&spt->post_shadow_list);
1158 return 0; 1140 return 0;
1159} 1141}
@@ -1162,15 +1144,14 @@ static int detach_oos_page(struct intel_vgpu *vgpu,
1162 struct intel_vgpu_oos_page *oos_page) 1144 struct intel_vgpu_oos_page *oos_page)
1163{ 1145{
1164 struct intel_gvt *gvt = vgpu->gvt; 1146 struct intel_gvt *gvt = vgpu->gvt;
1165 struct intel_vgpu_ppgtt_spt *spt = 1147 struct intel_vgpu_ppgtt_spt *spt = oos_page->spt;
1166 guest_page_to_ppgtt_spt(oos_page->guest_page);
1167 1148
1168 trace_oos_change(vgpu->id, "detach", oos_page->id, 1149 trace_oos_change(vgpu->id, "detach", oos_page->id,
1169 oos_page->guest_page, spt->guest_page_type); 1150 spt, spt->guest_page.type);
1170 1151
1171 oos_page->guest_page->write_cnt = 0; 1152 spt->guest_page.write_cnt = 0;
1172 oos_page->guest_page->oos_page = NULL; 1153 spt->guest_page.oos_page = NULL;
1173 oos_page->guest_page = NULL; 1154 oos_page->spt = NULL;
1174 1155
1175 list_del_init(&oos_page->vm_list); 1156 list_del_init(&oos_page->vm_list);
1176 list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head); 1157 list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);
@@ -1178,51 +1159,49 @@ static int detach_oos_page(struct intel_vgpu *vgpu,
1178 return 0; 1159 return 0;
1179} 1160}
1180 1161
1181static int attach_oos_page(struct intel_vgpu *vgpu, 1162static int attach_oos_page(struct intel_vgpu_oos_page *oos_page,
1182 struct intel_vgpu_oos_page *oos_page, 1163 struct intel_vgpu_ppgtt_spt *spt)
1183 struct intel_vgpu_guest_page *gpt)
1184{ 1164{
1185 struct intel_gvt *gvt = vgpu->gvt; 1165 struct intel_gvt *gvt = spt->vgpu->gvt;
1186 int ret; 1166 int ret;
1187 1167
1188 ret = intel_gvt_hypervisor_read_gpa(vgpu, 1168 ret = intel_gvt_hypervisor_read_gpa(spt->vgpu,
1189 gpt->track.gfn << I915_GTT_PAGE_SHIFT, 1169 spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
1190 oos_page->mem, I915_GTT_PAGE_SIZE); 1170 oos_page->mem, I915_GTT_PAGE_SIZE);
1191 if (ret) 1171 if (ret)
1192 return ret; 1172 return ret;
1193 1173
1194 oos_page->guest_page = gpt; 1174 oos_page->spt = spt;
1195 gpt->oos_page = oos_page; 1175 spt->guest_page.oos_page = oos_page;
1196 1176
1197 list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head); 1177 list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);
1198 1178
1199 trace_oos_change(vgpu->id, "attach", gpt->oos_page->id, 1179 trace_oos_change(spt->vgpu->id, "attach", oos_page->id,
1200 gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type); 1180 spt, spt->guest_page.type);
1201 return 0; 1181 return 0;
1202} 1182}
1203 1183
1204static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu, 1184static int ppgtt_set_guest_page_sync(struct intel_vgpu_ppgtt_spt *spt)
1205 struct intel_vgpu_guest_page *gpt)
1206{ 1185{
1186 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1207 int ret; 1187 int ret;
1208 1188
1209 ret = intel_gvt_hypervisor_enable_page_track(vgpu, &gpt->track); 1189 ret = intel_vgpu_enable_page_track(spt->vgpu, spt->guest_page.gfn);
1210 if (ret) 1190 if (ret)
1211 return ret; 1191 return ret;
1212 1192
1213 trace_oos_change(vgpu->id, "set page sync", gpt->oos_page->id, 1193 trace_oos_change(spt->vgpu->id, "set page sync", oos_page->id,
1214 gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type); 1194 spt, spt->guest_page.type);
1215 1195
1216 list_del_init(&gpt->oos_page->vm_list); 1196 list_del_init(&oos_page->vm_list);
1217 return sync_oos_page(vgpu, gpt->oos_page); 1197 return sync_oos_page(spt->vgpu, oos_page);
1218} 1198}
1219 1199
1220static int ppgtt_allocate_oos_page(struct intel_vgpu *vgpu, 1200static int ppgtt_allocate_oos_page(struct intel_vgpu_ppgtt_spt *spt)
1221 struct intel_vgpu_guest_page *gpt)
1222{ 1201{
1223 struct intel_gvt *gvt = vgpu->gvt; 1202 struct intel_gvt *gvt = spt->vgpu->gvt;
1224 struct intel_gvt_gtt *gtt = &gvt->gtt; 1203 struct intel_gvt_gtt *gtt = &gvt->gtt;
1225 struct intel_vgpu_oos_page *oos_page = gpt->oos_page; 1204 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1226 int ret; 1205 int ret;
1227 1206
1228 WARN(oos_page, "shadow PPGTT page has already has a oos page\n"); 1207 WARN(oos_page, "shadow PPGTT page has already has a oos page\n");
@@ -1230,31 +1209,30 @@ static int ppgtt_allocate_oos_page(struct intel_vgpu *vgpu,
1230 if (list_empty(&gtt->oos_page_free_list_head)) { 1209 if (list_empty(&gtt->oos_page_free_list_head)) {
1231 oos_page = container_of(gtt->oos_page_use_list_head.next, 1210 oos_page = container_of(gtt->oos_page_use_list_head.next,
1232 struct intel_vgpu_oos_page, list); 1211 struct intel_vgpu_oos_page, list);
1233 ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page); 1212 ret = ppgtt_set_guest_page_sync(oos_page->spt);
1234 if (ret) 1213 if (ret)
1235 return ret; 1214 return ret;
1236 ret = detach_oos_page(vgpu, oos_page); 1215 ret = detach_oos_page(spt->vgpu, oos_page);
1237 if (ret) 1216 if (ret)
1238 return ret; 1217 return ret;
1239 } else 1218 } else
1240 oos_page = container_of(gtt->oos_page_free_list_head.next, 1219 oos_page = container_of(gtt->oos_page_free_list_head.next,
1241 struct intel_vgpu_oos_page, list); 1220 struct intel_vgpu_oos_page, list);
1242 return attach_oos_page(vgpu, oos_page, gpt); 1221 return attach_oos_page(oos_page, spt);
1243} 1222}
1244 1223
1245static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu, 1224static int ppgtt_set_guest_page_oos(struct intel_vgpu_ppgtt_spt *spt)
1246 struct intel_vgpu_guest_page *gpt)
1247{ 1225{
1248 struct intel_vgpu_oos_page *oos_page = gpt->oos_page; 1226 struct intel_vgpu_oos_page *oos_page = spt->guest_page.oos_page;
1249 1227
1250 if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n")) 1228 if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n"))
1251 return -EINVAL; 1229 return -EINVAL;
1252 1230
1253 trace_oos_change(vgpu->id, "set page out of sync", gpt->oos_page->id, 1231 trace_oos_change(spt->vgpu->id, "set page out of sync", oos_page->id,
1254 gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type); 1232 spt, spt->guest_page.type);
1255 1233
1256 list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head); 1234 list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);
1257 return intel_gvt_hypervisor_disable_page_track(vgpu, &gpt->track); 1235 return intel_vgpu_disable_page_track(spt->vgpu, spt->guest_page.gfn);
1258} 1236}
1259 1237
1260/** 1238/**
@@ -1279,7 +1257,7 @@ int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)
1279 list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) { 1257 list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {
1280 oos_page = container_of(pos, 1258 oos_page = container_of(pos,
1281 struct intel_vgpu_oos_page, vm_list); 1259 struct intel_vgpu_oos_page, vm_list);
1282 ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page); 1260 ret = ppgtt_set_guest_page_sync(oos_page->spt);
1283 if (ret) 1261 if (ret)
1284 return ret; 1262 return ret;
1285 } 1263 }
@@ -1290,17 +1268,15 @@ int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)
1290 * The heart of PPGTT shadow page table. 1268 * The heart of PPGTT shadow page table.
1291 */ 1269 */
1292static int ppgtt_handle_guest_write_page_table( 1270static int ppgtt_handle_guest_write_page_table(
1293 struct intel_vgpu_guest_page *gpt, 1271 struct intel_vgpu_ppgtt_spt *spt,
1294 struct intel_gvt_gtt_entry *we, unsigned long index) 1272 struct intel_gvt_gtt_entry *we, unsigned long index)
1295{ 1273{
1296 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1297 struct intel_vgpu *vgpu = spt->vgpu; 1274 struct intel_vgpu *vgpu = spt->vgpu;
1298 int type = spt->shadow_page.type; 1275 int type = spt->shadow_page.type;
1299 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 1276 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1300 struct intel_gvt_gtt_entry se; 1277 struct intel_gvt_gtt_entry old_se;
1301
1302 int ret;
1303 int new_present; 1278 int new_present;
1279 int ret;
1304 1280
1305 new_present = ops->test_present(we); 1281 new_present = ops->test_present(we);
1306 1282
@@ -1309,21 +1285,21 @@ static int ppgtt_handle_guest_write_page_table(
1309 * guarantee the ppgtt table is validated during the window between 1285 * guarantee the ppgtt table is validated during the window between
1310 * adding and removal. 1286 * adding and removal.
1311 */ 1287 */
1312 ppgtt_get_shadow_entry(spt, &se, index); 1288 ppgtt_get_shadow_entry(spt, &old_se, index);
1313 1289
1314 if (new_present) { 1290 if (new_present) {
1315 ret = ppgtt_handle_guest_entry_add(gpt, we, index); 1291 ret = ppgtt_handle_guest_entry_add(spt, we, index);
1316 if (ret) 1292 if (ret)
1317 goto fail; 1293 goto fail;
1318 } 1294 }
1319 1295
1320 ret = ppgtt_handle_guest_entry_removal(gpt, &se, index); 1296 ret = ppgtt_handle_guest_entry_removal(spt, &old_se, index);
1321 if (ret) 1297 if (ret)
1322 goto fail; 1298 goto fail;
1323 1299
1324 if (!new_present) { 1300 if (!new_present) {
1325 ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn); 1301 ops->set_pfn(&old_se, vgpu->gtt.scratch_pt[type].page_mfn);
1326 ppgtt_set_shadow_entry(spt, &se, index); 1302 ppgtt_set_shadow_entry(spt, &old_se, index);
1327 } 1303 }
1328 1304
1329 return 0; 1305 return 0;
@@ -1333,12 +1309,13 @@ fail:
1333 return ret; 1309 return ret;
1334} 1310}
1335 1311
1336static inline bool can_do_out_of_sync(struct intel_vgpu_guest_page *gpt) 1312
1313
1314static inline bool can_do_out_of_sync(struct intel_vgpu_ppgtt_spt *spt)
1337{ 1315{
1338 return enable_out_of_sync 1316 return enable_out_of_sync
1339 && gtt_type_is_pte_pt( 1317 && gtt_type_is_pte_pt(spt->guest_page.type)
1340 guest_page_to_ppgtt_spt(gpt)->guest_page_type) 1318 && spt->guest_page.write_cnt >= 2;
1341 && gpt->write_cnt >= 2;
1342} 1319}
1343 1320
1344static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt, 1321static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt,
@@ -1378,8 +1355,8 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
1378 GTT_ENTRY_NUM_IN_ONE_PAGE) { 1355 GTT_ENTRY_NUM_IN_ONE_PAGE) {
1379 ppgtt_get_guest_entry(spt, &ge, index); 1356 ppgtt_get_guest_entry(spt, &ge, index);
1380 1357
1381 ret = ppgtt_handle_guest_write_page_table( 1358 ret = ppgtt_handle_guest_write_page_table(spt,
1382 &spt->guest_page, &ge, index); 1359 &ge, index);
1383 if (ret) 1360 if (ret)
1384 return ret; 1361 return ret;
1385 clear_bit(index, spt->post_shadow_bitmap); 1362 clear_bit(index, spt->post_shadow_bitmap);
@@ -1390,10 +1367,9 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
1390} 1367}
1391 1368
1392static int ppgtt_handle_guest_write_page_table_bytes( 1369static int ppgtt_handle_guest_write_page_table_bytes(
1393 struct intel_vgpu_guest_page *gpt, 1370 struct intel_vgpu_ppgtt_spt *spt,
1394 u64 pa, void *p_data, int bytes) 1371 u64 pa, void *p_data, int bytes)
1395{ 1372{
1396 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1397 struct intel_vgpu *vgpu = spt->vgpu; 1373 struct intel_vgpu *vgpu = spt->vgpu;
1398 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 1374 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1399 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; 1375 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
@@ -1408,7 +1384,7 @@ static int ppgtt_handle_guest_write_page_table_bytes(
1408 ops->test_pse(&we); 1384 ops->test_pse(&we);
1409 1385
1410 if (bytes == info->gtt_entry_size) { 1386 if (bytes == info->gtt_entry_size) {
1411 ret = ppgtt_handle_guest_write_page_table(gpt, &we, index); 1387 ret = ppgtt_handle_guest_write_page_table(spt, &we, index);
1412 if (ret) 1388 if (ret)
1413 return ret; 1389 return ret;
1414 } else { 1390 } else {
@@ -1416,7 +1392,7 @@ static int ppgtt_handle_guest_write_page_table_bytes(
1416 int type = spt->shadow_page.type; 1392 int type = spt->shadow_page.type;
1417 1393
1418 ppgtt_get_shadow_entry(spt, &se, index); 1394 ppgtt_get_shadow_entry(spt, &se, index);
1419 ret = ppgtt_handle_guest_entry_removal(gpt, &se, index); 1395 ret = ppgtt_handle_guest_entry_removal(spt, &se, index);
1420 if (ret) 1396 if (ret)
1421 return ret; 1397 return ret;
1422 ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn); 1398 ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);
@@ -1428,128 +1404,54 @@ static int ppgtt_handle_guest_write_page_table_bytes(
1428 if (!enable_out_of_sync) 1404 if (!enable_out_of_sync)
1429 return 0; 1405 return 0;
1430 1406
1431 gpt->write_cnt++; 1407 spt->guest_page.write_cnt++;
1432 1408
1433 if (gpt->oos_page) 1409 if (spt->guest_page.oos_page)
1434 ops->set_entry(gpt->oos_page->mem, &we, index, 1410 ops->set_entry(spt->guest_page.oos_page->mem, &we, index,
1435 false, 0, vgpu); 1411 false, 0, vgpu);
1436 1412
1437 if (can_do_out_of_sync(gpt)) { 1413 if (can_do_out_of_sync(spt)) {
1438 if (!gpt->oos_page) 1414 if (!spt->guest_page.oos_page)
1439 ppgtt_allocate_oos_page(vgpu, gpt); 1415 ppgtt_allocate_oos_page(spt);
1440 1416
1441 ret = ppgtt_set_guest_page_oos(vgpu, gpt); 1417 ret = ppgtt_set_guest_page_oos(spt);
1442 if (ret < 0) 1418 if (ret < 0)
1443 return ret; 1419 return ret;
1444 } 1420 }
1445 return 0; 1421 return 0;
1446} 1422}
1447 1423
1448/* 1424static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm)
1449 * mm page table allocation policy for bdw+
1450 * - for ggtt, only virtual page table will be allocated.
1451 * - for ppgtt, dedicated virtual/shadow page table will be allocated.
1452 */
1453static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
1454{
1455 struct intel_vgpu *vgpu = mm->vgpu;
1456 struct intel_gvt *gvt = vgpu->gvt;
1457 const struct intel_gvt_device_info *info = &gvt->device_info;
1458 void *mem;
1459
1460 if (mm->type == INTEL_GVT_MM_PPGTT) {
1461 mm->page_table_entry_cnt = 4;
1462 mm->page_table_entry_size = mm->page_table_entry_cnt *
1463 info->gtt_entry_size;
1464 mem = kzalloc(mm->has_shadow_page_table ?
1465 mm->page_table_entry_size * 2
1466 : mm->page_table_entry_size, GFP_KERNEL);
1467 if (!mem)
1468 return -ENOMEM;
1469 mm->virtual_page_table = mem;
1470 if (!mm->has_shadow_page_table)
1471 return 0;
1472 mm->shadow_page_table = mem + mm->page_table_entry_size;
1473 } else if (mm->type == INTEL_GVT_MM_GGTT) {
1474 mm->page_table_entry_cnt =
1475 (gvt_ggtt_gm_sz(gvt) >> I915_GTT_PAGE_SHIFT);
1476 mm->page_table_entry_size = mm->page_table_entry_cnt *
1477 info->gtt_entry_size;
1478 mem = vzalloc(mm->page_table_entry_size);
1479 if (!mem)
1480 return -ENOMEM;
1481 mm->virtual_page_table = mem;
1482 }
1483 return 0;
1484}
1485
1486static void gen8_mm_free_page_table(struct intel_vgpu_mm *mm)
1487{
1488 if (mm->type == INTEL_GVT_MM_PPGTT) {
1489 kfree(mm->virtual_page_table);
1490 } else if (mm->type == INTEL_GVT_MM_GGTT) {
1491 if (mm->virtual_page_table)
1492 vfree(mm->virtual_page_table);
1493 }
1494 mm->virtual_page_table = mm->shadow_page_table = NULL;
1495}
1496
1497static void invalidate_mm(struct intel_vgpu_mm *mm)
1498{ 1425{
1499 struct intel_vgpu *vgpu = mm->vgpu; 1426 struct intel_vgpu *vgpu = mm->vgpu;
1500 struct intel_gvt *gvt = vgpu->gvt; 1427 struct intel_gvt *gvt = vgpu->gvt;
1501 struct intel_gvt_gtt *gtt = &gvt->gtt; 1428 struct intel_gvt_gtt *gtt = &gvt->gtt;
1502 struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; 1429 struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
1503 struct intel_gvt_gtt_entry se; 1430 struct intel_gvt_gtt_entry se;
1504 int i; 1431 int index;
1505 1432
1506 if (WARN_ON(!mm->has_shadow_page_table || !mm->shadowed)) 1433 if (!mm->ppgtt_mm.shadowed)
1507 return; 1434 return;
1508 1435
1509 for (i = 0; i < mm->page_table_entry_cnt; i++) { 1436 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.shadow_pdps); index++) {
1510 ppgtt_get_shadow_root_entry(mm, &se, i); 1437 ppgtt_get_shadow_root_entry(mm, &se, index);
1438
1511 if (!ops->test_present(&se)) 1439 if (!ops->test_present(&se))
1512 continue; 1440 continue;
1513 ppgtt_invalidate_shadow_page_by_shadow_entry( 1441
1514 vgpu, &se); 1442 ppgtt_invalidate_spt_by_shadow_entry(vgpu, &se);
1515 se.val64 = 0; 1443 se.val64 = 0;
1516 ppgtt_set_shadow_root_entry(mm, &se, i); 1444 ppgtt_set_shadow_root_entry(mm, &se, index);
1517 1445
1518 trace_gpt_change(vgpu->id, "destroy root pointer", 1446 trace_spt_guest_change(vgpu->id, "destroy root pointer",
1519 NULL, se.type, se.val64, i); 1447 NULL, se.type, se.val64, index);
1520 } 1448 }
1521 mm->shadowed = false;
1522}
1523 1449
1524/** 1450 mm->ppgtt_mm.shadowed = false;
1525 * intel_vgpu_destroy_mm - destroy a mm object
1526 * @mm: a kref object
1527 *
1528 * This function is used to destroy a mm object for vGPU
1529 *
1530 */
1531void intel_vgpu_destroy_mm(struct kref *mm_ref)
1532{
1533 struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
1534 struct intel_vgpu *vgpu = mm->vgpu;
1535 struct intel_gvt *gvt = vgpu->gvt;
1536 struct intel_gvt_gtt *gtt = &gvt->gtt;
1537
1538 if (!mm->initialized)
1539 goto out;
1540
1541 list_del(&mm->list);
1542 list_del(&mm->lru_list);
1543
1544 if (mm->has_shadow_page_table)
1545 invalidate_mm(mm);
1546
1547 gtt->mm_free_page_table(mm);
1548out:
1549 kfree(mm);
1550} 1451}
1551 1452
1552static int shadow_mm(struct intel_vgpu_mm *mm) 1453
1454static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm)
1553{ 1455{
1554 struct intel_vgpu *vgpu = mm->vgpu; 1456 struct intel_vgpu *vgpu = mm->vgpu;
1555 struct intel_gvt *gvt = vgpu->gvt; 1457 struct intel_gvt *gvt = vgpu->gvt;
@@ -1557,119 +1459,155 @@ static int shadow_mm(struct intel_vgpu_mm *mm)
1557 struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops; 1459 struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
1558 struct intel_vgpu_ppgtt_spt *spt; 1460 struct intel_vgpu_ppgtt_spt *spt;
1559 struct intel_gvt_gtt_entry ge, se; 1461 struct intel_gvt_gtt_entry ge, se;
1560 int i; 1462 int index, ret;
1561 int ret;
1562 1463
1563 if (WARN_ON(!mm->has_shadow_page_table || mm->shadowed)) 1464 if (mm->ppgtt_mm.shadowed)
1564 return 0; 1465 return 0;
1565 1466
1566 mm->shadowed = true; 1467 mm->ppgtt_mm.shadowed = true;
1468
1469 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.guest_pdps); index++) {
1470 ppgtt_get_guest_root_entry(mm, &ge, index);
1567 1471
1568 for (i = 0; i < mm->page_table_entry_cnt; i++) {
1569 ppgtt_get_guest_root_entry(mm, &ge, i);
1570 if (!ops->test_present(&ge)) 1472 if (!ops->test_present(&ge))
1571 continue; 1473 continue;
1572 1474
1573 trace_gpt_change(vgpu->id, __func__, NULL, 1475 trace_spt_guest_change(vgpu->id, __func__, NULL,
1574 ge.type, ge.val64, i); 1476 ge.type, ge.val64, index);
1575 1477
1576 spt = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge); 1478 spt = ppgtt_populate_spt_by_guest_entry(vgpu, &ge);
1577 if (IS_ERR(spt)) { 1479 if (IS_ERR(spt)) {
1578 gvt_vgpu_err("fail to populate guest root pointer\n"); 1480 gvt_vgpu_err("fail to populate guest root pointer\n");
1579 ret = PTR_ERR(spt); 1481 ret = PTR_ERR(spt);
1580 goto fail; 1482 goto fail;
1581 } 1483 }
1582 ppgtt_generate_shadow_entry(&se, spt, &ge); 1484 ppgtt_generate_shadow_entry(&se, spt, &ge);
1583 ppgtt_set_shadow_root_entry(mm, &se, i); 1485 ppgtt_set_shadow_root_entry(mm, &se, index);
1584 1486
1585 trace_gpt_change(vgpu->id, "populate root pointer", 1487 trace_spt_guest_change(vgpu->id, "populate root pointer",
1586 NULL, se.type, se.val64, i); 1488 NULL, se.type, se.val64, index);
1587 } 1489 }
1490
1588 return 0; 1491 return 0;
1589fail: 1492fail:
1590 invalidate_mm(mm); 1493 invalidate_ppgtt_mm(mm);
1591 return ret; 1494 return ret;
1592} 1495}
1593 1496
1497static struct intel_vgpu_mm *vgpu_alloc_mm(struct intel_vgpu *vgpu)
1498{
1499 struct intel_vgpu_mm *mm;
1500
1501 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
1502 if (!mm)
1503 return NULL;
1504
1505 mm->vgpu = vgpu;
1506 kref_init(&mm->ref);
1507 atomic_set(&mm->pincount, 0);
1508
1509 return mm;
1510}
1511
1512static void vgpu_free_mm(struct intel_vgpu_mm *mm)
1513{
1514 kfree(mm);
1515}
1516
1594/** 1517/**
1595 * intel_vgpu_create_mm - create a mm object for a vGPU 1518 * intel_vgpu_create_ppgtt_mm - create a ppgtt mm object for a vGPU
1596 * @vgpu: a vGPU 1519 * @vgpu: a vGPU
1597 * @mm_type: mm object type, should be PPGTT or GGTT 1520 * @root_entry_type: ppgtt root entry type
1598 * @virtual_page_table: page table root pointers. Could be NULL if user wants 1521 * @pdps: guest pdps.
1599 * to populate shadow later.
1600 * @page_table_level: describe the page table level of the mm object
1601 * @pde_base_index: pde root pointer base in GGTT MMIO.
1602 * 1522 *
1603 * This function is used to create a mm object for a vGPU. 1523 * This function is used to create a ppgtt mm object for a vGPU.
1604 * 1524 *
1605 * Returns: 1525 * Returns:
1606 * Zero on success, negative error code in pointer if failed. 1526 * Zero on success, negative error code in pointer if failed.
1607 */ 1527 */
1608struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu, 1528struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,
1609 int mm_type, void *virtual_page_table, int page_table_level, 1529 intel_gvt_gtt_type_t root_entry_type, u64 pdps[])
1610 u32 pde_base_index)
1611{ 1530{
1612 struct intel_gvt *gvt = vgpu->gvt; 1531 struct intel_gvt *gvt = vgpu->gvt;
1613 struct intel_gvt_gtt *gtt = &gvt->gtt;
1614 struct intel_vgpu_mm *mm; 1532 struct intel_vgpu_mm *mm;
1615 int ret; 1533 int ret;
1616 1534
1617 mm = kzalloc(sizeof(*mm), GFP_KERNEL); 1535 mm = vgpu_alloc_mm(vgpu);
1618 if (!mm) { 1536 if (!mm)
1619 ret = -ENOMEM; 1537 return ERR_PTR(-ENOMEM);
1620 goto fail;
1621 }
1622 1538
1623 mm->type = mm_type; 1539 mm->type = INTEL_GVT_MM_PPGTT;
1624 1540
1625 if (page_table_level == 1) 1541 GEM_BUG_ON(root_entry_type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY &&
1626 mm->page_table_entry_type = GTT_TYPE_GGTT_PTE; 1542 root_entry_type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY);
1627 else if (page_table_level == 3) 1543 mm->ppgtt_mm.root_entry_type = root_entry_type;
1628 mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1629 else if (page_table_level == 4)
1630 mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1631 else {
1632 WARN_ON(1);
1633 ret = -EINVAL;
1634 goto fail;
1635 }
1636 1544
1637 mm->page_table_level = page_table_level; 1545 INIT_LIST_HEAD(&mm->ppgtt_mm.list);
1638 mm->pde_base_index = pde_base_index; 1546 INIT_LIST_HEAD(&mm->ppgtt_mm.lru_list);
1639 1547
1640 mm->vgpu = vgpu; 1548 if (root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
1641 mm->has_shadow_page_table = !!(mm_type == INTEL_GVT_MM_PPGTT); 1549 mm->ppgtt_mm.guest_pdps[0] = pdps[0];
1642 1550 else
1643 kref_init(&mm->ref); 1551 memcpy(mm->ppgtt_mm.guest_pdps, pdps,
1644 atomic_set(&mm->pincount, 0); 1552 sizeof(mm->ppgtt_mm.guest_pdps));
1645 INIT_LIST_HEAD(&mm->list);
1646 INIT_LIST_HEAD(&mm->lru_list);
1647 list_add_tail(&mm->list, &vgpu->gtt.mm_list_head);
1648 1553
1649 ret = gtt->mm_alloc_page_table(mm); 1554 ret = shadow_ppgtt_mm(mm);
1650 if (ret) { 1555 if (ret) {
1651 gvt_vgpu_err("fail to allocate page table for mm\n"); 1556 gvt_vgpu_err("failed to shadow ppgtt mm\n");
1652 goto fail; 1557 vgpu_free_mm(mm);
1558 return ERR_PTR(ret);
1653 } 1559 }
1654 1560
1655 mm->initialized = true; 1561 list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);
1562 list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);
1563 return mm;
1564}
1656 1565
1657 if (virtual_page_table) 1566static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu)
1658 memcpy(mm->virtual_page_table, virtual_page_table, 1567{
1659 mm->page_table_entry_size); 1568 struct intel_vgpu_mm *mm;
1569 unsigned long nr_entries;
1660 1570
1661 if (mm->has_shadow_page_table) { 1571 mm = vgpu_alloc_mm(vgpu);
1662 ret = shadow_mm(mm); 1572 if (!mm)
1663 if (ret) 1573 return ERR_PTR(-ENOMEM);
1664 goto fail; 1574
1665 list_add_tail(&mm->lru_list, &gvt->gtt.mm_lru_list_head); 1575 mm->type = INTEL_GVT_MM_GGTT;
1576
1577 nr_entries = gvt_ggtt_gm_sz(vgpu->gvt) >> I915_GTT_PAGE_SHIFT;
1578 mm->ggtt_mm.virtual_ggtt = vzalloc(nr_entries *
1579 vgpu->gvt->device_info.gtt_entry_size);
1580 if (!mm->ggtt_mm.virtual_ggtt) {
1581 vgpu_free_mm(mm);
1582 return ERR_PTR(-ENOMEM);
1666 } 1583 }
1584
1667 return mm; 1585 return mm;
1668fail: 1586}
1669 gvt_vgpu_err("fail to create mm\n"); 1587
1670 if (mm) 1588/**
1671 intel_gvt_mm_unreference(mm); 1589 * _intel_vgpu_mm_release - destroy a mm object
1672 return ERR_PTR(ret); 1590 * @mm_ref: a kref object
1591 *
1592 * This function is used to destroy a mm object for vGPU
1593 *
1594 */
1595void _intel_vgpu_mm_release(struct kref *mm_ref)
1596{
1597 struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
1598
1599 if (GEM_WARN_ON(atomic_read(&mm->pincount)))
1600 gvt_err("vgpu mm pin count bug detected\n");
1601
1602 if (mm->type == INTEL_GVT_MM_PPGTT) {
1603 list_del(&mm->ppgtt_mm.list);
1604 list_del(&mm->ppgtt_mm.lru_list);
1605 invalidate_ppgtt_mm(mm);
1606 } else {
1607 vfree(mm->ggtt_mm.virtual_ggtt);
1608 }
1609
1610 vgpu_free_mm(mm);
1673} 1611}
1674 1612
1675/** 1613/**
@@ -1680,9 +1618,6 @@ fail:
1680 */ 1618 */
1681void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm) 1619void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
1682{ 1620{
1683 if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
1684 return;
1685
1686 atomic_dec(&mm->pincount); 1621 atomic_dec(&mm->pincount);
1687} 1622}
1688 1623
@@ -1701,36 +1636,34 @@ int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
1701{ 1636{
1702 int ret; 1637 int ret;
1703 1638
1704 if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT)) 1639 atomic_inc(&mm->pincount);
1705 return 0;
1706 1640
1707 if (!mm->shadowed) { 1641 if (mm->type == INTEL_GVT_MM_PPGTT) {
1708 ret = shadow_mm(mm); 1642 ret = shadow_ppgtt_mm(mm);
1709 if (ret) 1643 if (ret)
1710 return ret; 1644 return ret;
1645
1646 list_move_tail(&mm->ppgtt_mm.lru_list,
1647 &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);
1648
1711 } 1649 }
1712 1650
1713 atomic_inc(&mm->pincount);
1714 list_del_init(&mm->lru_list);
1715 list_add_tail(&mm->lru_list, &mm->vgpu->gvt->gtt.mm_lru_list_head);
1716 return 0; 1651 return 0;
1717} 1652}
1718 1653
1719static int reclaim_one_mm(struct intel_gvt *gvt) 1654static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt)
1720{ 1655{
1721 struct intel_vgpu_mm *mm; 1656 struct intel_vgpu_mm *mm;
1722 struct list_head *pos, *n; 1657 struct list_head *pos, *n;
1723 1658
1724 list_for_each_safe(pos, n, &gvt->gtt.mm_lru_list_head) { 1659 list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {
1725 mm = container_of(pos, struct intel_vgpu_mm, lru_list); 1660 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list);
1726 1661
1727 if (mm->type != INTEL_GVT_MM_PPGTT)
1728 continue;
1729 if (atomic_read(&mm->pincount)) 1662 if (atomic_read(&mm->pincount))
1730 continue; 1663 continue;
1731 1664
1732 list_del_init(&mm->lru_list); 1665 list_del_init(&mm->ppgtt_mm.lru_list);
1733 invalidate_mm(mm); 1666 invalidate_ppgtt_mm(mm);
1734 return 1; 1667 return 1;
1735 } 1668 }
1736 return 0; 1669 return 0;
@@ -1746,10 +1679,7 @@ static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
1746 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 1679 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1747 struct intel_vgpu_ppgtt_spt *s; 1680 struct intel_vgpu_ppgtt_spt *s;
1748 1681
1749 if (WARN_ON(!mm->has_shadow_page_table)) 1682 s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e));
1750 return -EINVAL;
1751
1752 s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
1753 if (!s) 1683 if (!s)
1754 return -ENXIO; 1684 return -ENXIO;
1755 1685
@@ -1780,85 +1710,72 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
1780 unsigned long gpa = INTEL_GVT_INVALID_ADDR; 1710 unsigned long gpa = INTEL_GVT_INVALID_ADDR;
1781 unsigned long gma_index[4]; 1711 unsigned long gma_index[4];
1782 struct intel_gvt_gtt_entry e; 1712 struct intel_gvt_gtt_entry e;
1783 int i, index; 1713 int i, levels = 0;
1784 int ret; 1714 int ret;
1785 1715
1786 if (mm->type != INTEL_GVT_MM_GGTT && mm->type != INTEL_GVT_MM_PPGTT) 1716 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT &&
1787 return INTEL_GVT_INVALID_ADDR; 1717 mm->type != INTEL_GVT_MM_PPGTT);
1788 1718
1789 if (mm->type == INTEL_GVT_MM_GGTT) { 1719 if (mm->type == INTEL_GVT_MM_GGTT) {
1790 if (!vgpu_gmadr_is_valid(vgpu, gma)) 1720 if (!vgpu_gmadr_is_valid(vgpu, gma))
1791 goto err; 1721 goto err;
1792 1722
1793 ret = ggtt_get_guest_entry(mm, &e, 1723 ggtt_get_guest_entry(mm, &e,
1794 gma_ops->gma_to_ggtt_pte_index(gma)); 1724 gma_ops->gma_to_ggtt_pte_index(gma));
1795 if (ret) 1725
1796 goto err;
1797 gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) 1726 gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT)
1798 + (gma & ~I915_GTT_PAGE_MASK); 1727 + (gma & ~I915_GTT_PAGE_MASK);
1799 1728
1800 trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa); 1729 trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
1801 return gpa; 1730 } else {
1802 } 1731 switch (mm->ppgtt_mm.root_entry_type) {
1803 1732 case GTT_TYPE_PPGTT_ROOT_L4_ENTRY:
1804 switch (mm->page_table_level) { 1733 ppgtt_get_shadow_root_entry(mm, &e, 0);
1805 case 4: 1734
1806 ret = ppgtt_get_shadow_root_entry(mm, &e, 0); 1735 gma_index[0] = gma_ops->gma_to_pml4_index(gma);
1807 if (ret) 1736 gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma);
1808 goto err; 1737 gma_index[2] = gma_ops->gma_to_pde_index(gma);
1809 gma_index[0] = gma_ops->gma_to_pml4_index(gma); 1738 gma_index[3] = gma_ops->gma_to_pte_index(gma);
1810 gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma); 1739 levels = 4;
1811 gma_index[2] = gma_ops->gma_to_pde_index(gma); 1740 break;
1812 gma_index[3] = gma_ops->gma_to_pte_index(gma); 1741 case GTT_TYPE_PPGTT_ROOT_L3_ENTRY:
1813 index = 4; 1742 ppgtt_get_shadow_root_entry(mm, &e,
1814 break; 1743 gma_ops->gma_to_l3_pdp_index(gma));
1815 case 3: 1744
1816 ret = ppgtt_get_shadow_root_entry(mm, &e, 1745 gma_index[0] = gma_ops->gma_to_pde_index(gma);
1817 gma_ops->gma_to_l3_pdp_index(gma)); 1746 gma_index[1] = gma_ops->gma_to_pte_index(gma);
1818 if (ret) 1747 levels = 2;
1819 goto err; 1748 break;
1820 gma_index[0] = gma_ops->gma_to_pde_index(gma); 1749 default:
1821 gma_index[1] = gma_ops->gma_to_pte_index(gma); 1750 GEM_BUG_ON(1);
1822 index = 2; 1751 }
1823 break;
1824 case 2:
1825 ret = ppgtt_get_shadow_root_entry(mm, &e,
1826 gma_ops->gma_to_pde_index(gma));
1827 if (ret)
1828 goto err;
1829 gma_index[0] = gma_ops->gma_to_pte_index(gma);
1830 index = 1;
1831 break;
1832 default:
1833 WARN_ON(1);
1834 goto err;
1835 }
1836 1752
1837 /* walk into the shadow page table and get gpa from guest entry */ 1753 /* walk the shadow page table and get gpa from guest entry */
1838 for (i = 0; i < index; i++) { 1754 for (i = 0; i < levels; i++) {
1839 ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i], 1755 ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i],
1840 (i == index - 1)); 1756 (i == levels - 1));
1841 if (ret) 1757 if (ret)
1842 goto err; 1758 goto err;
1843 1759
1844 if (!pte_ops->test_present(&e)) { 1760 if (!pte_ops->test_present(&e)) {
1845 gvt_dbg_core("GMA 0x%lx is not present\n", gma); 1761 gvt_dbg_core("GMA 0x%lx is not present\n", gma);
1846 goto err; 1762 goto err;
1763 }
1847 } 1764 }
1848 }
1849 1765
1850 gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) 1766 gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT) +
1851 + (gma & ~I915_GTT_PAGE_MASK); 1767 (gma & ~I915_GTT_PAGE_MASK);
1768 trace_gma_translate(vgpu->id, "ppgtt", 0,
1769 mm->ppgtt_mm.root_entry_type, gma, gpa);
1770 }
1852 1771
1853 trace_gma_translate(vgpu->id, "ppgtt", 0,
1854 mm->page_table_level, gma, gpa);
1855 return gpa; 1772 return gpa;
1856err: 1773err:
1857 gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma); 1774 gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma);
1858 return INTEL_GVT_INVALID_ADDR; 1775 return INTEL_GVT_INVALID_ADDR;
1859} 1776}
1860 1777
1861static int emulate_gtt_mmio_read(struct intel_vgpu *vgpu, 1778static int emulate_ggtt_mmio_read(struct intel_vgpu *vgpu,
1862 unsigned int off, void *p_data, unsigned int bytes) 1779 unsigned int off, void *p_data, unsigned int bytes)
1863{ 1780{
1864 struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm; 1781 struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
@@ -1887,7 +1804,7 @@ static int emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
1887 * Returns: 1804 * Returns:
1888 * Zero on success, error code if failed. 1805 * Zero on success, error code if failed.
1889 */ 1806 */
1890int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off, 1807int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
1891 void *p_data, unsigned int bytes) 1808 void *p_data, unsigned int bytes)
1892{ 1809{
1893 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; 1810 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
@@ -1897,11 +1814,11 @@ int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
1897 return -EINVAL; 1814 return -EINVAL;
1898 1815
1899 off -= info->gtt_start_offset; 1816 off -= info->gtt_start_offset;
1900 ret = emulate_gtt_mmio_read(vgpu, off, p_data, bytes); 1817 ret = emulate_ggtt_mmio_read(vgpu, off, p_data, bytes);
1901 return ret; 1818 return ret;
1902} 1819}
1903 1820
1904static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, 1821static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1905 void *p_data, unsigned int bytes) 1822 void *p_data, unsigned int bytes)
1906{ 1823{
1907 struct intel_gvt *gvt = vgpu->gvt; 1824 struct intel_gvt *gvt = vgpu->gvt;
@@ -1911,6 +1828,7 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1911 unsigned long g_gtt_index = off >> info->gtt_entry_size_shift; 1828 unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
1912 unsigned long gma, gfn; 1829 unsigned long gma, gfn;
1913 struct intel_gvt_gtt_entry e, m; 1830 struct intel_gvt_gtt_entry e, m;
1831 dma_addr_t dma_addr;
1914 int ret; 1832 int ret;
1915 1833
1916 if (bytes != 4 && bytes != 8) 1834 if (bytes != 4 && bytes != 8)
@@ -1926,6 +1844,7 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1926 1844
1927 memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data, 1845 memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data,
1928 bytes); 1846 bytes);
1847 m = e;
1929 1848
1930 if (ops->test_present(&e)) { 1849 if (ops->test_present(&e)) {
1931 gfn = ops->get_pfn(&e); 1850 gfn = ops->get_pfn(&e);
@@ -1938,29 +1857,29 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1938 goto out; 1857 goto out;
1939 } 1858 }
1940 1859
1941 ret = gtt_entry_p2m(vgpu, &e, &m); 1860 ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn,
1861 &dma_addr);
1942 if (ret) { 1862 if (ret) {
1943 gvt_vgpu_err("fail to translate guest gtt entry\n"); 1863 gvt_vgpu_err("fail to populate guest ggtt entry\n");
1944 /* guest driver may read/write the entry when partial 1864 /* guest driver may read/write the entry when partial
1945 * update the entry in this situation p2m will fail 1865 * update the entry in this situation p2m will fail
1946 * settting the shadow entry to point to a scratch page 1866 * settting the shadow entry to point to a scratch page
1947 */ 1867 */
1948 ops->set_pfn(&m, gvt->gtt.scratch_mfn); 1868 ops->set_pfn(&m, gvt->gtt.scratch_mfn);
1949 } 1869 } else
1950 } else { 1870 ops->set_pfn(&m, dma_addr >> PAGE_SHIFT);
1951 m = e; 1871 } else
1952 ops->set_pfn(&m, gvt->gtt.scratch_mfn); 1872 ops->set_pfn(&m, gvt->gtt.scratch_mfn);
1953 }
1954 1873
1955out: 1874out:
1956 ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index); 1875 ggtt_set_host_entry(ggtt_mm, &m, g_gtt_index);
1957 gtt_invalidate(gvt->dev_priv); 1876 ggtt_invalidate(gvt->dev_priv);
1958 ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index); 1877 ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
1959 return 0; 1878 return 0;
1960} 1879}
1961 1880
1962/* 1881/*
1963 * intel_vgpu_emulate_gtt_mmio_write - emulate GTT MMIO register write 1882 * intel_vgpu_emulate_ggtt_mmio_write - emulate GTT MMIO register write
1964 * @vgpu: a vGPU 1883 * @vgpu: a vGPU
1965 * @off: register offset 1884 * @off: register offset
1966 * @p_data: data from guest write 1885 * @p_data: data from guest write
@@ -1971,8 +1890,8 @@ out:
1971 * Returns: 1890 * Returns:
1972 * Zero on success, error code if failed. 1891 * Zero on success, error code if failed.
1973 */ 1892 */
1974int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, 1893int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu,
1975 void *p_data, unsigned int bytes) 1894 unsigned int off, void *p_data, unsigned int bytes)
1976{ 1895{
1977 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; 1896 const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1978 int ret; 1897 int ret;
@@ -1981,43 +1900,10 @@ int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1981 return -EINVAL; 1900 return -EINVAL;
1982 1901
1983 off -= info->gtt_start_offset; 1902 off -= info->gtt_start_offset;
1984 ret = emulate_gtt_mmio_write(vgpu, off, p_data, bytes); 1903 ret = emulate_ggtt_mmio_write(vgpu, off, p_data, bytes);
1985 return ret; 1904 return ret;
1986} 1905}
1987 1906
1988int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa,
1989 void *p_data, unsigned int bytes)
1990{
1991 struct intel_gvt *gvt = vgpu->gvt;
1992 int ret = 0;
1993
1994 if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) {
1995 struct intel_vgpu_page_track *t;
1996
1997 mutex_lock(&gvt->lock);
1998
1999 t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT);
2000 if (t) {
2001 if (unlikely(vgpu->failsafe)) {
2002 /* remove write protection to prevent furture traps */
2003 intel_vgpu_clean_page_track(vgpu, t);
2004 } else {
2005 ret = t->handler(t, pa, p_data, bytes);
2006 if (ret) {
2007 gvt_err("guest page write error %d, "
2008 "gfn 0x%lx, pa 0x%llx, "
2009 "var 0x%x, len %d\n",
2010 ret, t->gfn, pa,
2011 *(u32 *)p_data, bytes);
2012 }
2013 }
2014 }
2015 mutex_unlock(&gvt->lock);
2016 }
2017 return ret;
2018}
2019
2020
2021static int alloc_scratch_pages(struct intel_vgpu *vgpu, 1907static int alloc_scratch_pages(struct intel_vgpu *vgpu,
2022 intel_gvt_gtt_type_t type) 1908 intel_gvt_gtt_type_t type)
2023{ 1909{
@@ -2131,45 +2017,49 @@ err:
2131int intel_vgpu_init_gtt(struct intel_vgpu *vgpu) 2017int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
2132{ 2018{
2133 struct intel_vgpu_gtt *gtt = &vgpu->gtt; 2019 struct intel_vgpu_gtt *gtt = &vgpu->gtt;
2134 struct intel_vgpu_mm *ggtt_mm;
2135 2020
2136 hash_init(gtt->tracked_guest_page_hash_table); 2021 INIT_RADIX_TREE(&gtt->spt_tree, GFP_KERNEL);
2137 hash_init(gtt->shadow_page_hash_table);
2138 2022
2139 INIT_LIST_HEAD(&gtt->mm_list_head); 2023 INIT_LIST_HEAD(&gtt->ppgtt_mm_list_head);
2140 INIT_LIST_HEAD(&gtt->oos_page_list_head); 2024 INIT_LIST_HEAD(&gtt->oos_page_list_head);
2141 INIT_LIST_HEAD(&gtt->post_shadow_list_head); 2025 INIT_LIST_HEAD(&gtt->post_shadow_list_head);
2142 2026
2143 intel_vgpu_reset_ggtt(vgpu); 2027 gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);
2144 2028 if (IS_ERR(gtt->ggtt_mm)) {
2145 ggtt_mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_GGTT,
2146 NULL, 1, 0);
2147 if (IS_ERR(ggtt_mm)) {
2148 gvt_vgpu_err("fail to create mm for ggtt.\n"); 2029 gvt_vgpu_err("fail to create mm for ggtt.\n");
2149 return PTR_ERR(ggtt_mm); 2030 return PTR_ERR(gtt->ggtt_mm);
2150 } 2031 }
2151 2032
2152 gtt->ggtt_mm = ggtt_mm; 2033 intel_vgpu_reset_ggtt(vgpu);
2153 2034
2154 return create_scratch_page_tree(vgpu); 2035 return create_scratch_page_tree(vgpu);
2155} 2036}
2156 2037
2157static void intel_vgpu_free_mm(struct intel_vgpu *vgpu, int type) 2038static void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu)
2158{ 2039{
2159 struct list_head *pos, *n; 2040 struct list_head *pos, *n;
2160 struct intel_vgpu_mm *mm; 2041 struct intel_vgpu_mm *mm;
2161 2042
2162 list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) { 2043 list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {
2163 mm = container_of(pos, struct intel_vgpu_mm, list); 2044 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
2164 if (mm->type == type) { 2045 intel_vgpu_destroy_mm(mm);
2165 vgpu->gvt->gtt.mm_free_page_table(mm); 2046 }
2166 list_del(&mm->list); 2047
2167 list_del(&mm->lru_list); 2048 if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))
2168 kfree(mm); 2049 gvt_err("vgpu ppgtt mm is not fully destoried\n");
2169 } 2050
2051 if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {
2052 gvt_err("Why we still has spt not freed?\n");
2053 ppgtt_free_all_spt(vgpu);
2170 } 2054 }
2171} 2055}
2172 2056
2057static void intel_vgpu_destroy_ggtt_mm(struct intel_vgpu *vgpu)
2058{
2059 intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);
2060 vgpu->gtt.ggtt_mm = NULL;
2061}
2062
2173/** 2063/**
2174 * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization 2064 * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization
2175 * @vgpu: a vGPU 2065 * @vgpu: a vGPU
@@ -2182,11 +2072,9 @@ static void intel_vgpu_free_mm(struct intel_vgpu *vgpu, int type)
2182 */ 2072 */
2183void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu) 2073void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
2184{ 2074{
2185 ppgtt_free_all_shadow_page(vgpu); 2075 intel_vgpu_destroy_all_ppgtt_mm(vgpu);
2076 intel_vgpu_destroy_ggtt_mm(vgpu);
2186 release_scratch_page_tree(vgpu); 2077 release_scratch_page_tree(vgpu);
2187
2188 intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
2189 intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_GGTT);
2190} 2078}
2191 2079
2192static void clean_spt_oos(struct intel_gvt *gvt) 2080static void clean_spt_oos(struct intel_gvt *gvt)
@@ -2248,99 +2136,78 @@ fail:
2248 * pointer to mm object on success, NULL if failed. 2136 * pointer to mm object on success, NULL if failed.
2249 */ 2137 */
2250struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu, 2138struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
2251 int page_table_level, void *root_entry) 2139 u64 pdps[])
2252{ 2140{
2253 struct list_head *pos;
2254 struct intel_vgpu_mm *mm; 2141 struct intel_vgpu_mm *mm;
2255 u64 *src, *dst; 2142 struct list_head *pos;
2256
2257 list_for_each(pos, &vgpu->gtt.mm_list_head) {
2258 mm = container_of(pos, struct intel_vgpu_mm, list);
2259 if (mm->type != INTEL_GVT_MM_PPGTT)
2260 continue;
2261
2262 if (mm->page_table_level != page_table_level)
2263 continue;
2264 2143
2265 src = root_entry; 2144 list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {
2266 dst = mm->virtual_page_table; 2145 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
2267 2146
2268 if (page_table_level == 3) { 2147 switch (mm->ppgtt_mm.root_entry_type) {
2269 if (src[0] == dst[0] 2148 case GTT_TYPE_PPGTT_ROOT_L4_ENTRY:
2270 && src[1] == dst[1] 2149 if (pdps[0] == mm->ppgtt_mm.guest_pdps[0])
2271 && src[2] == dst[2]
2272 && src[3] == dst[3])
2273 return mm; 2150 return mm;
2274 } else { 2151 break;
2275 if (src[0] == dst[0]) 2152 case GTT_TYPE_PPGTT_ROOT_L3_ENTRY:
2153 if (!memcmp(pdps, mm->ppgtt_mm.guest_pdps,
2154 sizeof(mm->ppgtt_mm.guest_pdps)))
2276 return mm; 2155 return mm;
2156 break;
2157 default:
2158 GEM_BUG_ON(1);
2277 } 2159 }
2278 } 2160 }
2279 return NULL; 2161 return NULL;
2280} 2162}
2281 2163
2282/** 2164/**
2283 * intel_vgpu_g2v_create_ppgtt_mm - create a PPGTT mm object from 2165 * intel_vgpu_get_ppgtt_mm - get or create a PPGTT mm object.
2284 * g2v notification
2285 * @vgpu: a vGPU 2166 * @vgpu: a vGPU
2286 * @page_table_level: PPGTT page table level 2167 * @root_entry_type: ppgtt root entry type
2168 * @pdps: guest pdps
2287 * 2169 *
2288 * This function is used to create a PPGTT mm object from a guest to GVT-g 2170 * This function is used to find or create a PPGTT mm object from a guest.
2289 * notification.
2290 * 2171 *
2291 * Returns: 2172 * Returns:
2292 * Zero on success, negative error code if failed. 2173 * Zero on success, negative error code if failed.
2293 */ 2174 */
2294int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu, 2175struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu,
2295 int page_table_level) 2176 intel_gvt_gtt_type_t root_entry_type, u64 pdps[])
2296{ 2177{
2297 u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0]));
2298 struct intel_vgpu_mm *mm; 2178 struct intel_vgpu_mm *mm;
2299 2179
2300 if (WARN_ON((page_table_level != 4) && (page_table_level != 3))) 2180 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);
2301 return -EINVAL;
2302
2303 mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
2304 if (mm) { 2181 if (mm) {
2305 intel_gvt_mm_reference(mm); 2182 intel_vgpu_mm_get(mm);
2306 } else { 2183 } else {
2307 mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_PPGTT, 2184 mm = intel_vgpu_create_ppgtt_mm(vgpu, root_entry_type, pdps);
2308 pdp, page_table_level, 0); 2185 if (IS_ERR(mm))
2309 if (IS_ERR(mm)) {
2310 gvt_vgpu_err("fail to create mm\n"); 2186 gvt_vgpu_err("fail to create mm\n");
2311 return PTR_ERR(mm);
2312 }
2313 } 2187 }
2314 return 0; 2188 return mm;
2315} 2189}
2316 2190
2317/** 2191/**
2318 * intel_vgpu_g2v_destroy_ppgtt_mm - destroy a PPGTT mm object from 2192 * intel_vgpu_put_ppgtt_mm - find and put a PPGTT mm object.
2319 * g2v notification
2320 * @vgpu: a vGPU 2193 * @vgpu: a vGPU
2321 * @page_table_level: PPGTT page table level 2194 * @pdps: guest pdps
2322 * 2195 *
2323 * This function is used to create a PPGTT mm object from a guest to GVT-g 2196 * This function is used to find a PPGTT mm object from a guest and destroy it.
2324 * notification.
2325 * 2197 *
2326 * Returns: 2198 * Returns:
2327 * Zero on success, negative error code if failed. 2199 * Zero on success, negative error code if failed.
2328 */ 2200 */
2329int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu, 2201int intel_vgpu_put_ppgtt_mm(struct intel_vgpu *vgpu, u64 pdps[])
2330 int page_table_level)
2331{ 2202{
2332 u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0]));
2333 struct intel_vgpu_mm *mm; 2203 struct intel_vgpu_mm *mm;
2334 2204
2335 if (WARN_ON((page_table_level != 4) && (page_table_level != 3))) 2205 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps);
2336 return -EINVAL;
2337
2338 mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
2339 if (!mm) { 2206 if (!mm) {
2340 gvt_vgpu_err("fail to find ppgtt instance.\n"); 2207 gvt_vgpu_err("fail to find ppgtt instance.\n");
2341 return -EINVAL; 2208 return -EINVAL;
2342 } 2209 }
2343 intel_gvt_mm_unreference(mm); 2210 intel_vgpu_mm_put(mm);
2344 return 0; 2211 return 0;
2345} 2212}
2346 2213
@@ -2367,8 +2234,6 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
2367 || IS_KABYLAKE(gvt->dev_priv)) { 2234 || IS_KABYLAKE(gvt->dev_priv)) {
2368 gvt->gtt.pte_ops = &gen8_gtt_pte_ops; 2235 gvt->gtt.pte_ops = &gen8_gtt_pte_ops;
2369 gvt->gtt.gma_ops = &gen8_gtt_gma_ops; 2236 gvt->gtt.gma_ops = &gen8_gtt_gma_ops;
2370 gvt->gtt.mm_alloc_page_table = gen8_mm_alloc_page_table;
2371 gvt->gtt.mm_free_page_table = gen8_mm_free_page_table;
2372 } else { 2237 } else {
2373 return -ENODEV; 2238 return -ENODEV;
2374 } 2239 }
@@ -2399,7 +2264,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
2399 return ret; 2264 return ret;
2400 } 2265 }
2401 } 2266 }
2402 INIT_LIST_HEAD(&gvt->gtt.mm_lru_list_head); 2267 INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);
2403 return 0; 2268 return 0;
2404} 2269}
2405 2270
@@ -2437,28 +2302,25 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
2437{ 2302{
2438 struct intel_gvt *gvt = vgpu->gvt; 2303 struct intel_gvt *gvt = vgpu->gvt;
2439 struct drm_i915_private *dev_priv = gvt->dev_priv; 2304 struct drm_i915_private *dev_priv = gvt->dev_priv;
2440 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 2305 struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
2306 struct intel_gvt_gtt_entry entry = {.type = GTT_TYPE_GGTT_PTE};
2441 u32 index; 2307 u32 index;
2442 u32 offset;
2443 u32 num_entries; 2308 u32 num_entries;
2444 struct intel_gvt_gtt_entry e;
2445 2309
2446 memset(&e, 0, sizeof(struct intel_gvt_gtt_entry)); 2310 pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);
2447 e.type = GTT_TYPE_GGTT_PTE; 2311 pte_ops->set_present(&entry);
2448 ops->set_pfn(&e, gvt->gtt.scratch_mfn);
2449 e.val64 |= _PAGE_PRESENT;
2450 2312
2451 index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT; 2313 index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;
2452 num_entries = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT; 2314 num_entries = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT;
2453 for (offset = 0; offset < num_entries; offset++) 2315 while (num_entries--)
2454 ops->set_entry(NULL, &e, index + offset, false, 0, vgpu); 2316 ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);
2455 2317
2456 index = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT; 2318 index = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT;
2457 num_entries = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT; 2319 num_entries = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT;
2458 for (offset = 0; offset < num_entries; offset++) 2320 while (num_entries--)
2459 ops->set_entry(NULL, &e, index + offset, false, 0, vgpu); 2321 ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);
2460 2322
2461 gtt_invalidate(dev_priv); 2323 ggtt_invalidate(dev_priv);
2462} 2324}
2463 2325
2464/** 2326/**
@@ -2471,13 +2333,10 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
2471 */ 2333 */
2472void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu) 2334void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
2473{ 2335{
2474 ppgtt_free_all_shadow_page(vgpu);
2475
2476 /* Shadow pages are only created when there is no page 2336 /* Shadow pages are only created when there is no page
2477 * table tracking data, so remove page tracking data after 2337 * table tracking data, so remove page tracking data after
2478 * removing the shadow pages. 2338 * removing the shadow pages.
2479 */ 2339 */
2480 intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT); 2340 intel_vgpu_destroy_all_ppgtt_mm(vgpu);
2481
2482 intel_vgpu_reset_ggtt(vgpu); 2341 intel_vgpu_reset_ggtt(vgpu);
2483} 2342}
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index 4cc13b5934f1..e831507e17c3 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -39,7 +39,6 @@
39 39
40struct intel_vgpu_mm; 40struct intel_vgpu_mm;
41 41
42#define INTEL_GVT_GTT_HASH_BITS 8
43#define INTEL_GVT_INVALID_ADDR (~0UL) 42#define INTEL_GVT_INVALID_ADDR (~0UL)
44 43
45struct intel_gvt_gtt_entry { 44struct intel_gvt_gtt_entry {
@@ -84,17 +83,12 @@ struct intel_gvt_gtt {
84 void (*mm_free_page_table)(struct intel_vgpu_mm *mm); 83 void (*mm_free_page_table)(struct intel_vgpu_mm *mm);
85 struct list_head oos_page_use_list_head; 84 struct list_head oos_page_use_list_head;
86 struct list_head oos_page_free_list_head; 85 struct list_head oos_page_free_list_head;
87 struct list_head mm_lru_list_head; 86 struct list_head ppgtt_mm_lru_list_head;
88 87
89 struct page *scratch_page; 88 struct page *scratch_page;
90 unsigned long scratch_mfn; 89 unsigned long scratch_mfn;
91}; 90};
92 91
93enum {
94 INTEL_GVT_MM_GGTT = 0,
95 INTEL_GVT_MM_PPGTT,
96};
97
98typedef enum { 92typedef enum {
99 GTT_TYPE_INVALID = -1, 93 GTT_TYPE_INVALID = -1,
100 94
@@ -125,66 +119,60 @@ typedef enum {
125 GTT_TYPE_MAX, 119 GTT_TYPE_MAX,
126} intel_gvt_gtt_type_t; 120} intel_gvt_gtt_type_t;
127 121
128struct intel_vgpu_mm { 122enum intel_gvt_mm_type {
129 int type; 123 INTEL_GVT_MM_GGTT,
130 bool initialized; 124 INTEL_GVT_MM_PPGTT,
131 bool shadowed; 125};
132 126
133 int page_table_entry_type; 127#define GVT_RING_CTX_NR_PDPS GEN8_3LVL_PDPES
134 u32 page_table_entry_size;
135 u32 page_table_entry_cnt;
136 void *virtual_page_table;
137 void *shadow_page_table;
138 128
139 int page_table_level; 129struct intel_vgpu_mm {
140 bool has_shadow_page_table; 130 enum intel_gvt_mm_type type;
141 u32 pde_base_index; 131 struct intel_vgpu *vgpu;
142 132
143 struct list_head list;
144 struct kref ref; 133 struct kref ref;
145 atomic_t pincount; 134 atomic_t pincount;
146 struct list_head lru_list;
147 struct intel_vgpu *vgpu;
148};
149
150extern int intel_vgpu_mm_get_entry(
151 struct intel_vgpu_mm *mm,
152 void *page_table, struct intel_gvt_gtt_entry *e,
153 unsigned long index);
154 135
155extern int intel_vgpu_mm_set_entry( 136 union {
156 struct intel_vgpu_mm *mm, 137 struct {
157 void *page_table, struct intel_gvt_gtt_entry *e, 138 intel_gvt_gtt_type_t root_entry_type;
158 unsigned long index); 139 /*
159 140 * The 4 PDPs in ring context. For 48bit addressing,
160#define ggtt_get_guest_entry(mm, e, index) \ 141 * only PDP0 is valid and point to PML4. For 32it
161 intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index) 142 * addressing, all 4 are used as true PDPs.
162 143 */
163#define ggtt_set_guest_entry(mm, e, index) \ 144 u64 guest_pdps[GVT_RING_CTX_NR_PDPS];
164 intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index) 145 u64 shadow_pdps[GVT_RING_CTX_NR_PDPS];
165 146 bool shadowed;
166#define ggtt_get_shadow_entry(mm, e, index) \ 147
167 intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index) 148 struct list_head list;
168 149 struct list_head lru_list;
169#define ggtt_set_shadow_entry(mm, e, index) \ 150 } ppgtt_mm;
170 intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index) 151 struct {
152 void *virtual_ggtt;
153 } ggtt_mm;
154 };
155};
171 156
172#define ppgtt_get_guest_root_entry(mm, e, index) \ 157struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,
173 intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index) 158 intel_gvt_gtt_type_t root_entry_type, u64 pdps[]);
174 159
175#define ppgtt_set_guest_root_entry(mm, e, index) \ 160static inline void intel_vgpu_mm_get(struct intel_vgpu_mm *mm)
176 intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index) 161{
162 kref_get(&mm->ref);
163}
177 164
178#define ppgtt_get_shadow_root_entry(mm, e, index) \ 165void _intel_vgpu_mm_release(struct kref *mm_ref);
179 intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index)
180 166
181#define ppgtt_set_shadow_root_entry(mm, e, index) \ 167static inline void intel_vgpu_mm_put(struct intel_vgpu_mm *mm)
182 intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index) 168{
169 kref_put(&mm->ref, _intel_vgpu_mm_release);
170}
183 171
184extern struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu, 172static inline void intel_vgpu_destroy_mm(struct intel_vgpu_mm *mm)
185 int mm_type, void *virtual_page_table, int page_table_level, 173{
186 u32 pde_base_index); 174 intel_vgpu_mm_put(mm);
187extern void intel_vgpu_destroy_mm(struct kref *mm_ref); 175}
188 176
189struct intel_vgpu_guest_page; 177struct intel_vgpu_guest_page;
190 178
@@ -196,10 +184,8 @@ struct intel_vgpu_scratch_pt {
196struct intel_vgpu_gtt { 184struct intel_vgpu_gtt {
197 struct intel_vgpu_mm *ggtt_mm; 185 struct intel_vgpu_mm *ggtt_mm;
198 unsigned long active_ppgtt_mm_bitmap; 186 unsigned long active_ppgtt_mm_bitmap;
199 struct list_head mm_list_head; 187 struct list_head ppgtt_mm_list_head;
200 DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS); 188 struct radix_tree_root spt_tree;
201 DECLARE_HASHTABLE(tracked_guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
202 atomic_t n_tracked_guest_page;
203 struct list_head oos_page_list_head; 189 struct list_head oos_page_list_head;
204 struct list_head post_shadow_list_head; 190 struct list_head post_shadow_list_head;
205 struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX]; 191 struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX];
@@ -216,32 +202,8 @@ extern void intel_gvt_clean_gtt(struct intel_gvt *gvt);
216extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu, 202extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu,
217 int page_table_level, void *root_entry); 203 int page_table_level, void *root_entry);
218 204
219struct intel_vgpu_oos_page;
220
221struct intel_vgpu_shadow_page {
222 void *vaddr;
223 struct page *page;
224 int type;
225 struct hlist_node node;
226 unsigned long mfn;
227};
228
229struct intel_vgpu_page_track {
230 struct hlist_node node;
231 bool tracked;
232 unsigned long gfn;
233 int (*handler)(void *, u64, void *, int);
234 void *data;
235};
236
237struct intel_vgpu_guest_page {
238 struct intel_vgpu_page_track track;
239 unsigned long write_cnt;
240 struct intel_vgpu_oos_page *oos_page;
241};
242
243struct intel_vgpu_oos_page { 205struct intel_vgpu_oos_page {
244 struct intel_vgpu_guest_page *guest_page; 206 struct intel_vgpu_ppgtt_spt *spt;
245 struct list_head list; 207 struct list_head list;
246 struct list_head vm_list; 208 struct list_head vm_list;
247 int id; 209 int id;
@@ -250,42 +212,33 @@ struct intel_vgpu_oos_page {
250 212
251#define GTT_ENTRY_NUM_IN_ONE_PAGE 512 213#define GTT_ENTRY_NUM_IN_ONE_PAGE 512
252 214
215/* Represent a vgpu shadow page table. */
253struct intel_vgpu_ppgtt_spt { 216struct intel_vgpu_ppgtt_spt {
254 struct intel_vgpu_shadow_page shadow_page;
255 struct intel_vgpu_guest_page guest_page;
256 int guest_page_type;
257 atomic_t refcount; 217 atomic_t refcount;
258 struct intel_vgpu *vgpu; 218 struct intel_vgpu *vgpu;
259 DECLARE_BITMAP(post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE);
260 struct list_head post_shadow_list;
261};
262 219
263int intel_vgpu_init_page_track(struct intel_vgpu *vgpu, 220 struct {
264 struct intel_vgpu_page_track *t, 221 intel_gvt_gtt_type_t type;
265 unsigned long gfn, 222 void *vaddr;
266 int (*handler)(void *gp, u64, void *, int), 223 struct page *page;
267 void *data); 224 unsigned long mfn;
225 } shadow_page;
268 226
269void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu, 227 struct {
270 struct intel_vgpu_page_track *t); 228 intel_gvt_gtt_type_t type;
229 unsigned long gfn;
230 unsigned long write_cnt;
231 struct intel_vgpu_oos_page *oos_page;
232 } guest_page;
271 233
272struct intel_vgpu_page_track *intel_vgpu_find_tracked_page( 234 DECLARE_BITMAP(post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE);
273 struct intel_vgpu *vgpu, unsigned long gfn); 235 struct list_head post_shadow_list;
236};
274 237
275int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu); 238int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu);
276 239
277int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu); 240int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu);
278 241
279static inline void intel_gvt_mm_reference(struct intel_vgpu_mm *mm)
280{
281 kref_get(&mm->ref);
282}
283
284static inline void intel_gvt_mm_unreference(struct intel_vgpu_mm *mm)
285{
286 kref_put(&mm->ref, intel_vgpu_destroy_mm);
287}
288
289int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm); 242int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm);
290 243
291void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm); 244void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm);
@@ -294,21 +247,17 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm,
294 unsigned long gma); 247 unsigned long gma);
295 248
296struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu, 249struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
297 int page_table_level, void *root_entry); 250 u64 pdps[]);
298 251
299int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu, 252struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu,
300 int page_table_level); 253 intel_gvt_gtt_type_t root_entry_type, u64 pdps[]);
301 254
302int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu, 255int intel_vgpu_put_ppgtt_mm(struct intel_vgpu *vgpu, u64 pdps[]);
303 int page_table_level);
304 256
305int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, 257int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu,
306 unsigned int off, void *p_data, unsigned int bytes); 258 unsigned int off, void *p_data, unsigned int bytes);
307 259
308int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, 260int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu,
309 unsigned int off, void *p_data, unsigned int bytes); 261 unsigned int off, void *p_data, unsigned int bytes);
310 262
311int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa,
312 void *p_data, unsigned int bytes);
313
314#endif /* _GVT_GTT_H_ */ 263#endif /* _GVT_GTT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index fac54f32d33f..61bd14fcb649 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -183,7 +183,7 @@ static const struct intel_gvt_ops intel_gvt_ops = {
183 .get_gvt_attrs = intel_get_gvt_attrs, 183 .get_gvt_attrs = intel_get_gvt_attrs,
184 .vgpu_query_plane = intel_vgpu_query_plane, 184 .vgpu_query_plane = intel_vgpu_query_plane,
185 .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, 185 .vgpu_get_dmabuf = intel_vgpu_get_dmabuf,
186 .write_protect_handler = intel_vgpu_write_protect_handler, 186 .write_protect_handler = intel_vgpu_page_track_handler,
187}; 187};
188 188
189/** 189/**
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index c6197d990818..efacd8abbedc 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -48,6 +48,7 @@
48#include "cmd_parser.h" 48#include "cmd_parser.h"
49#include "fb_decoder.h" 49#include "fb_decoder.h"
50#include "dmabuf.h" 50#include "dmabuf.h"
51#include "page_track.h"
51 52
52#define GVT_MAX_VGPU 8 53#define GVT_MAX_VGPU 8
53 54
@@ -131,11 +132,9 @@ struct intel_vgpu_opregion {
131 132
132#define vgpu_opregion(vgpu) (&(vgpu->opregion)) 133#define vgpu_opregion(vgpu) (&(vgpu->opregion))
133 134
134#define INTEL_GVT_MAX_PORT 5
135
136struct intel_vgpu_display { 135struct intel_vgpu_display {
137 struct intel_vgpu_i2c_edid i2c_edid; 136 struct intel_vgpu_i2c_edid i2c_edid;
138 struct intel_vgpu_port ports[INTEL_GVT_MAX_PORT]; 137 struct intel_vgpu_port ports[I915_MAX_PORTS];
139 struct intel_vgpu_sbi sbi; 138 struct intel_vgpu_sbi sbi;
140}; 139};
141 140
@@ -190,6 +189,7 @@ struct intel_vgpu {
190 struct intel_vgpu_opregion opregion; 189 struct intel_vgpu_opregion opregion;
191 struct intel_vgpu_display display; 190 struct intel_vgpu_display display;
192 struct intel_vgpu_submission submission; 191 struct intel_vgpu_submission submission;
192 struct radix_tree_root page_track_tree;
193 u32 hws_pga[I915_NUM_ENGINES]; 193 u32 hws_pga[I915_NUM_ENGINES];
194 194
195 struct dentry *debugfs; 195 struct dentry *debugfs;
@@ -201,8 +201,16 @@ struct intel_vgpu {
201 int num_regions; 201 int num_regions;
202 struct eventfd_ctx *intx_trigger; 202 struct eventfd_ctx *intx_trigger;
203 struct eventfd_ctx *msi_trigger; 203 struct eventfd_ctx *msi_trigger;
204 struct rb_root cache; 204
205 /*
206 * Two caches are used to avoid mapping duplicated pages (eg.
207 * scratch pages). This help to reduce dma setup overhead.
208 */
209 struct rb_root gfn_cache;
210 struct rb_root dma_addr_cache;
211 unsigned long nr_cache_entries;
205 struct mutex cache_lock; 212 struct mutex cache_lock;
213
206 struct notifier_block iommu_notifier; 214 struct notifier_block iommu_notifier;
207 struct notifier_block group_notifier; 215 struct notifier_block group_notifier;
208 struct kvm *kvm; 216 struct kvm *kvm;
@@ -308,7 +316,10 @@ struct intel_gvt {
308 wait_queue_head_t service_thread_wq; 316 wait_queue_head_t service_thread_wq;
309 unsigned long service_request; 317 unsigned long service_request;
310 318
311 struct engine_mmio *engine_mmio_list; 319 struct {
320 struct engine_mmio *mmio;
321 int ctx_mmio_count[I915_NUM_ENGINES];
322 } engine_mmio_list;
312 323
313 struct dentry *debugfs_root; 324 struct dentry *debugfs_root;
314}; 325};
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 9be639aa3b55..112f2ec7c25f 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -188,7 +188,9 @@ void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
188static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu, 188static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
189 unsigned int fence_num, void *p_data, unsigned int bytes) 189 unsigned int fence_num, void *p_data, unsigned int bytes)
190{ 190{
191 if (fence_num >= vgpu_fence_sz(vgpu)) { 191 unsigned int max_fence = vgpu_fence_sz(vgpu);
192
193 if (fence_num >= max_fence) {
192 194
193 /* When guest access oob fence regs without access 195 /* When guest access oob fence regs without access
194 * pv_info first, we treat guest not supporting GVT, 196 * pv_info first, we treat guest not supporting GVT,
@@ -201,7 +203,7 @@ static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
201 if (!vgpu->mmio.disable_warn_untrack) { 203 if (!vgpu->mmio.disable_warn_untrack) {
202 gvt_vgpu_err("found oob fence register access\n"); 204 gvt_vgpu_err("found oob fence register access\n");
203 gvt_vgpu_err("total fence %d, access fence %d\n", 205 gvt_vgpu_err("total fence %d, access fence %d\n",
204 vgpu_fence_sz(vgpu), fence_num); 206 max_fence, fence_num);
205 } 207 }
206 memset(p_data, 0, bytes); 208 memset(p_data, 0, bytes);
207 return -EINVAL; 209 return -EINVAL;
@@ -320,7 +322,7 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
320 intel_gvt_reset_vgpu_locked(vgpu, false, engine_mask); 322 intel_gvt_reset_vgpu_locked(vgpu, false, engine_mask);
321 323
322 /* sw will wait for the device to ack the reset request */ 324 /* sw will wait for the device to ack the reset request */
323 vgpu_vreg(vgpu, offset) = 0; 325 vgpu_vreg(vgpu, offset) = 0;
324 326
325 return 0; 327 return 0;
326} 328}
@@ -1139,21 +1141,21 @@ static int pvinfo_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
1139 1141
1140static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification) 1142static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification)
1141{ 1143{
1142 int ret = 0; 1144 intel_gvt_gtt_type_t root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1145 struct intel_vgpu_mm *mm;
1146 u64 *pdps;
1147
1148 pdps = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0]));
1143 1149
1144 switch (notification) { 1150 switch (notification) {
1145 case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE: 1151 case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE:
1146 ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 3); 1152 root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1147 break;
1148 case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY:
1149 ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 3);
1150 break;
1151 case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE: 1153 case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE:
1152 ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 4); 1154 mm = intel_vgpu_get_ppgtt_mm(vgpu, root_entry_type, pdps);
1153 break; 1155 return PTR_ERR_OR_ZERO(mm);
1156 case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY:
1154 case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY: 1157 case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY:
1155 ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 4); 1158 return intel_vgpu_put_ppgtt_mm(vgpu, pdps);
1156 break;
1157 case VGT_G2V_EXECLIST_CONTEXT_CREATE: 1159 case VGT_G2V_EXECLIST_CONTEXT_CREATE:
1158 case VGT_G2V_EXECLIST_CONTEXT_DESTROY: 1160 case VGT_G2V_EXECLIST_CONTEXT_DESTROY:
1159 case 1: /* Remove this in guest driver. */ 1161 case 1: /* Remove this in guest driver. */
@@ -1161,7 +1163,7 @@ static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification)
1161 default: 1163 default:
1162 gvt_vgpu_err("Invalid PV notification %d\n", notification); 1164 gvt_vgpu_err("Invalid PV notification %d\n", notification);
1163 } 1165 }
1164 return ret; 1166 return 0;
1165} 1167}
1166 1168
1167static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready) 1169static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready)
@@ -1389,8 +1391,8 @@ static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset,
1389 int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset); 1391 int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
1390 1392
1391 if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) { 1393 if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) {
1392 gvt_vgpu_err("VM(%d) write invalid HWSP address, reg:0x%x, value:0x%x\n", 1394 gvt_vgpu_err("write invalid HWSP address, reg:0x%x, value:0x%x\n",
1393 vgpu->id, offset, value); 1395 offset, value);
1394 return -EINVAL; 1396 return -EINVAL;
1395 } 1397 }
1396 /* 1398 /*
@@ -1399,8 +1401,8 @@ static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset,
1399 * support BDW, SKL or other platforms with same HWSP registers. 1401 * support BDW, SKL or other platforms with same HWSP registers.
1400 */ 1402 */
1401 if (unlikely(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) { 1403 if (unlikely(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) {
1402 gvt_vgpu_err("VM(%d) access unknown hardware status page register:0x%x\n", 1404 gvt_vgpu_err("access unknown hardware status page register:0x%x\n",
1403 vgpu->id, offset); 1405 offset);
1404 return -EINVAL; 1406 return -EINVAL;
1405 } 1407 }
1406 vgpu->hws_pga[ring_id] = value; 1408 vgpu->hws_pga[ring_id] = value;
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index f8e77e166246..f6dd9f717888 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -44,13 +44,18 @@ struct intel_gvt_mpt {
44 void (*detach_vgpu)(unsigned long handle); 44 void (*detach_vgpu)(unsigned long handle);
45 int (*inject_msi)(unsigned long handle, u32 addr, u16 data); 45 int (*inject_msi)(unsigned long handle, u32 addr, u16 data);
46 unsigned long (*from_virt_to_mfn)(void *p); 46 unsigned long (*from_virt_to_mfn)(void *p);
47 int (*set_wp_page)(unsigned long handle, u64 gfn); 47 int (*enable_page_track)(unsigned long handle, u64 gfn);
48 int (*unset_wp_page)(unsigned long handle, u64 gfn); 48 int (*disable_page_track)(unsigned long handle, u64 gfn);
49 int (*read_gpa)(unsigned long handle, unsigned long gpa, void *buf, 49 int (*read_gpa)(unsigned long handle, unsigned long gpa, void *buf,
50 unsigned long len); 50 unsigned long len);
51 int (*write_gpa)(unsigned long handle, unsigned long gpa, void *buf, 51 int (*write_gpa)(unsigned long handle, unsigned long gpa, void *buf,
52 unsigned long len); 52 unsigned long len);
53 unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn); 53 unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn);
54
55 int (*dma_map_guest_page)(unsigned long handle, unsigned long gfn,
56 dma_addr_t *dma_addr);
57 void (*dma_unmap_guest_page)(unsigned long handle, dma_addr_t dma_addr);
58
54 int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn, 59 int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn,
55 unsigned long mfn, unsigned int nr, bool map); 60 unsigned long mfn, unsigned int nr, bool map);
56 int (*set_trap_area)(unsigned long handle, u64 start, u64 end, 61 int (*set_trap_area)(unsigned long handle, u64 start, u64 end,
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index 909499b73d03..8a428678e4b5 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -41,6 +41,7 @@
41#include <linux/kvm_host.h> 41#include <linux/kvm_host.h>
42#include <linux/vfio.h> 42#include <linux/vfio.h>
43#include <linux/mdev.h> 43#include <linux/mdev.h>
44#include <linux/debugfs.h>
44 45
45#include "i915_drv.h" 46#include "i915_drv.h"
46#include "gvt.h" 47#include "gvt.h"
@@ -84,12 +85,16 @@ struct kvmgt_guest_info {
84#define NR_BKT (1 << 18) 85#define NR_BKT (1 << 18)
85 struct hlist_head ptable[NR_BKT]; 86 struct hlist_head ptable[NR_BKT];
86#undef NR_BKT 87#undef NR_BKT
88 struct dentry *debugfs_cache_entries;
87}; 89};
88 90
89struct gvt_dma { 91struct gvt_dma {
90 struct rb_node node; 92 struct intel_vgpu *vgpu;
93 struct rb_node gfn_node;
94 struct rb_node dma_addr_node;
91 gfn_t gfn; 95 gfn_t gfn;
92 unsigned long iova; 96 dma_addr_t dma_addr;
97 struct kref ref;
93}; 98};
94 99
95static inline bool handle_valid(unsigned long handle) 100static inline bool handle_valid(unsigned long handle)
@@ -101,165 +106,167 @@ static int kvmgt_guest_init(struct mdev_device *mdev);
101static void intel_vgpu_release_work(struct work_struct *work); 106static void intel_vgpu_release_work(struct work_struct *work);
102static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 107static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
103 108
104static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 109static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
105 unsigned long *iova) 110 dma_addr_t *dma_addr)
106{ 111{
107 struct page *page;
108 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 112 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
109 dma_addr_t daddr; 113 struct page *page;
114 unsigned long pfn;
115 int ret;
110 116
111 if (unlikely(!pfn_valid(pfn))) 117 /* Pin the page first. */
112 return -EFAULT; 118 ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1,
119 IOMMU_READ | IOMMU_WRITE, &pfn);
120 if (ret != 1) {
121 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n",
122 gfn, ret);
123 return -EINVAL;
124 }
113 125
126 /* Setup DMA mapping. */
114 page = pfn_to_page(pfn); 127 page = pfn_to_page(pfn);
115 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 128 *dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE,
116 PCI_DMA_BIDIRECTIONAL); 129 PCI_DMA_BIDIRECTIONAL);
117 if (dma_mapping_error(dev, daddr)) 130 if (dma_mapping_error(dev, *dma_addr)) {
131 gvt_vgpu_err("DMA mapping failed for gfn 0x%lx\n", gfn);
132 vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
118 return -ENOMEM; 133 return -ENOMEM;
134 }
119 135
120 *iova = (unsigned long)(daddr >> PAGE_SHIFT);
121 return 0; 136 return 0;
122} 137}
123 138
124static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 139static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
140 dma_addr_t dma_addr)
125{ 141{
126 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 142 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
127 dma_addr_t daddr; 143 int ret;
128 144
129 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 145 dma_unmap_page(dev, dma_addr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
130 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 146 ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &gfn, 1);
147 WARN_ON(ret != 1);
131} 148}
132 149
133static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 150static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
151 dma_addr_t dma_addr)
134{ 152{
135 struct rb_node *node = vgpu->vdev.cache.rb_node; 153 struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node;
136 struct gvt_dma *ret = NULL; 154 struct gvt_dma *itr;
137 155
138 while (node) { 156 while (node) {
139 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 157 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
140 158
141 if (gfn < itr->gfn) 159 if (dma_addr < itr->dma_addr)
142 node = node->rb_left; 160 node = node->rb_left;
143 else if (gfn > itr->gfn) 161 else if (dma_addr > itr->dma_addr)
144 node = node->rb_right; 162 node = node->rb_right;
145 else { 163 else
146 ret = itr; 164 return itr;
147 goto out;
148 }
149 } 165 }
150 166 return NULL;
151out:
152 return ret;
153} 167}
154 168
155static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 169static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
156{ 170{
157 struct gvt_dma *entry; 171 struct rb_node *node = vgpu->vdev.gfn_cache.rb_node;
158 unsigned long iova; 172 struct gvt_dma *itr;
159
160 mutex_lock(&vgpu->vdev.cache_lock);
161 173
162 entry = __gvt_cache_find(vgpu, gfn); 174 while (node) {
163 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 175 itr = rb_entry(node, struct gvt_dma, gfn_node);
164 176
165 mutex_unlock(&vgpu->vdev.cache_lock); 177 if (gfn < itr->gfn)
166 return iova; 178 node = node->rb_left;
179 else if (gfn > itr->gfn)
180 node = node->rb_right;
181 else
182 return itr;
183 }
184 return NULL;
167} 185}
168 186
169static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 187static void __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
170 unsigned long iova) 188 dma_addr_t dma_addr)
171{ 189{
172 struct gvt_dma *new, *itr; 190 struct gvt_dma *new, *itr;
173 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 191 struct rb_node **link, *parent = NULL;
174 192
175 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 193 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
176 if (!new) 194 if (!new)
177 return; 195 return;
178 196
197 new->vgpu = vgpu;
179 new->gfn = gfn; 198 new->gfn = gfn;
180 new->iova = iova; 199 new->dma_addr = dma_addr;
200 kref_init(&new->ref);
181 201
182 mutex_lock(&vgpu->vdev.cache_lock); 202 /* gfn_cache maps gfn to struct gvt_dma. */
203 link = &vgpu->vdev.gfn_cache.rb_node;
183 while (*link) { 204 while (*link) {
184 parent = *link; 205 parent = *link;
185 itr = rb_entry(parent, struct gvt_dma, node); 206 itr = rb_entry(parent, struct gvt_dma, gfn_node);
186 207
187 if (gfn == itr->gfn) 208 if (gfn < itr->gfn)
188 goto out;
189 else if (gfn < itr->gfn)
190 link = &parent->rb_left; 209 link = &parent->rb_left;
191 else 210 else
192 link = &parent->rb_right; 211 link = &parent->rb_right;
193 } 212 }
213 rb_link_node(&new->gfn_node, parent, link);
214 rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache);
194 215
195 rb_link_node(&new->node, parent, link); 216 /* dma_addr_cache maps dma addr to struct gvt_dma. */
196 rb_insert_color(&new->node, &vgpu->vdev.cache); 217 parent = NULL;
197 mutex_unlock(&vgpu->vdev.cache_lock); 218 link = &vgpu->vdev.dma_addr_cache.rb_node;
198 return; 219 while (*link) {
220 parent = *link;
221 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
199 222
200out: 223 if (dma_addr < itr->dma_addr)
201 mutex_unlock(&vgpu->vdev.cache_lock); 224 link = &parent->rb_left;
202 kfree(new); 225 else
226 link = &parent->rb_right;
227 }
228 rb_link_node(&new->dma_addr_node, parent, link);
229 rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache);
230
231 vgpu->vdev.nr_cache_entries++;
203} 232}
204 233
205static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 234static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
206 struct gvt_dma *entry) 235 struct gvt_dma *entry)
207{ 236{
208 rb_erase(&entry->node, &vgpu->vdev.cache); 237 rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache);
238 rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache);
209 kfree(entry); 239 kfree(entry);
210} 240 vgpu->vdev.nr_cache_entries--;
211
212static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn)
213{
214 struct device *dev = mdev_dev(vgpu->vdev.mdev);
215 struct gvt_dma *this;
216 unsigned long g1;
217 int rc;
218
219 mutex_lock(&vgpu->vdev.cache_lock);
220 this = __gvt_cache_find(vgpu, gfn);
221 if (!this) {
222 mutex_unlock(&vgpu->vdev.cache_lock);
223 return;
224 }
225
226 g1 = gfn;
227 gvt_dma_unmap_iova(vgpu, this->iova);
228 rc = vfio_unpin_pages(dev, &g1, 1);
229 WARN_ON(rc != 1);
230 __gvt_cache_remove_entry(vgpu, this);
231 mutex_unlock(&vgpu->vdev.cache_lock);
232}
233
234static void gvt_cache_init(struct intel_vgpu *vgpu)
235{
236 vgpu->vdev.cache = RB_ROOT;
237 mutex_init(&vgpu->vdev.cache_lock);
238} 241}
239 242
240static void gvt_cache_destroy(struct intel_vgpu *vgpu) 243static void gvt_cache_destroy(struct intel_vgpu *vgpu)
241{ 244{
242 struct gvt_dma *dma; 245 struct gvt_dma *dma;
243 struct rb_node *node = NULL; 246 struct rb_node *node = NULL;
244 struct device *dev = mdev_dev(vgpu->vdev.mdev);
245 unsigned long gfn;
246 247
247 for (;;) { 248 for (;;) {
248 mutex_lock(&vgpu->vdev.cache_lock); 249 mutex_lock(&vgpu->vdev.cache_lock);
249 node = rb_first(&vgpu->vdev.cache); 250 node = rb_first(&vgpu->vdev.gfn_cache);
250 if (!node) { 251 if (!node) {
251 mutex_unlock(&vgpu->vdev.cache_lock); 252 mutex_unlock(&vgpu->vdev.cache_lock);
252 break; 253 break;
253 } 254 }
254 dma = rb_entry(node, struct gvt_dma, node); 255 dma = rb_entry(node, struct gvt_dma, gfn_node);
255 gvt_dma_unmap_iova(vgpu, dma->iova); 256 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr);
256 gfn = dma->gfn;
257 __gvt_cache_remove_entry(vgpu, dma); 257 __gvt_cache_remove_entry(vgpu, dma);
258 mutex_unlock(&vgpu->vdev.cache_lock); 258 mutex_unlock(&vgpu->vdev.cache_lock);
259 vfio_unpin_pages(dev, &gfn, 1);
260 } 259 }
261} 260}
262 261
262static void gvt_cache_init(struct intel_vgpu *vgpu)
263{
264 vgpu->vdev.gfn_cache = RB_ROOT;
265 vgpu->vdev.dma_addr_cache = RB_ROOT;
266 vgpu->vdev.nr_cache_entries = 0;
267 mutex_init(&vgpu->vdev.cache_lock);
268}
269
263static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 270static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
264{ 271{
265 hash_init(info->ptable); 272 hash_init(info->ptable);
@@ -452,7 +459,7 @@ static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
452 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 459 vgpu = intel_gvt_ops->vgpu_create(gvt, type);
453 if (IS_ERR_OR_NULL(vgpu)) { 460 if (IS_ERR_OR_NULL(vgpu)) {
454 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 461 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
455 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 462 gvt_err("failed to create intel vgpu: %d\n", ret);
456 goto out; 463 goto out;
457 } 464 }
458 465
@@ -489,13 +496,22 @@ static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
489 496
490 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 497 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
491 struct vfio_iommu_type1_dma_unmap *unmap = data; 498 struct vfio_iommu_type1_dma_unmap *unmap = data;
492 unsigned long gfn, end_gfn; 499 struct gvt_dma *entry;
500 unsigned long iov_pfn, end_iov_pfn;
493 501
494 gfn = unmap->iova >> PAGE_SHIFT; 502 iov_pfn = unmap->iova >> PAGE_SHIFT;
495 end_gfn = gfn + unmap->size / PAGE_SIZE; 503 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
496 504
497 while (gfn < end_gfn) 505 mutex_lock(&vgpu->vdev.cache_lock);
498 gvt_cache_remove(vgpu, gfn++); 506 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
507 entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
508 if (!entry)
509 continue;
510
511 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr);
512 __gvt_cache_remove_entry(vgpu, entry);
513 }
514 mutex_unlock(&vgpu->vdev.cache_lock);
499 } 515 }
500 516
501 return NOTIFY_OK; 517 return NOTIFY_OK;
@@ -1321,7 +1337,7 @@ static void kvmgt_host_exit(struct device *dev, void *gvt)
1321 mdev_unregister_device(dev); 1337 mdev_unregister_device(dev);
1322} 1338}
1323 1339
1324static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1340static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1325{ 1341{
1326 struct kvmgt_guest_info *info; 1342 struct kvmgt_guest_info *info;
1327 struct kvm *kvm; 1343 struct kvm *kvm;
@@ -1355,7 +1371,7 @@ out:
1355 return 0; 1371 return 0;
1356} 1372}
1357 1373
1358static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1374static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1359{ 1375{
1360 struct kvmgt_guest_info *info; 1376 struct kvmgt_guest_info *info;
1361 struct kvm *kvm; 1377 struct kvm *kvm;
@@ -1483,11 +1499,20 @@ static int kvmgt_guest_init(struct mdev_device *mdev)
1483 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1499 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1484 kvm_page_track_register_notifier(kvm, &info->track_node); 1500 kvm_page_track_register_notifier(kvm, &info->track_node);
1485 1501
1502 info->debugfs_cache_entries = debugfs_create_ulong(
1503 "kvmgt_nr_cache_entries",
1504 0444, vgpu->debugfs,
1505 &vgpu->vdev.nr_cache_entries);
1506 if (!info->debugfs_cache_entries)
1507 gvt_vgpu_err("Cannot create kvmgt debugfs entry\n");
1508
1486 return 0; 1509 return 0;
1487} 1510}
1488 1511
1489static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1512static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1490{ 1513{
1514 debugfs_remove(info->debugfs_cache_entries);
1515
1491 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1516 kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1492 kvm_put_kvm(info->kvm); 1517 kvm_put_kvm(info->kvm);
1493 kvmgt_protect_table_destroy(info); 1518 kvmgt_protect_table_destroy(info);
@@ -1527,39 +1552,77 @@ static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1527 1552
1528static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1553static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1529{ 1554{
1530 unsigned long iova, pfn;
1531 struct kvmgt_guest_info *info; 1555 struct kvmgt_guest_info *info;
1532 struct device *dev; 1556 kvm_pfn_t pfn;
1533 struct intel_vgpu *vgpu;
1534 int rc;
1535 1557
1536 if (!handle_valid(handle)) 1558 if (!handle_valid(handle))
1537 return INTEL_GVT_INVALID_ADDR; 1559 return INTEL_GVT_INVALID_ADDR;
1538 1560
1539 info = (struct kvmgt_guest_info *)handle; 1561 info = (struct kvmgt_guest_info *)handle;
1540 vgpu = info->vgpu; 1562
1541 iova = gvt_cache_find(info->vgpu, gfn); 1563 pfn = gfn_to_pfn(info->kvm, gfn);
1542 if (iova != INTEL_GVT_INVALID_ADDR) 1564 if (is_error_noslot_pfn(pfn))
1543 return iova;
1544
1545 pfn = INTEL_GVT_INVALID_ADDR;
1546 dev = mdev_dev(info->vgpu->vdev.mdev);
1547 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn);
1548 if (rc != 1) {
1549 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n",
1550 gfn, rc);
1551 return INTEL_GVT_INVALID_ADDR;
1552 }
1553 /* transfer to host iova for GFX to use DMA */
1554 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova);
1555 if (rc) {
1556 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn);
1557 vfio_unpin_pages(dev, &gfn, 1);
1558 return INTEL_GVT_INVALID_ADDR; 1565 return INTEL_GVT_INVALID_ADDR;
1566
1567 return pfn;
1568}
1569
1570int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1571 dma_addr_t *dma_addr)
1572{
1573 struct kvmgt_guest_info *info;
1574 struct intel_vgpu *vgpu;
1575 struct gvt_dma *entry;
1576 int ret;
1577
1578 if (!handle_valid(handle))
1579 return -EINVAL;
1580
1581 info = (struct kvmgt_guest_info *)handle;
1582 vgpu = info->vgpu;
1583
1584 mutex_lock(&info->vgpu->vdev.cache_lock);
1585
1586 entry = __gvt_cache_find_gfn(info->vgpu, gfn);
1587 if (!entry) {
1588 ret = gvt_dma_map_page(vgpu, gfn, dma_addr);
1589 if (ret) {
1590 mutex_unlock(&info->vgpu->vdev.cache_lock);
1591 return ret;
1592 }
1593 __gvt_cache_add(info->vgpu, gfn, *dma_addr);
1594 } else {
1595 kref_get(&entry->ref);
1596 *dma_addr = entry->dma_addr;
1559 } 1597 }
1560 1598
1561 gvt_cache_add(info->vgpu, gfn, iova); 1599 mutex_unlock(&info->vgpu->vdev.cache_lock);
1562 return iova; 1600 return 0;
1601}
1602
1603static void __gvt_dma_release(struct kref *ref)
1604{
1605 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1606
1607 gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr);
1608 __gvt_cache_remove_entry(entry->vgpu, entry);
1609}
1610
1611void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
1612{
1613 struct kvmgt_guest_info *info;
1614 struct gvt_dma *entry;
1615
1616 if (!handle_valid(handle))
1617 return;
1618
1619 info = (struct kvmgt_guest_info *)handle;
1620
1621 mutex_lock(&info->vgpu->vdev.cache_lock);
1622 entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
1623 if (entry)
1624 kref_put(&entry->ref, __gvt_dma_release);
1625 mutex_unlock(&info->vgpu->vdev.cache_lock);
1563} 1626}
1564 1627
1565static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1628static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
@@ -1629,11 +1692,13 @@ struct intel_gvt_mpt kvmgt_mpt = {
1629 .detach_vgpu = kvmgt_detach_vgpu, 1692 .detach_vgpu = kvmgt_detach_vgpu,
1630 .inject_msi = kvmgt_inject_msi, 1693 .inject_msi = kvmgt_inject_msi,
1631 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1694 .from_virt_to_mfn = kvmgt_virt_to_pfn,
1632 .set_wp_page = kvmgt_write_protect_add, 1695 .enable_page_track = kvmgt_page_track_add,
1633 .unset_wp_page = kvmgt_write_protect_remove, 1696 .disable_page_track = kvmgt_page_track_remove,
1634 .read_gpa = kvmgt_read_gpa, 1697 .read_gpa = kvmgt_read_gpa,
1635 .write_gpa = kvmgt_write_gpa, 1698 .write_gpa = kvmgt_write_gpa,
1636 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1699 .gfn_to_mfn = kvmgt_gfn_to_pfn,
1700 .dma_map_guest_page = kvmgt_dma_map_guest_page,
1701 .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
1637 .set_opregion = kvmgt_set_opregion, 1702 .set_opregion = kvmgt_set_opregion,
1638 .get_vfio_device = kvmgt_get_vfio_device, 1703 .get_vfio_device = kvmgt_get_vfio_device,
1639 .put_vfio_device = kvmgt_put_vfio_device, 1704 .put_vfio_device = kvmgt_put_vfio_device,
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 5c869e3fdf3b..11b71b33f1c0 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -76,10 +76,9 @@ static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
76 else 76 else
77 intel_vgpu_default_mmio_write(vgpu, offset, p_data, 77 intel_vgpu_default_mmio_write(vgpu, offset, p_data,
78 bytes); 78 bytes);
79 } else if (reg_is_gtt(gvt, offset) && 79 } else if (reg_is_gtt(gvt, offset)) {
80 vgpu->gtt.ggtt_mm->virtual_page_table) {
81 offset -= gvt->device_info.gtt_start_offset; 80 offset -= gvt->device_info.gtt_start_offset;
82 pt = vgpu->gtt.ggtt_mm->virtual_page_table + offset; 81 pt = vgpu->gtt.ggtt_mm->ggtt_mm.virtual_ggtt + offset;
83 if (read) 82 if (read)
84 memcpy(p_data, pt, bytes); 83 memcpy(p_data, pt, bytes);
85 else 84 else
@@ -125,7 +124,7 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
125 if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) 124 if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
126 goto err; 125 goto err;
127 126
128 ret = intel_vgpu_emulate_gtt_mmio_read(vgpu, offset, 127 ret = intel_vgpu_emulate_ggtt_mmio_read(vgpu, offset,
129 p_data, bytes); 128 p_data, bytes);
130 if (ret) 129 if (ret)
131 goto err; 130 goto err;
@@ -198,7 +197,7 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
198 if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) 197 if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
199 goto err; 198 goto err;
200 199
201 ret = intel_vgpu_emulate_gtt_mmio_write(vgpu, offset, 200 ret = intel_vgpu_emulate_ggtt_mmio_write(vgpu, offset,
202 p_data, bytes); 201 p_data, bytes);
203 if (ret) 202 if (ret)
204 goto err; 203 goto err;
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
index 73ad6e90e49d..74a9c7b5516e 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.c
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -50,6 +50,8 @@
50#define RING_GFX_MODE(base) _MMIO((base) + 0x29c) 50#define RING_GFX_MODE(base) _MMIO((base) + 0x29c)
51#define VF_GUARDBAND _MMIO(0x83a4) 51#define VF_GUARDBAND _MMIO(0x83a4)
52 52
53#define GEN9_MOCS_SIZE 64
54
53/* Raw offset is appened to each line for convenience. */ 55/* Raw offset is appened to each line for convenience. */
54static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = { 56static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = {
55 {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */ 57 {RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
@@ -151,8 +153,8 @@ static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = {
151 153
152static struct { 154static struct {
153 bool initialized; 155 bool initialized;
154 u32 control_table[I915_NUM_ENGINES][64]; 156 u32 control_table[I915_NUM_ENGINES][GEN9_MOCS_SIZE];
155 u32 l3cc_table[32]; 157 u32 l3cc_table[GEN9_MOCS_SIZE / 2];
156} gen9_render_mocs; 158} gen9_render_mocs;
157 159
158static void load_render_mocs(struct drm_i915_private *dev_priv) 160static void load_render_mocs(struct drm_i915_private *dev_priv)
@@ -169,7 +171,7 @@ static void load_render_mocs(struct drm_i915_private *dev_priv)
169 171
170 for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) { 172 for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) {
171 offset.reg = regs[ring_id]; 173 offset.reg = regs[ring_id];
172 for (i = 0; i < 64; i++) { 174 for (i = 0; i < GEN9_MOCS_SIZE; i++) {
173 gen9_render_mocs.control_table[ring_id][i] = 175 gen9_render_mocs.control_table[ring_id][i] =
174 I915_READ_FW(offset); 176 I915_READ_FW(offset);
175 offset.reg += 4; 177 offset.reg += 4;
@@ -177,7 +179,7 @@ static void load_render_mocs(struct drm_i915_private *dev_priv)
177 } 179 }
178 180
179 offset.reg = 0xb020; 181 offset.reg = 0xb020;
180 for (i = 0; i < 32; i++) { 182 for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
181 gen9_render_mocs.l3cc_table[i] = 183 gen9_render_mocs.l3cc_table[i] =
182 I915_READ_FW(offset); 184 I915_READ_FW(offset);
183 offset.reg += 4; 185 offset.reg += 4;
@@ -185,6 +187,153 @@ static void load_render_mocs(struct drm_i915_private *dev_priv)
185 gen9_render_mocs.initialized = true; 187 gen9_render_mocs.initialized = true;
186} 188}
187 189
190static int
191restore_context_mmio_for_inhibit(struct intel_vgpu *vgpu,
192 struct i915_request *req)
193{
194 u32 *cs;
195 int ret;
196 struct engine_mmio *mmio;
197 struct intel_gvt *gvt = vgpu->gvt;
198 int ring_id = req->engine->id;
199 int count = gvt->engine_mmio_list.ctx_mmio_count[ring_id];
200
201 if (count == 0)
202 return 0;
203
204 ret = req->engine->emit_flush(req, EMIT_BARRIER);
205 if (ret)
206 return ret;
207
208 cs = intel_ring_begin(req, count * 2 + 2);
209 if (IS_ERR(cs))
210 return PTR_ERR(cs);
211
212 *cs++ = MI_LOAD_REGISTER_IMM(count);
213 for (mmio = gvt->engine_mmio_list.mmio;
214 i915_mmio_reg_valid(mmio->reg); mmio++) {
215 if (mmio->ring_id != ring_id ||
216 !mmio->in_context)
217 continue;
218
219 *cs++ = i915_mmio_reg_offset(mmio->reg);
220 *cs++ = vgpu_vreg_t(vgpu, mmio->reg) |
221 (mmio->mask << 16);
222 gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
223 *(cs-2), *(cs-1), vgpu->id, ring_id);
224 }
225
226 *cs++ = MI_NOOP;
227 intel_ring_advance(req, cs);
228
229 ret = req->engine->emit_flush(req, EMIT_BARRIER);
230 if (ret)
231 return ret;
232
233 return 0;
234}
235
236static int
237restore_render_mocs_control_for_inhibit(struct intel_vgpu *vgpu,
238 struct i915_request *req)
239{
240 unsigned int index;
241 u32 *cs;
242
243 cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE + 2);
244 if (IS_ERR(cs))
245 return PTR_ERR(cs);
246
247 *cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE);
248
249 for (index = 0; index < GEN9_MOCS_SIZE; index++) {
250 *cs++ = i915_mmio_reg_offset(GEN9_GFX_MOCS(index));
251 *cs++ = vgpu_vreg_t(vgpu, GEN9_GFX_MOCS(index));
252 gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
253 *(cs-2), *(cs-1), vgpu->id, req->engine->id);
254
255 }
256
257 *cs++ = MI_NOOP;
258 intel_ring_advance(req, cs);
259
260 return 0;
261}
262
263static int
264restore_render_mocs_l3cc_for_inhibit(struct intel_vgpu *vgpu,
265 struct i915_request *req)
266{
267 unsigned int index;
268 u32 *cs;
269
270 cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE / 2 + 2);
271 if (IS_ERR(cs))
272 return PTR_ERR(cs);
273
274 *cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE / 2);
275
276 for (index = 0; index < GEN9_MOCS_SIZE / 2; index++) {
277 *cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(index));
278 *cs++ = vgpu_vreg_t(vgpu, GEN9_LNCFCMOCS(index));
279 gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
280 *(cs-2), *(cs-1), vgpu->id, req->engine->id);
281
282 }
283
284 *cs++ = MI_NOOP;
285 intel_ring_advance(req, cs);
286
287 return 0;
288}
289
290/*
291 * Use lri command to initialize the mmio which is in context state image for
292 * inhibit context, it contains tracked engine mmio, render_mocs and
293 * render_mocs_l3cc.
294 */
295int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu,
296 struct i915_request *req)
297{
298 int ret;
299 u32 *cs;
300
301 cs = intel_ring_begin(req, 2);
302 if (IS_ERR(cs))
303 return PTR_ERR(cs);
304
305 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
306 *cs++ = MI_NOOP;
307 intel_ring_advance(req, cs);
308
309 ret = restore_context_mmio_for_inhibit(vgpu, req);
310 if (ret)
311 goto out;
312
313 /* no MOCS register in context except render engine */
314 if (req->engine->id != RCS)
315 goto out;
316
317 ret = restore_render_mocs_control_for_inhibit(vgpu, req);
318 if (ret)
319 goto out;
320
321 ret = restore_render_mocs_l3cc_for_inhibit(vgpu, req);
322 if (ret)
323 goto out;
324
325out:
326 cs = intel_ring_begin(req, 2);
327 if (IS_ERR(cs))
328 return PTR_ERR(cs);
329
330 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
331 *cs++ = MI_NOOP;
332 intel_ring_advance(req, cs);
333
334 return ret;
335}
336
188static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) 337static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
189{ 338{
190 struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; 339 struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
@@ -251,11 +400,14 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
251 if (WARN_ON(ring_id >= ARRAY_SIZE(regs))) 400 if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
252 return; 401 return;
253 402
403 if (IS_KABYLAKE(dev_priv) && ring_id == RCS)
404 return;
405
254 if (!pre && !gen9_render_mocs.initialized) 406 if (!pre && !gen9_render_mocs.initialized)
255 load_render_mocs(dev_priv); 407 load_render_mocs(dev_priv);
256 408
257 offset.reg = regs[ring_id]; 409 offset.reg = regs[ring_id];
258 for (i = 0; i < 64; i++) { 410 for (i = 0; i < GEN9_MOCS_SIZE; i++) {
259 if (pre) 411 if (pre)
260 old_v = vgpu_vreg_t(pre, offset); 412 old_v = vgpu_vreg_t(pre, offset);
261 else 413 else
@@ -273,7 +425,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
273 425
274 if (ring_id == RCS) { 426 if (ring_id == RCS) {
275 l3_offset.reg = 0xb020; 427 l3_offset.reg = 0xb020;
276 for (i = 0; i < 32; i++) { 428 for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
277 if (pre) 429 if (pre)
278 old_v = vgpu_vreg_t(pre, l3_offset); 430 old_v = vgpu_vreg_t(pre, l3_offset);
279 else 431 else
@@ -293,6 +445,16 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
293 445
294#define CTX_CONTEXT_CONTROL_VAL 0x03 446#define CTX_CONTEXT_CONTROL_VAL 0x03
295 447
448bool is_inhibit_context(struct i915_gem_context *ctx, int ring_id)
449{
450 u32 *reg_state = ctx->engine[ring_id].lrc_reg_state;
451 u32 inhibit_mask =
452 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
453
454 return inhibit_mask ==
455 (reg_state[CTX_CONTEXT_CONTROL_VAL] & inhibit_mask);
456}
457
296/* Switch ring mmio values (context). */ 458/* Switch ring mmio values (context). */
297static void switch_mmio(struct intel_vgpu *pre, 459static void switch_mmio(struct intel_vgpu *pre,
298 struct intel_vgpu *next, 460 struct intel_vgpu *next,
@@ -300,9 +462,6 @@ static void switch_mmio(struct intel_vgpu *pre,
300{ 462{
301 struct drm_i915_private *dev_priv; 463 struct drm_i915_private *dev_priv;
302 struct intel_vgpu_submission *s; 464 struct intel_vgpu_submission *s;
303 u32 *reg_state, ctx_ctrl;
304 u32 inhibit_mask =
305 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
306 struct engine_mmio *mmio; 465 struct engine_mmio *mmio;
307 u32 old_v, new_v; 466 u32 old_v, new_v;
308 467
@@ -310,10 +469,18 @@ static void switch_mmio(struct intel_vgpu *pre,
310 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) 469 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
311 switch_mocs(pre, next, ring_id); 470 switch_mocs(pre, next, ring_id);
312 471
313 for (mmio = dev_priv->gvt->engine_mmio_list; 472 for (mmio = dev_priv->gvt->engine_mmio_list.mmio;
314 i915_mmio_reg_valid(mmio->reg); mmio++) { 473 i915_mmio_reg_valid(mmio->reg); mmio++) {
315 if (mmio->ring_id != ring_id) 474 if (mmio->ring_id != ring_id)
316 continue; 475 continue;
476 /*
477 * No need to do save or restore of the mmio which is in context
478 * state image on kabylake, it's initialized by lri command and
479 * save or restore with context together.
480 */
481 if (IS_KABYLAKE(dev_priv) && mmio->in_context)
482 continue;
483
317 // save 484 // save
318 if (pre) { 485 if (pre) {
319 vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg); 486 vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg);
@@ -327,16 +494,13 @@ static void switch_mmio(struct intel_vgpu *pre,
327 // restore 494 // restore
328 if (next) { 495 if (next) {
329 s = &next->submission; 496 s = &next->submission;
330 reg_state =
331 s->shadow_ctx->engine[ring_id].lrc_reg_state;
332 ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL];
333 /* 497 /*
334 * if it is an inhibit context, load in_context mmio 498 * No need to restore the mmio which is in context state
335 * into HW by mmio write. If it is not, skip this mmio 499 * image if it's not inhibit context, it will restore
336 * write. 500 * itself.
337 */ 501 */
338 if (mmio->in_context && 502 if (mmio->in_context &&
339 (ctx_ctrl & inhibit_mask) != inhibit_mask) 503 !is_inhibit_context(s->shadow_ctx, ring_id))
340 continue; 504 continue;
341 505
342 if (mmio->mask) 506 if (mmio->mask)
@@ -405,8 +569,16 @@ void intel_gvt_switch_mmio(struct intel_vgpu *pre,
405 */ 569 */
406void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt) 570void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt)
407{ 571{
572 struct engine_mmio *mmio;
573
408 if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv)) 574 if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv))
409 gvt->engine_mmio_list = gen9_engine_mmio_list; 575 gvt->engine_mmio_list.mmio = gen9_engine_mmio_list;
410 else 576 else
411 gvt->engine_mmio_list = gen8_engine_mmio_list; 577 gvt->engine_mmio_list.mmio = gen8_engine_mmio_list;
578
579 for (mmio = gvt->engine_mmio_list.mmio;
580 i915_mmio_reg_valid(mmio->reg); mmio++) {
581 if (mmio->in_context)
582 gvt->engine_mmio_list.ctx_mmio_count[mmio->ring_id]++;
583 }
412} 584}
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.h b/drivers/gpu/drm/i915/gvt/mmio_context.h
index ca2c6a745673..0439eb8057a8 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.h
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.h
@@ -49,4 +49,9 @@ void intel_gvt_switch_mmio(struct intel_vgpu *pre,
49 49
50void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt); 50void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt);
51 51
52bool is_inhibit_context(struct i915_gem_context *ctx, int ring_id);
53
54int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu,
55 struct i915_request *req);
56
52#endif 57#endif
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 81aff4eacbfe..32ffcd566cdd 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -154,54 +154,31 @@ static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p)
154} 154}
155 155
156/** 156/**
157 * intel_gvt_hypervisor_enable - set a guest page to write-protected 157 * intel_gvt_hypervisor_enable_page_track - track a guest page
158 * @vgpu: a vGPU 158 * @vgpu: a vGPU
159 * @t: page track data structure 159 * @gfn: the gfn of guest
160 * 160 *
161 * Returns: 161 * Returns:
162 * Zero on success, negative error code if failed. 162 * Zero on success, negative error code if failed.
163 */ 163 */
164static inline int intel_gvt_hypervisor_enable_page_track( 164static inline int intel_gvt_hypervisor_enable_page_track(
165 struct intel_vgpu *vgpu, 165 struct intel_vgpu *vgpu, unsigned long gfn)
166 struct intel_vgpu_page_track *t)
167{ 166{
168 int ret; 167 return intel_gvt_host.mpt->enable_page_track(vgpu->handle, gfn);
169
170 if (t->tracked)
171 return 0;
172
173 ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, t->gfn);
174 if (ret)
175 return ret;
176 t->tracked = true;
177 atomic_inc(&vgpu->gtt.n_tracked_guest_page);
178 return 0;
179} 168}
180 169
181/** 170/**
182 * intel_gvt_hypervisor_disable_page_track - remove the write-protection of a 171 * intel_gvt_hypervisor_disable_page_track - untrack a guest page
183 * guest page
184 * @vgpu: a vGPU 172 * @vgpu: a vGPU
185 * @t: page track data structure 173 * @gfn: the gfn of guest
186 * 174 *
187 * Returns: 175 * Returns:
188 * Zero on success, negative error code if failed. 176 * Zero on success, negative error code if failed.
189 */ 177 */
190static inline int intel_gvt_hypervisor_disable_page_track( 178static inline int intel_gvt_hypervisor_disable_page_track(
191 struct intel_vgpu *vgpu, 179 struct intel_vgpu *vgpu, unsigned long gfn)
192 struct intel_vgpu_page_track *t)
193{ 180{
194 int ret; 181 return intel_gvt_host.mpt->disable_page_track(vgpu->handle, gfn);
195
196 if (!t->tracked)
197 return 0;
198
199 ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, t->gfn);
200 if (ret)
201 return ret;
202 t->tracked = false;
203 atomic_dec(&vgpu->gtt.n_tracked_guest_page);
204 return 0;
205} 182}
206 183
207/** 184/**
@@ -251,6 +228,34 @@ static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn(
251} 228}
252 229
253/** 230/**
231 * intel_gvt_hypervisor_dma_map_guest_page - setup dma map for guest page
232 * @vgpu: a vGPU
233 * @gpfn: guest pfn
234 * @dma_addr: retrieve allocated dma addr
235 *
236 * Returns:
237 * 0 on success, negative error code if failed.
238 */
239static inline int intel_gvt_hypervisor_dma_map_guest_page(
240 struct intel_vgpu *vgpu, unsigned long gfn,
241 dma_addr_t *dma_addr)
242{
243 return intel_gvt_host.mpt->dma_map_guest_page(vgpu->handle, gfn,
244 dma_addr);
245}
246
247/**
248 * intel_gvt_hypervisor_dma_unmap_guest_page - cancel dma map for guest page
249 * @vgpu: a vGPU
250 * @dma_addr: the mapped dma addr
251 */
252static inline void intel_gvt_hypervisor_dma_unmap_guest_page(
253 struct intel_vgpu *vgpu, dma_addr_t dma_addr)
254{
255 intel_gvt_host.mpt->dma_unmap_guest_page(vgpu->handle, dma_addr);
256}
257
258/**
254 * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN 259 * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN
255 * @vgpu: a vGPU 260 * @vgpu: a vGPU
256 * @gfn: guest PFN 261 * @gfn: guest PFN
diff --git a/drivers/gpu/drm/i915/gvt/page_track.c b/drivers/gpu/drm/i915/gvt/page_track.c
new file mode 100644
index 000000000000..53e2bd79c97d
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/page_track.c
@@ -0,0 +1,184 @@
1/*
2 * Copyright(c) 2011-2017 Intel Corporation. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23#include "i915_drv.h"
24#include "gvt.h"
25
26/**
27 * intel_vgpu_find_page_track - find page track rcord of guest page
28 * @vgpu: a vGPU
29 * @gfn: the gfn of guest page
30 *
31 * Returns:
32 * A pointer to struct intel_vgpu_page_track if found, else NULL returned.
33 */
34struct intel_vgpu_page_track *intel_vgpu_find_page_track(
35 struct intel_vgpu *vgpu, unsigned long gfn)
36{
37 return radix_tree_lookup(&vgpu->page_track_tree, gfn);
38}
39
40/**
41 * intel_vgpu_register_page_track - register a guest page to be tacked
42 * @vgpu: a vGPU
43 * @gfn: the gfn of guest page
44 *
45 * Returns:
46 * zero on success, negative error code if failed.
47 */
48int intel_vgpu_register_page_track(struct intel_vgpu *vgpu, unsigned long gfn,
49 gvt_page_track_handler_t handler, void *priv)
50{
51 struct intel_vgpu_page_track *track;
52 int ret;
53
54 track = intel_vgpu_find_page_track(vgpu, gfn);
55 if (track)
56 return -EEXIST;
57
58 track = kzalloc(sizeof(*track), GFP_KERNEL);
59 if (!track)
60 return -ENOMEM;
61
62 track->handler = handler;
63 track->priv_data = priv;
64
65 ret = radix_tree_insert(&vgpu->page_track_tree, gfn, track);
66 if (ret) {
67 kfree(track);
68 return ret;
69 }
70
71 return 0;
72}
73
74/**
75 * intel_vgpu_unregister_page_track - unregister the tracked guest page
76 * @vgpu: a vGPU
77 * @gfn: the gfn of guest page
78 *
79 */
80void intel_vgpu_unregister_page_track(struct intel_vgpu *vgpu,
81 unsigned long gfn)
82{
83 struct intel_vgpu_page_track *track;
84
85 track = radix_tree_delete(&vgpu->page_track_tree, gfn);
86 if (track) {
87 if (track->tracked)
88 intel_gvt_hypervisor_disable_page_track(vgpu, gfn);
89 kfree(track);
90 }
91}
92
93/**
94 * intel_vgpu_enable_page_track - set write-protection on guest page
95 * @vgpu: a vGPU
96 * @gfn: the gfn of guest page
97 *
98 * Returns:
99 * zero on success, negative error code if failed.
100 */
101int intel_vgpu_enable_page_track(struct intel_vgpu *vgpu, unsigned long gfn)
102{
103 struct intel_vgpu_page_track *track;
104 int ret;
105
106 track = intel_vgpu_find_page_track(vgpu, gfn);
107 if (!track)
108 return -ENXIO;
109
110 if (track->tracked)
111 return 0;
112
113 ret = intel_gvt_hypervisor_enable_page_track(vgpu, gfn);
114 if (ret)
115 return ret;
116 track->tracked = true;
117 return 0;
118}
119
120/**
121 * intel_vgpu_enable_page_track - cancel write-protection on guest page
122 * @vgpu: a vGPU
123 * @gfn: the gfn of guest page
124 *
125 * Returns:
126 * zero on success, negative error code if failed.
127 */
128int intel_vgpu_disable_page_track(struct intel_vgpu *vgpu, unsigned long gfn)
129{
130 struct intel_vgpu_page_track *track;
131 int ret;
132
133 track = intel_vgpu_find_page_track(vgpu, gfn);
134 if (!track)
135 return -ENXIO;
136
137 if (!track->tracked)
138 return 0;
139
140 ret = intel_gvt_hypervisor_disable_page_track(vgpu, gfn);
141 if (ret)
142 return ret;
143 track->tracked = false;
144 return 0;
145}
146
147/**
148 * intel_vgpu_page_track_handler - called when write to write-protected page
149 * @vgpu: a vGPU
150 * @gpa: the gpa of this write
151 * @data: the writed data
152 * @bytes: the length of this write
153 *
154 * Returns:
155 * zero on success, negative error code if failed.
156 */
157int intel_vgpu_page_track_handler(struct intel_vgpu *vgpu, u64 gpa,
158 void *data, unsigned int bytes)
159{
160 struct intel_gvt *gvt = vgpu->gvt;
161 struct intel_vgpu_page_track *page_track;
162 int ret = 0;
163
164 mutex_lock(&gvt->lock);
165
166 page_track = intel_vgpu_find_page_track(vgpu, gpa >> PAGE_SHIFT);
167 if (!page_track) {
168 ret = -ENXIO;
169 goto out;
170 }
171
172 if (unlikely(vgpu->failsafe)) {
173 /* Remove write protection to prevent furture traps. */
174 intel_vgpu_disable_page_track(vgpu, gpa >> PAGE_SHIFT);
175 } else {
176 ret = page_track->handler(page_track, gpa, data, bytes);
177 if (ret)
178 gvt_err("guest page write error, gpa %llx\n", gpa);
179 }
180
181out:
182 mutex_unlock(&gvt->lock);
183 return ret;
184}
diff --git a/drivers/gpu/drm/i915/gvt/page_track.h b/drivers/gpu/drm/i915/gvt/page_track.h
new file mode 100644
index 000000000000..fa607a71c3c0
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/page_track.h
@@ -0,0 +1,56 @@
1/*
2 * Copyright(c) 2011-2017 Intel Corporation. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 */
24
25#ifndef _GVT_PAGE_TRACK_H_
26#define _GVT_PAGE_TRACK_H_
27
28struct intel_vgpu_page_track;
29
30typedef int (*gvt_page_track_handler_t)(
31 struct intel_vgpu_page_track *page_track,
32 u64 gpa, void *data, int bytes);
33
34/* Track record for a write-protected guest page. */
35struct intel_vgpu_page_track {
36 gvt_page_track_handler_t handler;
37 bool tracked;
38 void *priv_data;
39};
40
41struct intel_vgpu_page_track *intel_vgpu_find_page_track(
42 struct intel_vgpu *vgpu, unsigned long gfn);
43
44int intel_vgpu_register_page_track(struct intel_vgpu *vgpu,
45 unsigned long gfn, gvt_page_track_handler_t handler,
46 void *priv);
47void intel_vgpu_unregister_page_track(struct intel_vgpu *vgpu,
48 unsigned long gfn);
49
50int intel_vgpu_enable_page_track(struct intel_vgpu *vgpu, unsigned long gfn);
51int intel_vgpu_disable_page_track(struct intel_vgpu *vgpu, unsigned long gfn);
52
53int intel_vgpu_page_track_handler(struct intel_vgpu *vgpu, u64 gpa,
54 void *data, unsigned int bytes);
55
56#endif
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
index cc1ce361cd76..75b7bc7b344c 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.c
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -103,9 +103,8 @@ static void gvt_balance_timeslice(struct gvt_sched_data *sched_data)
103 103
104 list_for_each(pos, &sched_data->lru_runq_head) { 104 list_for_each(pos, &sched_data->lru_runq_head) {
105 vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list); 105 vgpu_data = container_of(pos, struct vgpu_sched_data, lru_list);
106 fair_timeslice = ms_to_ktime(GVT_TS_BALANCE_PERIOD_MS) * 106 fair_timeslice = ktime_divns(ms_to_ktime(GVT_TS_BALANCE_PERIOD_MS),
107 vgpu_data->sched_ctl.weight / 107 total_weight) * vgpu_data->sched_ctl.weight;
108 total_weight;
109 108
110 vgpu_data->allocated_ts = fair_timeslice; 109 vgpu_data->allocated_ts = fair_timeslice;
111 vgpu_data->left_ts = vgpu_data->allocated_ts; 110 vgpu_data->left_ts = vgpu_data->allocated_ts;
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index b55b3580ca1d..9b92b4e25a20 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -113,7 +113,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
113#undef COPY_REG 113#undef COPY_REG
114 114
115 set_context_pdp_root_pointer(shadow_ring_context, 115 set_context_pdp_root_pointer(shadow_ring_context,
116 workload->shadow_mm->shadow_page_table); 116 (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);
117 117
118 intel_gvt_hypervisor_read_gpa(vgpu, 118 intel_gvt_hypervisor_read_gpa(vgpu,
119 workload->ring_context_gpa + 119 workload->ring_context_gpa +
@@ -126,7 +126,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
126 return 0; 126 return 0;
127} 127}
128 128
129static inline bool is_gvt_request(struct drm_i915_gem_request *req) 129static inline bool is_gvt_request(struct i915_request *req)
130{ 130{
131 return i915_gem_context_force_single_submission(req->ctx); 131 return i915_gem_context_force_single_submission(req->ctx);
132} 132}
@@ -148,7 +148,7 @@ static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id)
148static int shadow_context_status_change(struct notifier_block *nb, 148static int shadow_context_status_change(struct notifier_block *nb,
149 unsigned long action, void *data) 149 unsigned long action, void *data)
150{ 150{
151 struct drm_i915_gem_request *req = (struct drm_i915_gem_request *)data; 151 struct i915_request *req = data;
152 struct intel_gvt *gvt = container_of(nb, struct intel_gvt, 152 struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
153 shadow_ctx_notifier_block[req->engine->id]); 153 shadow_ctx_notifier_block[req->engine->id]);
154 struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; 154 struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
@@ -225,6 +225,11 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
225 struct intel_vgpu *vgpu = workload->vgpu; 225 struct intel_vgpu *vgpu = workload->vgpu;
226 void *shadow_ring_buffer_va; 226 void *shadow_ring_buffer_va;
227 u32 *cs; 227 u32 *cs;
228 struct i915_request *req = workload->req;
229
230 if (IS_KABYLAKE(req->i915) &&
231 is_inhibit_context(req->ctx, req->engine->id))
232 intel_vgpu_restore_inhibit_context(vgpu, req);
228 233
229 /* allocate shadow ring buffer */ 234 /* allocate shadow ring buffer */
230 cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32)); 235 cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
@@ -333,13 +338,13 @@ static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
333 int ring_id = workload->ring_id; 338 int ring_id = workload->ring_id;
334 struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; 339 struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
335 struct intel_engine_cs *engine = dev_priv->engine[ring_id]; 340 struct intel_engine_cs *engine = dev_priv->engine[ring_id];
336 struct drm_i915_gem_request *rq; 341 struct i915_request *rq;
337 struct intel_vgpu *vgpu = workload->vgpu; 342 struct intel_vgpu *vgpu = workload->vgpu;
338 struct intel_vgpu_submission *s = &vgpu->submission; 343 struct intel_vgpu_submission *s = &vgpu->submission;
339 struct i915_gem_context *shadow_ctx = s->shadow_ctx; 344 struct i915_gem_context *shadow_ctx = s->shadow_ctx;
340 int ret; 345 int ret;
341 346
342 rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx); 347 rq = i915_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
343 if (IS_ERR(rq)) { 348 if (IS_ERR(rq)) {
344 gvt_vgpu_err("fail to allocate gem request\n"); 349 gvt_vgpu_err("fail to allocate gem request\n");
345 ret = PTR_ERR(rq); 350 ret = PTR_ERR(rq);
@@ -348,7 +353,7 @@ static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
348 353
349 gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq); 354 gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
350 355
351 workload->req = i915_gem_request_get(rq); 356 workload->req = i915_request_get(rq);
352 ret = copy_workload_to_ring_buffer(workload); 357 ret = copy_workload_to_ring_buffer(workload);
353 if (ret) 358 if (ret)
354 goto err_unpin; 359 goto err_unpin;
@@ -582,7 +587,7 @@ out:
582 if (!IS_ERR_OR_NULL(workload->req)) { 587 if (!IS_ERR_OR_NULL(workload->req)) {
583 gvt_dbg_sched("ring id %d submit workload to i915 %p\n", 588 gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
584 ring_id, workload->req); 589 ring_id, workload->req);
585 i915_add_request(workload->req); 590 i915_request_add(workload->req);
586 workload->dispatched = true; 591 workload->dispatched = true;
587 } 592 }
588 593
@@ -769,7 +774,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
769 workload->status = 0; 774 workload->status = 0;
770 } 775 }
771 776
772 i915_gem_request_put(fetch_and_zero(&workload->req)); 777 i915_request_put(fetch_and_zero(&workload->req));
773 778
774 if (!workload->status && !(vgpu->resetting_eng & 779 if (!workload->status && !(vgpu->resetting_eng &
775 ENGINE_MASK(ring_id))) { 780 ENGINE_MASK(ring_id))) {
@@ -886,7 +891,7 @@ static int workload_thread(void *priv)
886 891
887 gvt_dbg_sched("ring id %d wait workload %p\n", 892 gvt_dbg_sched("ring id %d wait workload %p\n",
888 workload->ring_id, workload); 893 workload->ring_id, workload);
889 i915_wait_request(workload->req, 0, MAX_SCHEDULE_TIMEOUT); 894 i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
890 895
891complete: 896complete:
892 gvt_dbg_sched("will complete workload %p, status: %d\n", 897 gvt_dbg_sched("will complete workload %p, status: %d\n",
@@ -1132,7 +1137,7 @@ void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
1132 struct intel_vgpu_submission *s = &workload->vgpu->submission; 1137 struct intel_vgpu_submission *s = &workload->vgpu->submission;
1133 1138
1134 if (workload->shadow_mm) 1139 if (workload->shadow_mm)
1135 intel_gvt_mm_unreference(workload->shadow_mm); 1140 intel_vgpu_mm_put(workload->shadow_mm);
1136 1141
1137 kmem_cache_free(s->workloads, workload); 1142 kmem_cache_free(s->workloads, workload);
1138} 1143}
@@ -1181,32 +1186,27 @@ static int prepare_mm(struct intel_vgpu_workload *workload)
1181 struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; 1186 struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
1182 struct intel_vgpu_mm *mm; 1187 struct intel_vgpu_mm *mm;
1183 struct intel_vgpu *vgpu = workload->vgpu; 1188 struct intel_vgpu *vgpu = workload->vgpu;
1184 int page_table_level; 1189 intel_gvt_gtt_type_t root_entry_type;
1185 u32 pdp[8]; 1190 u64 pdps[GVT_RING_CTX_NR_PDPS];
1186 1191
1187 if (desc->addressing_mode == 1) { /* legacy 32-bit */ 1192 switch (desc->addressing_mode) {
1188 page_table_level = 3; 1193 case 1: /* legacy 32-bit */
1189 } else if (desc->addressing_mode == 3) { /* legacy 64 bit */ 1194 root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1190 page_table_level = 4; 1195 break;
1191 } else { 1196 case 3: /* legacy 64-bit */
1197 root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1198 break;
1199 default:
1192 gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); 1200 gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
1193 return -EINVAL; 1201 return -EINVAL;
1194 } 1202 }
1195 1203
1196 read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp); 1204 read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);
1197 1205
1198 mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp); 1206 mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);
1199 if (mm) { 1207 if (IS_ERR(mm))
1200 intel_gvt_mm_reference(mm); 1208 return PTR_ERR(mm);
1201 } else {
1202 1209
1203 mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
1204 pdp, page_table_level, 0);
1205 if (IS_ERR(mm)) {
1206 gvt_vgpu_err("fail to create mm object.\n");
1207 return PTR_ERR(mm);
1208 }
1209 }
1210 workload->shadow_mm = mm; 1210 workload->shadow_mm = mm;
1211 return 0; 1211 return 0;
1212} 1212}
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index ff175a98b19e..bab4097aa6d7 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -80,7 +80,7 @@ struct intel_shadow_wa_ctx {
80struct intel_vgpu_workload { 80struct intel_vgpu_workload {
81 struct intel_vgpu *vgpu; 81 struct intel_vgpu *vgpu;
82 int ring_id; 82 int ring_id;
83 struct drm_i915_gem_request *req; 83 struct i915_request *req;
84 /* if this workload has been dispatched to i915? */ 84 /* if this workload has been dispatched to i915? */
85 bool dispatched; 85 bool dispatched;
86 bool shadowed; 86 bool shadowed;
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
index 7a2511538f34..fc7831a62121 100644
--- a/drivers/gpu/drm/i915/gvt/trace.h
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -113,10 +113,10 @@ TRACE_EVENT(gma_index,
113); 113);
114 114
115TRACE_EVENT(gma_translate, 115TRACE_EVENT(gma_translate,
116 TP_PROTO(int id, char *type, int ring_id, int pt_level, 116 TP_PROTO(int id, char *type, int ring_id, int root_entry_type,
117 unsigned long gma, unsigned long gpa), 117 unsigned long gma, unsigned long gpa),
118 118
119 TP_ARGS(id, type, ring_id, pt_level, gma, gpa), 119 TP_ARGS(id, type, ring_id, root_entry_type, gma, gpa),
120 120
121 TP_STRUCT__entry( 121 TP_STRUCT__entry(
122 __array(char, buf, MAX_BUF_LEN) 122 __array(char, buf, MAX_BUF_LEN)
@@ -124,8 +124,8 @@ TRACE_EVENT(gma_translate,
124 124
125 TP_fast_assign( 125 TP_fast_assign(
126 snprintf(__entry->buf, MAX_BUF_LEN, 126 snprintf(__entry->buf, MAX_BUF_LEN,
127 "VM%d %s ring %d pt_level %d gma 0x%lx -> gpa 0x%lx\n", 127 "VM%d %s ring %d root_entry_type %d gma 0x%lx -> gpa 0x%lx\n",
128 id, type, ring_id, pt_level, gma, gpa); 128 id, type, ring_id, root_entry_type, gma, gpa);
129 ), 129 ),
130 130
131 TP_printk("%s", __entry->buf) 131 TP_printk("%s", __entry->buf)
@@ -168,7 +168,7 @@ TRACE_EVENT(spt_change,
168 TP_printk("%s", __entry->buf) 168 TP_printk("%s", __entry->buf)
169); 169);
170 170
171TRACE_EVENT(gpt_change, 171TRACE_EVENT(spt_guest_change,
172 TP_PROTO(int id, const char *tag, void *spt, int type, u64 v, 172 TP_PROTO(int id, const char *tag, void *spt, int type, u64 v,
173 unsigned long index), 173 unsigned long index),
174 174
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index b87b19d8443c..41f76e86aa1f 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -354,6 +354,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
354 vgpu->gvt = gvt; 354 vgpu->gvt = gvt;
355 vgpu->sched_ctl.weight = param->weight; 355 vgpu->sched_ctl.weight = param->weight;
356 INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); 356 INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head);
357 INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL);
357 idr_init(&vgpu->object_idr); 358 idr_init(&vgpu->object_idr);
358 intel_vgpu_init_cfg_space(vgpu, param->primary); 359 intel_vgpu_init_cfg_space(vgpu, param->primary);
359 360
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 05b41045b8f9..89f7ff2c652e 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -519,7 +519,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
519 list_for_each_entry_reverse(file, &dev->filelist, lhead) { 519 list_for_each_entry_reverse(file, &dev->filelist, lhead) {
520 struct file_stats stats; 520 struct file_stats stats;
521 struct drm_i915_file_private *file_priv = file->driver_priv; 521 struct drm_i915_file_private *file_priv = file->driver_priv;
522 struct drm_i915_gem_request *request; 522 struct i915_request *request;
523 struct task_struct *task; 523 struct task_struct *task;
524 524
525 mutex_lock(&dev->struct_mutex); 525 mutex_lock(&dev->struct_mutex);
@@ -536,7 +536,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
536 * Therefore, we need to protect this ->comm access using RCU. 536 * Therefore, we need to protect this ->comm access using RCU.
537 */ 537 */
538 request = list_first_entry_or_null(&file_priv->mm.request_list, 538 request = list_first_entry_or_null(&file_priv->mm.request_list,
539 struct drm_i915_gem_request, 539 struct i915_request,
540 client_link); 540 client_link);
541 rcu_read_lock(); 541 rcu_read_lock();
542 task = pid_task(request && request->ctx->pid ? 542 task = pid_task(request && request->ctx->pid ?
@@ -646,6 +646,56 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
646 return 0; 646 return 0;
647} 647}
648 648
649static void gen8_display_interrupt_info(struct seq_file *m)
650{
651 struct drm_i915_private *dev_priv = node_to_i915(m->private);
652 int pipe;
653
654 for_each_pipe(dev_priv, pipe) {
655 enum intel_display_power_domain power_domain;
656
657 power_domain = POWER_DOMAIN_PIPE(pipe);
658 if (!intel_display_power_get_if_enabled(dev_priv,
659 power_domain)) {
660 seq_printf(m, "Pipe %c power disabled\n",
661 pipe_name(pipe));
662 continue;
663 }
664 seq_printf(m, "Pipe %c IMR:\t%08x\n",
665 pipe_name(pipe),
666 I915_READ(GEN8_DE_PIPE_IMR(pipe)));
667 seq_printf(m, "Pipe %c IIR:\t%08x\n",
668 pipe_name(pipe),
669 I915_READ(GEN8_DE_PIPE_IIR(pipe)));
670 seq_printf(m, "Pipe %c IER:\t%08x\n",
671 pipe_name(pipe),
672 I915_READ(GEN8_DE_PIPE_IER(pipe)));
673
674 intel_display_power_put(dev_priv, power_domain);
675 }
676
677 seq_printf(m, "Display Engine port interrupt mask:\t%08x\n",
678 I915_READ(GEN8_DE_PORT_IMR));
679 seq_printf(m, "Display Engine port interrupt identity:\t%08x\n",
680 I915_READ(GEN8_DE_PORT_IIR));
681 seq_printf(m, "Display Engine port interrupt enable:\t%08x\n",
682 I915_READ(GEN8_DE_PORT_IER));
683
684 seq_printf(m, "Display Engine misc interrupt mask:\t%08x\n",
685 I915_READ(GEN8_DE_MISC_IMR));
686 seq_printf(m, "Display Engine misc interrupt identity:\t%08x\n",
687 I915_READ(GEN8_DE_MISC_IIR));
688 seq_printf(m, "Display Engine misc interrupt enable:\t%08x\n",
689 I915_READ(GEN8_DE_MISC_IER));
690
691 seq_printf(m, "PCU interrupt mask:\t%08x\n",
692 I915_READ(GEN8_PCU_IMR));
693 seq_printf(m, "PCU interrupt identity:\t%08x\n",
694 I915_READ(GEN8_PCU_IIR));
695 seq_printf(m, "PCU interrupt enable:\t%08x\n",
696 I915_READ(GEN8_PCU_IER));
697}
698
649static int i915_interrupt_info(struct seq_file *m, void *data) 699static int i915_interrupt_info(struct seq_file *m, void *data)
650{ 700{
651 struct drm_i915_private *dev_priv = node_to_i915(m->private); 701 struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -709,6 +759,27 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
709 I915_READ(GEN8_PCU_IIR)); 759 I915_READ(GEN8_PCU_IIR));
710 seq_printf(m, "PCU interrupt enable:\t%08x\n", 760 seq_printf(m, "PCU interrupt enable:\t%08x\n",
711 I915_READ(GEN8_PCU_IER)); 761 I915_READ(GEN8_PCU_IER));
762 } else if (INTEL_GEN(dev_priv) >= 11) {
763 seq_printf(m, "Master Interrupt Control: %08x\n",
764 I915_READ(GEN11_GFX_MSTR_IRQ));
765
766 seq_printf(m, "Render/Copy Intr Enable: %08x\n",
767 I915_READ(GEN11_RENDER_COPY_INTR_ENABLE));
768 seq_printf(m, "VCS/VECS Intr Enable: %08x\n",
769 I915_READ(GEN11_VCS_VECS_INTR_ENABLE));
770 seq_printf(m, "GUC/SG Intr Enable:\t %08x\n",
771 I915_READ(GEN11_GUC_SG_INTR_ENABLE));
772 seq_printf(m, "GPM/WGBOXPERF Intr Enable: %08x\n",
773 I915_READ(GEN11_GPM_WGBOXPERF_INTR_ENABLE));
774 seq_printf(m, "Crypto Intr Enable:\t %08x\n",
775 I915_READ(GEN11_CRYPTO_RSVD_INTR_ENABLE));
776 seq_printf(m, "GUnit/CSME Intr Enable:\t %08x\n",
777 I915_READ(GEN11_GUNIT_CSME_INTR_ENABLE));
778
779 seq_printf(m, "Display Interrupt Control:\t%08x\n",
780 I915_READ(GEN11_DISPLAY_INT_CTL));
781
782 gen8_display_interrupt_info(m);
712 } else if (INTEL_GEN(dev_priv) >= 8) { 783 } else if (INTEL_GEN(dev_priv) >= 8) {
713 seq_printf(m, "Master Interrupt Control:\t%08x\n", 784 seq_printf(m, "Master Interrupt Control:\t%08x\n",
714 I915_READ(GEN8_MASTER_IRQ)); 785 I915_READ(GEN8_MASTER_IRQ));
@@ -722,49 +793,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
722 i, I915_READ(GEN8_GT_IER(i))); 793 i, I915_READ(GEN8_GT_IER(i)));
723 } 794 }
724 795
725 for_each_pipe(dev_priv, pipe) { 796 gen8_display_interrupt_info(m);
726 enum intel_display_power_domain power_domain;
727
728 power_domain = POWER_DOMAIN_PIPE(pipe);
729 if (!intel_display_power_get_if_enabled(dev_priv,
730 power_domain)) {
731 seq_printf(m, "Pipe %c power disabled\n",
732 pipe_name(pipe));
733 continue;
734 }
735 seq_printf(m, "Pipe %c IMR:\t%08x\n",
736 pipe_name(pipe),
737 I915_READ(GEN8_DE_PIPE_IMR(pipe)));
738 seq_printf(m, "Pipe %c IIR:\t%08x\n",
739 pipe_name(pipe),
740 I915_READ(GEN8_DE_PIPE_IIR(pipe)));
741 seq_printf(m, "Pipe %c IER:\t%08x\n",
742 pipe_name(pipe),
743 I915_READ(GEN8_DE_PIPE_IER(pipe)));
744
745 intel_display_power_put(dev_priv, power_domain);
746 }
747
748 seq_printf(m, "Display Engine port interrupt mask:\t%08x\n",
749 I915_READ(GEN8_DE_PORT_IMR));
750 seq_printf(m, "Display Engine port interrupt identity:\t%08x\n",
751 I915_READ(GEN8_DE_PORT_IIR));
752 seq_printf(m, "Display Engine port interrupt enable:\t%08x\n",
753 I915_READ(GEN8_DE_PORT_IER));
754
755 seq_printf(m, "Display Engine misc interrupt mask:\t%08x\n",
756 I915_READ(GEN8_DE_MISC_IMR));
757 seq_printf(m, "Display Engine misc interrupt identity:\t%08x\n",
758 I915_READ(GEN8_DE_MISC_IIR));
759 seq_printf(m, "Display Engine misc interrupt enable:\t%08x\n",
760 I915_READ(GEN8_DE_MISC_IER));
761
762 seq_printf(m, "PCU interrupt mask:\t%08x\n",
763 I915_READ(GEN8_PCU_IMR));
764 seq_printf(m, "PCU interrupt identity:\t%08x\n",
765 I915_READ(GEN8_PCU_IIR));
766 seq_printf(m, "PCU interrupt enable:\t%08x\n",
767 I915_READ(GEN8_PCU_IER));
768 } else if (IS_VALLEYVIEW(dev_priv)) { 797 } else if (IS_VALLEYVIEW(dev_priv)) {
769 seq_printf(m, "Display IER:\t%08x\n", 798 seq_printf(m, "Display IER:\t%08x\n",
770 I915_READ(VLV_IER)); 799 I915_READ(VLV_IER));
@@ -846,13 +875,35 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
846 seq_printf(m, "Graphics Interrupt mask: %08x\n", 875 seq_printf(m, "Graphics Interrupt mask: %08x\n",
847 I915_READ(GTIMR)); 876 I915_READ(GTIMR));
848 } 877 }
849 if (INTEL_GEN(dev_priv) >= 6) { 878
879 if (INTEL_GEN(dev_priv) >= 11) {
880 seq_printf(m, "RCS Intr Mask:\t %08x\n",
881 I915_READ(GEN11_RCS0_RSVD_INTR_MASK));
882 seq_printf(m, "BCS Intr Mask:\t %08x\n",
883 I915_READ(GEN11_BCS_RSVD_INTR_MASK));
884 seq_printf(m, "VCS0/VCS1 Intr Mask:\t %08x\n",
885 I915_READ(GEN11_VCS0_VCS1_INTR_MASK));
886 seq_printf(m, "VCS2/VCS3 Intr Mask:\t %08x\n",
887 I915_READ(GEN11_VCS2_VCS3_INTR_MASK));
888 seq_printf(m, "VECS0/VECS1 Intr Mask:\t %08x\n",
889 I915_READ(GEN11_VECS0_VECS1_INTR_MASK));
890 seq_printf(m, "GUC/SG Intr Mask:\t %08x\n",
891 I915_READ(GEN11_GUC_SG_INTR_MASK));
892 seq_printf(m, "GPM/WGBOXPERF Intr Mask: %08x\n",
893 I915_READ(GEN11_GPM_WGBOXPERF_INTR_MASK));
894 seq_printf(m, "Crypto Intr Mask:\t %08x\n",
895 I915_READ(GEN11_CRYPTO_RSVD_INTR_MASK));
896 seq_printf(m, "Gunit/CSME Intr Mask:\t %08x\n",
897 I915_READ(GEN11_GUNIT_CSME_INTR_MASK));
898
899 } else if (INTEL_GEN(dev_priv) >= 6) {
850 for_each_engine(engine, dev_priv, id) { 900 for_each_engine(engine, dev_priv, id) {
851 seq_printf(m, 901 seq_printf(m,
852 "Graphics Interrupt mask (%s): %08x\n", 902 "Graphics Interrupt mask (%s): %08x\n",
853 engine->name, I915_READ_IMR(engine)); 903 engine->name, I915_READ_IMR(engine));
854 } 904 }
855 } 905 }
906
856 intel_runtime_pm_put(dev_priv); 907 intel_runtime_pm_put(dev_priv);
857 908
858 return 0; 909 return 0;
@@ -3150,6 +3201,16 @@ static int i915_engine_info(struct seq_file *m, void *unused)
3150 return 0; 3201 return 0;
3151} 3202}
3152 3203
3204static int i915_rcs_topology(struct seq_file *m, void *unused)
3205{
3206 struct drm_i915_private *dev_priv = node_to_i915(m->private);
3207 struct drm_printer p = drm_seq_file_printer(m);
3208
3209 intel_device_info_dump_topology(&INTEL_INFO(dev_priv)->sseu, &p);
3210
3211 return 0;
3212}
3213
3153static int i915_shrinker_info(struct seq_file *m, void *unused) 3214static int i915_shrinker_info(struct seq_file *m, void *unused)
3154{ 3215{
3155 struct drm_i915_private *i915 = node_to_i915(m->private); 3216 struct drm_i915_private *i915 = node_to_i915(m->private);
@@ -3926,7 +3987,8 @@ i915_wedged_set(void *data, u64 val)
3926 engine->hangcheck.stalled = true; 3987 engine->hangcheck.stalled = true;
3927 } 3988 }
3928 3989
3929 i915_handle_error(i915, val, "Manually setting wedged to %llu", val); 3990 i915_handle_error(i915, val, "Manually set wedged engine mask = %llx",
3991 val);
3930 3992
3931 wait_on_bit(&i915->gpu_error.flags, 3993 wait_on_bit(&i915->gpu_error.flags,
3932 I915_RESET_HANDOFF, 3994 I915_RESET_HANDOFF,
@@ -4060,7 +4122,7 @@ i915_drop_caches_set(void *data, u64 val)
4060 I915_WAIT_LOCKED); 4122 I915_WAIT_LOCKED);
4061 4123
4062 if (val & DROP_RETIRE) 4124 if (val & DROP_RETIRE)
4063 i915_gem_retire_requests(dev_priv); 4125 i915_retire_requests(dev_priv);
4064 4126
4065 mutex_unlock(&dev->struct_mutex); 4127 mutex_unlock(&dev->struct_mutex);
4066 } 4128 }
@@ -4271,7 +4333,7 @@ static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
4271 continue; 4333 continue;
4272 4334
4273 sseu->slice_mask = BIT(0); 4335 sseu->slice_mask = BIT(0);
4274 sseu->subslice_mask |= BIT(ss); 4336 sseu->subslice_mask[0] |= BIT(ss);
4275 eu_cnt = ((sig1[ss] & CHV_EU08_PG_ENABLE) ? 0 : 2) + 4337 eu_cnt = ((sig1[ss] & CHV_EU08_PG_ENABLE) ? 0 : 2) +
4276 ((sig1[ss] & CHV_EU19_PG_ENABLE) ? 0 : 2) + 4338 ((sig1[ss] & CHV_EU19_PG_ENABLE) ? 0 : 2) +
4277 ((sig1[ss] & CHV_EU210_PG_ENABLE) ? 0 : 2) + 4339 ((sig1[ss] & CHV_EU210_PG_ENABLE) ? 0 : 2) +
@@ -4286,11 +4348,11 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
4286 struct sseu_dev_info *sseu) 4348 struct sseu_dev_info *sseu)
4287{ 4349{
4288 const struct intel_device_info *info = INTEL_INFO(dev_priv); 4350 const struct intel_device_info *info = INTEL_INFO(dev_priv);
4289 int s_max = 6, ss_max = 4;
4290 int s, ss; 4351 int s, ss;
4291 u32 s_reg[s_max], eu_reg[2 * s_max], eu_mask[2]; 4352 u32 s_reg[info->sseu.max_slices];
4353 u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
4292 4354
4293 for (s = 0; s < s_max; s++) { 4355 for (s = 0; s < info->sseu.max_slices; s++) {
4294 /* 4356 /*
4295 * FIXME: Valid SS Mask respects the spec and read 4357 * FIXME: Valid SS Mask respects the spec and read
4296 * only valid bits for those registers, excluding reserverd 4358 * only valid bits for those registers, excluding reserverd
@@ -4312,15 +4374,15 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
4312 GEN9_PGCTL_SSB_EU210_ACK | 4374 GEN9_PGCTL_SSB_EU210_ACK |
4313 GEN9_PGCTL_SSB_EU311_ACK; 4375 GEN9_PGCTL_SSB_EU311_ACK;
4314 4376
4315 for (s = 0; s < s_max; s++) { 4377 for (s = 0; s < info->sseu.max_slices; s++) {
4316 if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0) 4378 if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0)
4317 /* skip disabled slice */ 4379 /* skip disabled slice */
4318 continue; 4380 continue;
4319 4381
4320 sseu->slice_mask |= BIT(s); 4382 sseu->slice_mask |= BIT(s);
4321 sseu->subslice_mask = info->sseu.subslice_mask; 4383 sseu->subslice_mask[s] = info->sseu.subslice_mask[s];
4322 4384
4323 for (ss = 0; ss < ss_max; ss++) { 4385 for (ss = 0; ss < info->sseu.max_subslices; ss++) {
4324 unsigned int eu_cnt; 4386 unsigned int eu_cnt;
4325 4387
4326 if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss)))) 4388 if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss))))
@@ -4340,17 +4402,12 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
4340static void gen9_sseu_device_status(struct drm_i915_private *dev_priv, 4402static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
4341 struct sseu_dev_info *sseu) 4403 struct sseu_dev_info *sseu)
4342{ 4404{
4343 int s_max = 3, ss_max = 4; 4405 const struct intel_device_info *info = INTEL_INFO(dev_priv);
4344 int s, ss; 4406 int s, ss;
4345 u32 s_reg[s_max], eu_reg[2*s_max], eu_mask[2]; 4407 u32 s_reg[info->sseu.max_slices];
4346 4408 u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
4347 /* BXT has a single slice and at most 3 subslices. */
4348 if (IS_GEN9_LP(dev_priv)) {
4349 s_max = 1;
4350 ss_max = 3;
4351 }
4352 4409
4353 for (s = 0; s < s_max; s++) { 4410 for (s = 0; s < info->sseu.max_slices; s++) {
4354 s_reg[s] = I915_READ(GEN9_SLICE_PGCTL_ACK(s)); 4411 s_reg[s] = I915_READ(GEN9_SLICE_PGCTL_ACK(s));
4355 eu_reg[2*s] = I915_READ(GEN9_SS01_EU_PGCTL_ACK(s)); 4412 eu_reg[2*s] = I915_READ(GEN9_SS01_EU_PGCTL_ACK(s));
4356 eu_reg[2*s + 1] = I915_READ(GEN9_SS23_EU_PGCTL_ACK(s)); 4413 eu_reg[2*s + 1] = I915_READ(GEN9_SS23_EU_PGCTL_ACK(s));
@@ -4365,7 +4422,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
4365 GEN9_PGCTL_SSB_EU210_ACK | 4422 GEN9_PGCTL_SSB_EU210_ACK |
4366 GEN9_PGCTL_SSB_EU311_ACK; 4423 GEN9_PGCTL_SSB_EU311_ACK;
4367 4424
4368 for (s = 0; s < s_max; s++) { 4425 for (s = 0; s < info->sseu.max_slices; s++) {
4369 if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0) 4426 if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0)
4370 /* skip disabled slice */ 4427 /* skip disabled slice */
4371 continue; 4428 continue;
@@ -4373,10 +4430,10 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
4373 sseu->slice_mask |= BIT(s); 4430 sseu->slice_mask |= BIT(s);
4374 4431
4375 if (IS_GEN9_BC(dev_priv)) 4432 if (IS_GEN9_BC(dev_priv))
4376 sseu->subslice_mask = 4433 sseu->subslice_mask[s] =
4377 INTEL_INFO(dev_priv)->sseu.subslice_mask; 4434 INTEL_INFO(dev_priv)->sseu.subslice_mask[s];
4378 4435
4379 for (ss = 0; ss < ss_max; ss++) { 4436 for (ss = 0; ss < info->sseu.max_subslices; ss++) {
4380 unsigned int eu_cnt; 4437 unsigned int eu_cnt;
4381 4438
4382 if (IS_GEN9_LP(dev_priv)) { 4439 if (IS_GEN9_LP(dev_priv)) {
@@ -4384,7 +4441,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
4384 /* skip disabled subslice */ 4441 /* skip disabled subslice */
4385 continue; 4442 continue;
4386 4443
4387 sseu->subslice_mask |= BIT(ss); 4444 sseu->subslice_mask[s] |= BIT(ss);
4388 } 4445 }
4389 4446
4390 eu_cnt = 2 * hweight32(eu_reg[2*s + ss/2] & 4447 eu_cnt = 2 * hweight32(eu_reg[2*s + ss/2] &
@@ -4406,9 +4463,12 @@ static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv,
4406 sseu->slice_mask = slice_info & GEN8_LSLICESTAT_MASK; 4463 sseu->slice_mask = slice_info & GEN8_LSLICESTAT_MASK;
4407 4464
4408 if (sseu->slice_mask) { 4465 if (sseu->slice_mask) {
4409 sseu->subslice_mask = INTEL_INFO(dev_priv)->sseu.subslice_mask;
4410 sseu->eu_per_subslice = 4466 sseu->eu_per_subslice =
4411 INTEL_INFO(dev_priv)->sseu.eu_per_subslice; 4467 INTEL_INFO(dev_priv)->sseu.eu_per_subslice;
4468 for (s = 0; s < fls(sseu->slice_mask); s++) {
4469 sseu->subslice_mask[s] =
4470 INTEL_INFO(dev_priv)->sseu.subslice_mask[s];
4471 }
4412 sseu->eu_total = sseu->eu_per_subslice * 4472 sseu->eu_total = sseu->eu_per_subslice *
4413 sseu_subslice_total(sseu); 4473 sseu_subslice_total(sseu);
4414 4474
@@ -4427,6 +4487,7 @@ static void i915_print_sseu_info(struct seq_file *m, bool is_available_info,
4427{ 4487{
4428 struct drm_i915_private *dev_priv = node_to_i915(m->private); 4488 struct drm_i915_private *dev_priv = node_to_i915(m->private);
4429 const char *type = is_available_info ? "Available" : "Enabled"; 4489 const char *type = is_available_info ? "Available" : "Enabled";
4490 int s;
4430 4491
4431 seq_printf(m, " %s Slice Mask: %04x\n", type, 4492 seq_printf(m, " %s Slice Mask: %04x\n", type,
4432 sseu->slice_mask); 4493 sseu->slice_mask);
@@ -4434,10 +4495,10 @@ static void i915_print_sseu_info(struct seq_file *m, bool is_available_info,
4434 hweight8(sseu->slice_mask)); 4495 hweight8(sseu->slice_mask));
4435 seq_printf(m, " %s Subslice Total: %u\n", type, 4496 seq_printf(m, " %s Subslice Total: %u\n", type,
4436 sseu_subslice_total(sseu)); 4497 sseu_subslice_total(sseu));
4437 seq_printf(m, " %s Subslice Mask: %04x\n", type, 4498 for (s = 0; s < fls(sseu->slice_mask); s++) {
4438 sseu->subslice_mask); 4499 seq_printf(m, " %s Slice%i subslices: %u\n", type,
4439 seq_printf(m, " %s Subslice Per Slice: %u\n", type, 4500 s, hweight8(sseu->subslice_mask[s]));
4440 hweight8(sseu->subslice_mask)); 4501 }
4441 seq_printf(m, " %s EU Total: %u\n", type, 4502 seq_printf(m, " %s EU Total: %u\n", type,
4442 sseu->eu_total); 4503 sseu->eu_total);
4443 seq_printf(m, " %s EU Per Subslice: %u\n", type, 4504 seq_printf(m, " %s EU Per Subslice: %u\n", type,
@@ -4471,6 +4532,10 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
4471 4532
4472 seq_puts(m, "SSEU Device Status\n"); 4533 seq_puts(m, "SSEU Device Status\n");
4473 memset(&sseu, 0, sizeof(sseu)); 4534 memset(&sseu, 0, sizeof(sseu));
4535 sseu.max_slices = INTEL_INFO(dev_priv)->sseu.max_slices;
4536 sseu.max_subslices = INTEL_INFO(dev_priv)->sseu.max_subslices;
4537 sseu.max_eus_per_subslice =
4538 INTEL_INFO(dev_priv)->sseu.max_eus_per_subslice;
4474 4539
4475 intel_runtime_pm_get(dev_priv); 4540 intel_runtime_pm_get(dev_priv);
4476 4541
@@ -4678,6 +4743,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
4678 {"i915_dmc_info", i915_dmc_info, 0}, 4743 {"i915_dmc_info", i915_dmc_info, 0},
4679 {"i915_display_info", i915_display_info, 0}, 4744 {"i915_display_info", i915_display_info, 0},
4680 {"i915_engine_info", i915_engine_info, 0}, 4745 {"i915_engine_info", i915_engine_info, 0},
4746 {"i915_rcs_topology", i915_rcs_topology, 0},
4681 {"i915_shrinker_info", i915_shrinker_info, 0}, 4747 {"i915_shrinker_info", i915_shrinker_info, 0},
4682 {"i915_shared_dplls_info", i915_shared_dplls_info, 0}, 4748 {"i915_shared_dplls_info", i915_shared_dplls_info, 0},
4683 {"i915_dp_mst_info", i915_dp_mst_info, 0}, 4749 {"i915_dp_mst_info", i915_dp_mst_info, 0},
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index d09f8e661fbd..d7c4de45644d 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -49,6 +49,7 @@
49#include "i915_drv.h" 49#include "i915_drv.h"
50#include "i915_trace.h" 50#include "i915_trace.h"
51#include "i915_pmu.h" 51#include "i915_pmu.h"
52#include "i915_query.h"
52#include "i915_vgpu.h" 53#include "i915_vgpu.h"
53#include "intel_drv.h" 54#include "intel_drv.h"
54#include "intel_uc.h" 55#include "intel_uc.h"
@@ -428,7 +429,7 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
428 return -ENODEV; 429 return -ENODEV;
429 break; 430 break;
430 case I915_PARAM_SUBSLICE_MASK: 431 case I915_PARAM_SUBSLICE_MASK:
431 value = INTEL_INFO(dev_priv)->sseu.subslice_mask; 432 value = INTEL_INFO(dev_priv)->sseu.subslice_mask[0];
432 if (!value) 433 if (!value)
433 return -ENODEV; 434 return -ENODEV;
434 break; 435 break;
@@ -808,7 +809,7 @@ static int i915_workqueues_init(struct drm_i915_private *dev_priv)
808 /* 809 /*
809 * The i915 workqueue is primarily used for batched retirement of 810 * The i915 workqueue is primarily used for batched retirement of
810 * requests (and thus managing bo) once the task has been completed 811 * requests (and thus managing bo) once the task has been completed
811 * by the GPU. i915_gem_retire_requests() is called directly when we 812 * by the GPU. i915_retire_requests() is called directly when we
812 * need high-priority retirement, such as waiting for an explicit 813 * need high-priority retirement, such as waiting for an explicit
813 * bo. 814 * bo.
814 * 815 *
@@ -1992,7 +1993,7 @@ taint:
1992 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 1993 add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
1993error: 1994error:
1994 i915_gem_set_wedged(i915); 1995 i915_gem_set_wedged(i915);
1995 i915_gem_retire_requests(i915); 1996 i915_retire_requests(i915);
1996 intel_gpu_reset(i915, ALL_ENGINES); 1997 intel_gpu_reset(i915, ALL_ENGINES);
1997 goto finish; 1998 goto finish;
1998} 1999}
@@ -2019,7 +2020,7 @@ static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
2019int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags) 2020int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
2020{ 2021{
2021 struct i915_gpu_error *error = &engine->i915->gpu_error; 2022 struct i915_gpu_error *error = &engine->i915->gpu_error;
2022 struct drm_i915_gem_request *active_request; 2023 struct i915_request *active_request;
2023 int ret; 2024 int ret;
2024 2025
2025 GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags)); 2026 GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
@@ -2575,7 +2576,7 @@ static int intel_runtime_suspend(struct device *kdev)
2575 */ 2576 */
2576 i915_gem_runtime_suspend(dev_priv); 2577 i915_gem_runtime_suspend(dev_priv);
2577 2578
2578 intel_guc_suspend(dev_priv); 2579 intel_uc_suspend(dev_priv);
2579 2580
2580 intel_runtime_pm_disable_interrupts(dev_priv); 2581 intel_runtime_pm_disable_interrupts(dev_priv);
2581 2582
@@ -2597,7 +2598,7 @@ static int intel_runtime_suspend(struct device *kdev)
2597 2598
2598 intel_runtime_pm_enable_interrupts(dev_priv); 2599 intel_runtime_pm_enable_interrupts(dev_priv);
2599 2600
2600 intel_guc_resume(dev_priv); 2601 intel_uc_resume(dev_priv);
2601 2602
2602 i915_gem_init_swizzling(dev_priv); 2603 i915_gem_init_swizzling(dev_priv);
2603 i915_gem_restore_fences(dev_priv); 2604 i915_gem_restore_fences(dev_priv);
@@ -2683,7 +2684,7 @@ static int intel_runtime_resume(struct device *kdev)
2683 2684
2684 intel_runtime_pm_enable_interrupts(dev_priv); 2685 intel_runtime_pm_enable_interrupts(dev_priv);
2685 2686
2686 intel_guc_resume(dev_priv); 2687 intel_uc_resume(dev_priv);
2687 2688
2688 /* 2689 /*
2689 * No point of rolling back things in case of an error, as the best 2690 * No point of rolling back things in case of an error, as the best
@@ -2832,6 +2833,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = {
2832 DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW), 2833 DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
2833 DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), 2834 DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2834 DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), 2835 DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2836 DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2835}; 2837};
2836 2838
2837static struct drm_driver driver = { 2839static struct drm_driver driver = {
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 92883a40bdd5..6e740f6fe33f 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -71,9 +71,9 @@
71#include "i915_gem_fence_reg.h" 71#include "i915_gem_fence_reg.h"
72#include "i915_gem_object.h" 72#include "i915_gem_object.h"
73#include "i915_gem_gtt.h" 73#include "i915_gem_gtt.h"
74#include "i915_gem_request.h"
75#include "i915_gem_timeline.h" 74#include "i915_gem_timeline.h"
76 75
76#include "i915_request.h"
77#include "i915_vma.h" 77#include "i915_vma.h"
78 78
79#include "intel_gvt.h" 79#include "intel_gvt.h"
@@ -83,8 +83,8 @@
83 83
84#define DRIVER_NAME "i915" 84#define DRIVER_NAME "i915"
85#define DRIVER_DESC "Intel Graphics" 85#define DRIVER_DESC "Intel Graphics"
86#define DRIVER_DATE "20180221" 86#define DRIVER_DATE "20180308"
87#define DRIVER_TIMESTAMP 1519219289 87#define DRIVER_TIMESTAMP 1520513379
88 88
89/* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and 89/* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
90 * WARN_ON()) for hw state sanity checks to check for unexpected conditions 90 * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -1231,7 +1231,7 @@ struct i915_gpu_error {
1231 * 1231 *
1232 * #I915_WEDGED - If reset fails and we can no longer use the GPU, 1232 * #I915_WEDGED - If reset fails and we can no longer use the GPU,
1233 * we set the #I915_WEDGED bit. Prior to command submission, e.g. 1233 * we set the #I915_WEDGED bit. Prior to command submission, e.g.
1234 * i915_gem_request_alloc(), this bit is checked and the sequence 1234 * i915_request_alloc(), this bit is checked and the sequence
1235 * aborted (with -EIO reported to userspace) if set. 1235 * aborted (with -EIO reported to userspace) if set.
1236 */ 1236 */
1237 unsigned long flags; 1237 unsigned long flags;
@@ -2103,6 +2103,7 @@ struct drm_i915_private {
2103 */ 2103 */
2104 struct ida hw_ida; 2104 struct ida hw_ida;
2105#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */ 2105#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */
2106#define GEN11_MAX_CONTEXT_HW_ID (1<<11) /* exclusive */
2106 } contexts; 2107 } contexts;
2107 2108
2108 u32 fdi_rx_config; 2109 u32 fdi_rx_config;
@@ -2746,6 +2747,9 @@ intel_info(const struct drm_i915_private *dev_priv)
2746#define BLT_RING ENGINE_MASK(BCS) 2747#define BLT_RING ENGINE_MASK(BCS)
2747#define VEBOX_RING ENGINE_MASK(VECS) 2748#define VEBOX_RING ENGINE_MASK(VECS)
2748#define BSD2_RING ENGINE_MASK(VCS2) 2749#define BSD2_RING ENGINE_MASK(VCS2)
2750#define BSD3_RING ENGINE_MASK(VCS3)
2751#define BSD4_RING ENGINE_MASK(VCS4)
2752#define VEBOX2_RING ENGINE_MASK(VECS2)
2749#define ALL_ENGINES (~0) 2753#define ALL_ENGINES (~0)
2750 2754
2751#define HAS_ENGINE(dev_priv, id) \ 2755#define HAS_ENGINE(dev_priv, id) \
@@ -2768,6 +2772,8 @@ intel_info(const struct drm_i915_private *dev_priv)
2768 2772
2769#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ 2773#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
2770 ((dev_priv)->info.has_logical_ring_contexts) 2774 ((dev_priv)->info.has_logical_ring_contexts)
2775#define HAS_LOGICAL_RING_ELSQ(dev_priv) \
2776 ((dev_priv)->info.has_logical_ring_elsq)
2771#define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \ 2777#define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \
2772 ((dev_priv)->info.has_logical_ring_preemption) 2778 ((dev_priv)->info.has_logical_ring_preemption)
2773 2779
@@ -2788,9 +2794,10 @@ intel_info(const struct drm_i915_private *dev_priv)
2788/* Early gen2 have a totally busted CS tlb and require pinned batches. */ 2794/* Early gen2 have a totally busted CS tlb and require pinned batches. */
2789#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) 2795#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv))
2790 2796
2791/* WaRsDisableCoarsePowerGating:skl,bxt */ 2797/* WaRsDisableCoarsePowerGating:skl,cnl */
2792#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ 2798#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
2793 (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv)) 2799 (IS_CANNONLAKE(dev_priv) || \
2800 IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
2794 2801
2795/* 2802/*
2796 * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts 2803 * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
@@ -3329,7 +3336,7 @@ i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
3329 3336
3330int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); 3337int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
3331void i915_vma_move_to_active(struct i915_vma *vma, 3338void i915_vma_move_to_active(struct i915_vma *vma,
3332 struct drm_i915_gem_request *req, 3339 struct i915_request *rq,
3333 unsigned int flags); 3340 unsigned int flags);
3334int i915_gem_dumb_create(struct drm_file *file_priv, 3341int i915_gem_dumb_create(struct drm_file *file_priv,
3335 struct drm_device *dev, 3342 struct drm_device *dev,
@@ -3344,11 +3351,9 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
3344 3351
3345int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); 3352int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
3346 3353
3347struct drm_i915_gem_request * 3354struct i915_request *
3348i915_gem_find_active_request(struct intel_engine_cs *engine); 3355i915_gem_find_active_request(struct intel_engine_cs *engine);
3349 3356
3350void i915_gem_retire_requests(struct drm_i915_private *dev_priv);
3351
3352static inline bool i915_reset_backoff(struct i915_gpu_error *error) 3357static inline bool i915_reset_backoff(struct i915_gpu_error *error)
3353{ 3358{
3354 return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags)); 3359 return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags));
@@ -3380,7 +3385,7 @@ static inline u32 i915_reset_engine_count(struct i915_gpu_error *error,
3380 return READ_ONCE(error->reset_engine_count[engine->id]); 3385 return READ_ONCE(error->reset_engine_count[engine->id]);
3381} 3386}
3382 3387
3383struct drm_i915_gem_request * 3388struct i915_request *
3384i915_gem_reset_prepare_engine(struct intel_engine_cs *engine); 3389i915_gem_reset_prepare_engine(struct intel_engine_cs *engine);
3385int i915_gem_reset_prepare(struct drm_i915_private *dev_priv); 3390int i915_gem_reset_prepare(struct drm_i915_private *dev_priv);
3386void i915_gem_reset(struct drm_i915_private *dev_priv); 3391void i915_gem_reset(struct drm_i915_private *dev_priv);
@@ -3389,7 +3394,7 @@ void i915_gem_reset_finish(struct drm_i915_private *dev_priv);
3389void i915_gem_set_wedged(struct drm_i915_private *dev_priv); 3394void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
3390bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv); 3395bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv);
3391void i915_gem_reset_engine(struct intel_engine_cs *engine, 3396void i915_gem_reset_engine(struct intel_engine_cs *engine,
3392 struct drm_i915_gem_request *request); 3397 struct i915_request *request);
3393 3398
3394void i915_gem_init_mmio(struct drm_i915_private *i915); 3399void i915_gem_init_mmio(struct drm_i915_private *i915);
3395int __must_check i915_gem_init(struct drm_i915_private *dev_priv); 3400int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
@@ -4008,9 +4013,9 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
4008} 4013}
4009 4014
4010static inline bool 4015static inline bool
4011__i915_request_irq_complete(const struct drm_i915_gem_request *req) 4016__i915_request_irq_complete(const struct i915_request *rq)
4012{ 4017{
4013 struct intel_engine_cs *engine = req->engine; 4018 struct intel_engine_cs *engine = rq->engine;
4014 u32 seqno; 4019 u32 seqno;
4015 4020
4016 /* Note that the engine may have wrapped around the seqno, and 4021 /* Note that the engine may have wrapped around the seqno, and
@@ -4019,7 +4024,7 @@ __i915_request_irq_complete(const struct drm_i915_gem_request *req)
4019 * this by kicking all the waiters before resetting the seqno 4024 * this by kicking all the waiters before resetting the seqno
4020 * in hardware, and also signal the fence. 4025 * in hardware, and also signal the fence.
4021 */ 4026 */
4022 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &req->fence.flags)) 4027 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
4023 return true; 4028 return true;
4024 4029
4025 /* The request was dequeued before we were awoken. We check after 4030 /* The request was dequeued before we were awoken. We check after
@@ -4028,14 +4033,14 @@ __i915_request_irq_complete(const struct drm_i915_gem_request *req)
4028 * the request execution are sufficient to ensure that a check 4033 * the request execution are sufficient to ensure that a check
4029 * after reading the value from hw matches this request. 4034 * after reading the value from hw matches this request.
4030 */ 4035 */
4031 seqno = i915_gem_request_global_seqno(req); 4036 seqno = i915_request_global_seqno(rq);
4032 if (!seqno) 4037 if (!seqno)
4033 return false; 4038 return false;
4034 4039
4035 /* Before we do the heavier coherent read of the seqno, 4040 /* Before we do the heavier coherent read of the seqno,
4036 * check the value (hopefully) in the CPU cacheline. 4041 * check the value (hopefully) in the CPU cacheline.
4037 */ 4042 */
4038 if (__i915_gem_request_completed(req, seqno)) 4043 if (__i915_request_completed(rq, seqno))
4039 return true; 4044 return true;
4040 4045
4041 /* Ensure our read of the seqno is coherent so that we 4046 /* Ensure our read of the seqno is coherent so that we
@@ -4084,7 +4089,7 @@ __i915_request_irq_complete(const struct drm_i915_gem_request *req)
4084 wake_up_process(b->irq_wait->tsk); 4089 wake_up_process(b->irq_wait->tsk);
4085 spin_unlock_irq(&b->irq_lock); 4090 spin_unlock_irq(&b->irq_lock);
4086 4091
4087 if (__i915_gem_request_completed(req, seqno)) 4092 if (__i915_request_completed(rq, seqno))
4088 return true; 4093 return true;
4089 } 4094 }
4090 4095
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 43afa1c1b14f..a5bd07338b46 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -353,7 +353,7 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
353 long timeout, 353 long timeout,
354 struct intel_rps_client *rps_client) 354 struct intel_rps_client *rps_client)
355{ 355{
356 struct drm_i915_gem_request *rq; 356 struct i915_request *rq;
357 357
358 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 358 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
359 359
@@ -366,7 +366,7 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
366 timeout); 366 timeout);
367 367
368 rq = to_request(fence); 368 rq = to_request(fence);
369 if (i915_gem_request_completed(rq)) 369 if (i915_request_completed(rq))
370 goto out; 370 goto out;
371 371
372 /* 372 /*
@@ -385,16 +385,16 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
385 * forcing the clocks too high for the whole system, we only allow 385 * forcing the clocks too high for the whole system, we only allow
386 * each client to waitboost once in a busy period. 386 * each client to waitboost once in a busy period.
387 */ 387 */
388 if (rps_client && !i915_gem_request_started(rq)) { 388 if (rps_client && !i915_request_started(rq)) {
389 if (INTEL_GEN(rq->i915) >= 6) 389 if (INTEL_GEN(rq->i915) >= 6)
390 gen6_rps_boost(rq, rps_client); 390 gen6_rps_boost(rq, rps_client);
391 } 391 }
392 392
393 timeout = i915_wait_request(rq, flags, timeout); 393 timeout = i915_request_wait(rq, flags, timeout);
394 394
395out: 395out:
396 if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) 396 if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
397 i915_gem_request_retire_upto(rq); 397 i915_request_retire_upto(rq);
398 398
399 return timeout; 399 return timeout;
400} 400}
@@ -463,7 +463,7 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
463 463
464static void __fence_set_priority(struct dma_fence *fence, int prio) 464static void __fence_set_priority(struct dma_fence *fence, int prio)
465{ 465{
466 struct drm_i915_gem_request *rq; 466 struct i915_request *rq;
467 struct intel_engine_cs *engine; 467 struct intel_engine_cs *engine;
468 468
469 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 469 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
@@ -2856,10 +2856,10 @@ static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
2856 atomic_inc(&ctx->active_count); 2856 atomic_inc(&ctx->active_count);
2857} 2857}
2858 2858
2859struct drm_i915_gem_request * 2859struct i915_request *
2860i915_gem_find_active_request(struct intel_engine_cs *engine) 2860i915_gem_find_active_request(struct intel_engine_cs *engine)
2861{ 2861{
2862 struct drm_i915_gem_request *request, *active = NULL; 2862 struct i915_request *request, *active = NULL;
2863 unsigned long flags; 2863 unsigned long flags;
2864 2864
2865 /* We are called by the error capture and reset at a random 2865 /* We are called by the error capture and reset at a random
@@ -2872,8 +2872,7 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
2872 */ 2872 */
2873 spin_lock_irqsave(&engine->timeline->lock, flags); 2873 spin_lock_irqsave(&engine->timeline->lock, flags);
2874 list_for_each_entry(request, &engine->timeline->requests, link) { 2874 list_for_each_entry(request, &engine->timeline->requests, link) {
2875 if (__i915_gem_request_completed(request, 2875 if (__i915_request_completed(request, request->global_seqno))
2876 request->global_seqno))
2877 continue; 2876 continue;
2878 2877
2879 GEM_BUG_ON(request->engine != engine); 2878 GEM_BUG_ON(request->engine != engine);
@@ -2906,10 +2905,10 @@ static bool engine_stalled(struct intel_engine_cs *engine)
2906 * Ensure irq handler finishes, and not run again. 2905 * Ensure irq handler finishes, and not run again.
2907 * Also return the active request so that we only search for it once. 2906 * Also return the active request so that we only search for it once.
2908 */ 2907 */
2909struct drm_i915_gem_request * 2908struct i915_request *
2910i915_gem_reset_prepare_engine(struct intel_engine_cs *engine) 2909i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
2911{ 2910{
2912 struct drm_i915_gem_request *request = NULL; 2911 struct i915_request *request = NULL;
2913 2912
2914 /* 2913 /*
2915 * During the reset sequence, we must prevent the engine from 2914 * During the reset sequence, we must prevent the engine from
@@ -2967,7 +2966,7 @@ i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
2967int i915_gem_reset_prepare(struct drm_i915_private *dev_priv) 2966int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
2968{ 2967{
2969 struct intel_engine_cs *engine; 2968 struct intel_engine_cs *engine;
2970 struct drm_i915_gem_request *request; 2969 struct i915_request *request;
2971 enum intel_engine_id id; 2970 enum intel_engine_id id;
2972 int err = 0; 2971 int err = 0;
2973 2972
@@ -2986,7 +2985,7 @@ int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
2986 return err; 2985 return err;
2987} 2986}
2988 2987
2989static void skip_request(struct drm_i915_gem_request *request) 2988static void skip_request(struct i915_request *request)
2990{ 2989{
2991 void *vaddr = request->ring->vaddr; 2990 void *vaddr = request->ring->vaddr;
2992 u32 head; 2991 u32 head;
@@ -3005,7 +3004,7 @@ static void skip_request(struct drm_i915_gem_request *request)
3005 dma_fence_set_error(&request->fence, -EIO); 3004 dma_fence_set_error(&request->fence, -EIO);
3006} 3005}
3007 3006
3008static void engine_skip_context(struct drm_i915_gem_request *request) 3007static void engine_skip_context(struct i915_request *request)
3009{ 3008{
3010 struct intel_engine_cs *engine = request->engine; 3009 struct intel_engine_cs *engine = request->engine;
3011 struct i915_gem_context *hung_ctx = request->ctx; 3010 struct i915_gem_context *hung_ctx = request->ctx;
@@ -3029,9 +3028,9 @@ static void engine_skip_context(struct drm_i915_gem_request *request)
3029} 3028}
3030 3029
3031/* Returns the request if it was guilty of the hang */ 3030/* Returns the request if it was guilty of the hang */
3032static struct drm_i915_gem_request * 3031static struct i915_request *
3033i915_gem_reset_request(struct intel_engine_cs *engine, 3032i915_gem_reset_request(struct intel_engine_cs *engine,
3034 struct drm_i915_gem_request *request) 3033 struct i915_request *request)
3035{ 3034{
3036 /* The guilty request will get skipped on a hung engine. 3035 /* The guilty request will get skipped on a hung engine.
3037 * 3036 *
@@ -3085,7 +3084,7 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
3085} 3084}
3086 3085
3087void i915_gem_reset_engine(struct intel_engine_cs *engine, 3086void i915_gem_reset_engine(struct intel_engine_cs *engine,
3088 struct drm_i915_gem_request *request) 3087 struct i915_request *request)
3089{ 3088{
3090 /* 3089 /*
3091 * Make sure this write is visible before we re-enable the interrupt 3090 * Make sure this write is visible before we re-enable the interrupt
@@ -3113,7 +3112,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
3113 3112
3114 lockdep_assert_held(&dev_priv->drm.struct_mutex); 3113 lockdep_assert_held(&dev_priv->drm.struct_mutex);
3115 3114
3116 i915_gem_retire_requests(dev_priv); 3115 i915_retire_requests(dev_priv);
3117 3116
3118 for_each_engine(engine, dev_priv, id) { 3117 for_each_engine(engine, dev_priv, id) {
3119 struct i915_gem_context *ctx; 3118 struct i915_gem_context *ctx;
@@ -3134,12 +3133,12 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
3134 * empty request appears sufficient to paper over the glitch. 3133 * empty request appears sufficient to paper over the glitch.
3135 */ 3134 */
3136 if (intel_engine_is_idle(engine)) { 3135 if (intel_engine_is_idle(engine)) {
3137 struct drm_i915_gem_request *rq; 3136 struct i915_request *rq;
3138 3137
3139 rq = i915_gem_request_alloc(engine, 3138 rq = i915_request_alloc(engine,
3140 dev_priv->kernel_context); 3139 dev_priv->kernel_context);
3141 if (!IS_ERR(rq)) 3140 if (!IS_ERR(rq))
3142 __i915_add_request(rq, false); 3141 __i915_request_add(rq, false);
3143 } 3142 }
3144 } 3143 }
3145 3144
@@ -3174,21 +3173,21 @@ void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
3174 } 3173 }
3175} 3174}
3176 3175
3177static void nop_submit_request(struct drm_i915_gem_request *request) 3176static void nop_submit_request(struct i915_request *request)
3178{ 3177{
3179 dma_fence_set_error(&request->fence, -EIO); 3178 dma_fence_set_error(&request->fence, -EIO);
3180 3179
3181 i915_gem_request_submit(request); 3180 i915_request_submit(request);
3182} 3181}
3183 3182
3184static void nop_complete_submit_request(struct drm_i915_gem_request *request) 3183static void nop_complete_submit_request(struct i915_request *request)
3185{ 3184{
3186 unsigned long flags; 3185 unsigned long flags;
3187 3186
3188 dma_fence_set_error(&request->fence, -EIO); 3187 dma_fence_set_error(&request->fence, -EIO);
3189 3188
3190 spin_lock_irqsave(&request->engine->timeline->lock, flags); 3189 spin_lock_irqsave(&request->engine->timeline->lock, flags);
3191 __i915_gem_request_submit(request); 3190 __i915_request_submit(request);
3192 intel_engine_init_global_seqno(request->engine, request->global_seqno); 3191 intel_engine_init_global_seqno(request->engine, request->global_seqno);
3193 spin_unlock_irqrestore(&request->engine->timeline->lock, flags); 3192 spin_unlock_irqrestore(&request->engine->timeline->lock, flags);
3194} 3193}
@@ -3213,8 +3212,10 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
3213 * rolling the global seqno forward (since this would complete requests 3212 * rolling the global seqno forward (since this would complete requests
3214 * for which we haven't set the fence error to EIO yet). 3213 * for which we haven't set the fence error to EIO yet).
3215 */ 3214 */
3216 for_each_engine(engine, i915, id) 3215 for_each_engine(engine, i915, id) {
3216 i915_gem_reset_prepare_engine(engine);
3217 engine->submit_request = nop_submit_request; 3217 engine->submit_request = nop_submit_request;
3218 }
3218 3219
3219 /* 3220 /*
3220 * Make sure no one is running the old callback before we proceed with 3221 * Make sure no one is running the old callback before we proceed with
@@ -3256,6 +3257,8 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
3256 intel_engine_init_global_seqno(engine, 3257 intel_engine_init_global_seqno(engine,
3257 intel_engine_last_submit(engine)); 3258 intel_engine_last_submit(engine));
3258 spin_unlock_irqrestore(&engine->timeline->lock, flags); 3259 spin_unlock_irqrestore(&engine->timeline->lock, flags);
3260
3261 i915_gem_reset_finish_engine(engine);
3259 } 3262 }
3260 3263
3261 wake_up_all(&i915->gpu_error.reset_queue); 3264 wake_up_all(&i915->gpu_error.reset_queue);
@@ -3281,7 +3284,7 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
3281 */ 3284 */
3282 list_for_each_entry(tl, &i915->gt.timelines, link) { 3285 list_for_each_entry(tl, &i915->gt.timelines, link) {
3283 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { 3286 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
3284 struct drm_i915_gem_request *rq; 3287 struct i915_request *rq;
3285 3288
3286 rq = i915_gem_active_peek(&tl->engine[i].last_request, 3289 rq = i915_gem_active_peek(&tl->engine[i].last_request,
3287 &i915->drm.struct_mutex); 3290 &i915->drm.struct_mutex);
@@ -3330,7 +3333,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
3330 3333
3331 /* Come back later if the device is busy... */ 3334 /* Come back later if the device is busy... */
3332 if (mutex_trylock(&dev->struct_mutex)) { 3335 if (mutex_trylock(&dev->struct_mutex)) {
3333 i915_gem_retire_requests(dev_priv); 3336 i915_retire_requests(dev_priv);
3334 mutex_unlock(&dev->struct_mutex); 3337 mutex_unlock(&dev->struct_mutex);
3335 } 3338 }
3336 3339
@@ -3418,25 +3421,22 @@ i915_gem_idle_work_handler(struct work_struct *work)
3418 container_of(work, typeof(*dev_priv), gt.idle_work.work); 3421 container_of(work, typeof(*dev_priv), gt.idle_work.work);
3419 unsigned int epoch = I915_EPOCH_INVALID; 3422 unsigned int epoch = I915_EPOCH_INVALID;
3420 bool rearm_hangcheck; 3423 bool rearm_hangcheck;
3421 ktime_t end;
3422 3424
3423 if (!READ_ONCE(dev_priv->gt.awake)) 3425 if (!READ_ONCE(dev_priv->gt.awake))
3424 return; 3426 return;
3425 3427
3426 /* 3428 /*
3427 * Wait for last execlists context complete, but bail out in case a 3429 * Wait for last execlists context complete, but bail out in case a
3428 * new request is submitted. 3430 * new request is submitted. As we don't trust the hardware, we
3431 * continue on if the wait times out. This is necessary to allow
3432 * the machine to suspend even if the hardware dies, and we will
3433 * try to recover in resume (after depriving the hardware of power,
3434 * it may be in a better mmod).
3429 */ 3435 */
3430 end = ktime_add_ms(ktime_get(), I915_IDLE_ENGINES_TIMEOUT); 3436 __wait_for(if (new_requests_since_last_retire(dev_priv)) return,
3431 do { 3437 intel_engines_are_idle(dev_priv),
3432 if (new_requests_since_last_retire(dev_priv)) 3438 I915_IDLE_ENGINES_TIMEOUT * 1000,
3433 return; 3439 10, 500);
3434
3435 if (intel_engines_are_idle(dev_priv))
3436 break;
3437
3438 usleep_range(100, 500);
3439 } while (ktime_before(ktime_get(), end));
3440 3440
3441 rearm_hangcheck = 3441 rearm_hangcheck =
3442 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 3442 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
@@ -3684,7 +3684,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
3684 if (ret) 3684 if (ret)
3685 return ret; 3685 return ret;
3686 } 3686 }
3687 i915_gem_retire_requests(i915); 3687 i915_retire_requests(i915);
3688 3688
3689 ret = wait_for_engines(i915); 3689 ret = wait_for_engines(i915);
3690 } else { 3690 } else {
@@ -4224,7 +4224,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4224 struct drm_i915_private *dev_priv = to_i915(dev); 4224 struct drm_i915_private *dev_priv = to_i915(dev);
4225 struct drm_i915_file_private *file_priv = file->driver_priv; 4225 struct drm_i915_file_private *file_priv = file->driver_priv;
4226 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; 4226 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4227 struct drm_i915_gem_request *request, *target = NULL; 4227 struct i915_request *request, *target = NULL;
4228 long ret; 4228 long ret;
4229 4229
4230 /* ABI: return -EIO if already wedged */ 4230 /* ABI: return -EIO if already wedged */
@@ -4244,16 +4244,16 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4244 target = request; 4244 target = request;
4245 } 4245 }
4246 if (target) 4246 if (target)
4247 i915_gem_request_get(target); 4247 i915_request_get(target);
4248 spin_unlock(&file_priv->mm.lock); 4248 spin_unlock(&file_priv->mm.lock);
4249 4249
4250 if (target == NULL) 4250 if (target == NULL)
4251 return 0; 4251 return 0;
4252 4252
4253 ret = i915_wait_request(target, 4253 ret = i915_request_wait(target,
4254 I915_WAIT_INTERRUPTIBLE, 4254 I915_WAIT_INTERRUPTIBLE,
4255 MAX_SCHEDULE_TIMEOUT); 4255 MAX_SCHEDULE_TIMEOUT);
4256 i915_gem_request_put(target); 4256 i915_request_put(target);
4257 4257
4258 return ret < 0 ? ret : 0; 4258 return ret < 0 ? ret : 0;
4259} 4259}
@@ -4367,7 +4367,7 @@ static __always_inline unsigned int
4367__busy_set_if_active(const struct dma_fence *fence, 4367__busy_set_if_active(const struct dma_fence *fence,
4368 unsigned int (*flag)(unsigned int id)) 4368 unsigned int (*flag)(unsigned int id))
4369{ 4369{
4370 struct drm_i915_gem_request *rq; 4370 struct i915_request *rq;
4371 4371
4372 /* We have to check the current hw status of the fence as the uABI 4372 /* We have to check the current hw status of the fence as the uABI
4373 * guarantees forward progress. We could rely on the idle worker 4373 * guarantees forward progress. We could rely on the idle worker
@@ -4380,8 +4380,8 @@ __busy_set_if_active(const struct dma_fence *fence,
4380 return 0; 4380 return 0;
4381 4381
4382 /* opencode to_request() in order to avoid const warnings */ 4382 /* opencode to_request() in order to avoid const warnings */
4383 rq = container_of(fence, struct drm_i915_gem_request, fence); 4383 rq = container_of(fence, struct i915_request, fence);
4384 if (i915_gem_request_completed(rq)) 4384 if (i915_request_completed(rq))
4385 return 0; 4385 return 0;
4386 4386
4387 return flag(rq->engine->uabi_id); 4387 return flag(rq->engine->uabi_id);
@@ -4526,8 +4526,7 @@ out:
4526} 4526}
4527 4527
4528static void 4528static void
4529frontbuffer_retire(struct i915_gem_active *active, 4529frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request)
4530 struct drm_i915_gem_request *request)
4531{ 4530{
4532 struct drm_i915_gem_object *obj = 4531 struct drm_i915_gem_object *obj =
4533 container_of(active, typeof(*obj), frontbuffer_write); 4532 container_of(active, typeof(*obj), frontbuffer_write);
@@ -4921,7 +4920,7 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
4921 i915_gem_contexts_lost(dev_priv); 4920 i915_gem_contexts_lost(dev_priv);
4922 mutex_unlock(&dev->struct_mutex); 4921 mutex_unlock(&dev->struct_mutex);
4923 4922
4924 intel_guc_suspend(dev_priv); 4923 intel_uc_suspend(dev_priv);
4925 4924
4926 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 4925 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4927 cancel_delayed_work_sync(&dev_priv->gt.retire_work); 4926 cancel_delayed_work_sync(&dev_priv->gt.retire_work);
@@ -4988,7 +4987,7 @@ void i915_gem_resume(struct drm_i915_private *i915)
4988 if (i915_gem_init_hw(i915)) 4987 if (i915_gem_init_hw(i915))
4989 goto err_wedged; 4988 goto err_wedged;
4990 4989
4991 intel_guc_resume(i915); 4990 intel_uc_resume(i915);
4992 4991
4993 /* Always reload a context for powersaving. */ 4992 /* Always reload a context for powersaving. */
4994 if (i915_gem_switch_to_kernel_context(i915)) 4993 if (i915_gem_switch_to_kernel_context(i915))
@@ -5161,9 +5160,9 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
5161 return PTR_ERR(ctx); 5160 return PTR_ERR(ctx);
5162 5161
5163 for_each_engine(engine, i915, id) { 5162 for_each_engine(engine, i915, id) {
5164 struct drm_i915_gem_request *rq; 5163 struct i915_request *rq;
5165 5164
5166 rq = i915_gem_request_alloc(engine, ctx); 5165 rq = i915_request_alloc(engine, ctx);
5167 if (IS_ERR(rq)) { 5166 if (IS_ERR(rq)) {
5168 err = PTR_ERR(rq); 5167 err = PTR_ERR(rq);
5169 goto out_ctx; 5168 goto out_ctx;
@@ -5173,7 +5172,7 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
5173 if (engine->init_context) 5172 if (engine->init_context)
5174 err = engine->init_context(rq); 5173 err = engine->init_context(rq);
5175 5174
5176 __i915_add_request(rq, true); 5175 __i915_request_add(rq, true);
5177 if (err) 5176 if (err)
5178 goto err_active; 5177 goto err_active;
5179 } 5178 }
@@ -5479,7 +5478,7 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
5479 if (!dev_priv->luts) 5478 if (!dev_priv->luts)
5480 goto err_vmas; 5479 goto err_vmas;
5481 5480
5482 dev_priv->requests = KMEM_CACHE(drm_i915_gem_request, 5481 dev_priv->requests = KMEM_CACHE(i915_request,
5483 SLAB_HWCACHE_ALIGN | 5482 SLAB_HWCACHE_ALIGN |
5484 SLAB_RECLAIM_ACCOUNT | 5483 SLAB_RECLAIM_ACCOUNT |
5485 SLAB_TYPESAFE_BY_RCU); 5484 SLAB_TYPESAFE_BY_RCU);
@@ -5612,7 +5611,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
5612void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5611void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5613{ 5612{
5614 struct drm_i915_file_private *file_priv = file->driver_priv; 5613 struct drm_i915_file_private *file_priv = file->driver_priv;
5615 struct drm_i915_gem_request *request; 5614 struct i915_request *request;
5616 5615
5617 /* Clean up our request list when the client is going away, so that 5616 /* Clean up our request list when the client is going away, so that
5618 * later retire_requests won't dereference our soon-to-be-gone 5617 * later retire_requests won't dereference our soon-to-be-gone
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index e920dab7f1b8..f54c4ff74ded 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -29,7 +29,10 @@
29 29
30#ifdef CONFIG_DRM_I915_DEBUG_GEM 30#ifdef CONFIG_DRM_I915_DEBUG_GEM
31#define GEM_BUG_ON(condition) do { if (unlikely((condition))) { \ 31#define GEM_BUG_ON(condition) do { if (unlikely((condition))) { \
32 printk(KERN_ERR "GEM_BUG_ON(%s)\n", __stringify(condition)); \ 32 pr_err("%s:%d GEM_BUG_ON(%s)\n", \
33 __func__, __LINE__, __stringify(condition)); \
34 GEM_TRACE("%s:%d GEM_BUG_ON(%s)\n", \
35 __func__, __LINE__, __stringify(condition)); \
33 BUG(); \ 36 BUG(); \
34 } \ 37 } \
35 } while(0) 38 } while(0)
@@ -54,6 +57,6 @@
54#define GEM_TRACE(...) do { } while (0) 57#define GEM_TRACE(...) do { } while (0)
55#endif 58#endif
56 59
57#define I915_NUM_ENGINES 5 60#define I915_NUM_ENGINES 8
58 61
59#endif /* __I915_GEM_H__ */ 62#endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index c93005c2e0fb..d3cbe8432f48 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -119,7 +119,7 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
119 if (!reservation_object_test_signaled_rcu(resv, true)) 119 if (!reservation_object_test_signaled_rcu(resv, true))
120 break; 120 break;
121 121
122 i915_gem_retire_requests(pool->engine->i915); 122 i915_retire_requests(pool->engine->i915);
123 GEM_BUG_ON(i915_gem_object_is_active(obj)); 123 GEM_BUG_ON(i915_gem_object_is_active(obj));
124 124
125 /* 125 /*
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 3d75f484f6e5..f2cbea7cf940 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -211,17 +211,23 @@ static void context_close(struct i915_gem_context *ctx)
211static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out) 211static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
212{ 212{
213 int ret; 213 int ret;
214 unsigned int max;
215
216 if (INTEL_GEN(dev_priv) >= 11)
217 max = GEN11_MAX_CONTEXT_HW_ID;
218 else
219 max = MAX_CONTEXT_HW_ID;
214 220
215 ret = ida_simple_get(&dev_priv->contexts.hw_ida, 221 ret = ida_simple_get(&dev_priv->contexts.hw_ida,
216 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); 222 0, max, GFP_KERNEL);
217 if (ret < 0) { 223 if (ret < 0) {
218 /* Contexts are only released when no longer active. 224 /* Contexts are only released when no longer active.
219 * Flush any pending retires to hopefully release some 225 * Flush any pending retires to hopefully release some
220 * stale contexts and try again. 226 * stale contexts and try again.
221 */ 227 */
222 i915_gem_retire_requests(dev_priv); 228 i915_retire_requests(dev_priv);
223 ret = ida_simple_get(&dev_priv->contexts.hw_ida, 229 ret = ida_simple_get(&dev_priv->contexts.hw_ida,
224 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); 230 0, max, GFP_KERNEL);
225 if (ret < 0) 231 if (ret < 0)
226 return ret; 232 return ret;
227 } 233 }
@@ -463,6 +469,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
463 469
464 /* Using the simple ida interface, the max is limited by sizeof(int) */ 470 /* Using the simple ida interface, the max is limited by sizeof(int) */
465 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); 471 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX);
472 BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > INT_MAX);
466 ida_init(&dev_priv->contexts.hw_ida); 473 ida_init(&dev_priv->contexts.hw_ida);
467 474
468 /* lowest priority; idle task */ 475 /* lowest priority; idle task */
@@ -590,28 +597,28 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
590 597
591 lockdep_assert_held(&dev_priv->drm.struct_mutex); 598 lockdep_assert_held(&dev_priv->drm.struct_mutex);
592 599
593 i915_gem_retire_requests(dev_priv); 600 i915_retire_requests(dev_priv);
594 601
595 for_each_engine(engine, dev_priv, id) { 602 for_each_engine(engine, dev_priv, id) {
596 struct drm_i915_gem_request *req; 603 struct i915_request *rq;
597 604
598 if (engine_has_idle_kernel_context(engine)) 605 if (engine_has_idle_kernel_context(engine))
599 continue; 606 continue;
600 607
601 req = i915_gem_request_alloc(engine, dev_priv->kernel_context); 608 rq = i915_request_alloc(engine, dev_priv->kernel_context);
602 if (IS_ERR(req)) 609 if (IS_ERR(rq))
603 return PTR_ERR(req); 610 return PTR_ERR(rq);
604 611
605 /* Queue this switch after all other activity */ 612 /* Queue this switch after all other activity */
606 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { 613 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
607 struct drm_i915_gem_request *prev; 614 struct i915_request *prev;
608 struct intel_timeline *tl; 615 struct intel_timeline *tl;
609 616
610 tl = &timeline->engine[engine->id]; 617 tl = &timeline->engine[engine->id];
611 prev = i915_gem_active_raw(&tl->last_request, 618 prev = i915_gem_active_raw(&tl->last_request,
612 &dev_priv->drm.struct_mutex); 619 &dev_priv->drm.struct_mutex);
613 if (prev) 620 if (prev)
614 i915_sw_fence_await_sw_fence_gfp(&req->submit, 621 i915_sw_fence_await_sw_fence_gfp(&rq->submit,
615 &prev->submit, 622 &prev->submit,
616 I915_FENCE_GFP); 623 I915_FENCE_GFP);
617 } 624 }
@@ -623,7 +630,7 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
623 * but an extra layer of paranoia before we declare the system 630 * but an extra layer of paranoia before we declare the system
624 * idle (on suspend etc) is advisable! 631 * idle (on suspend etc) is advisable!
625 */ 632 */
626 __i915_add_request(req, true); 633 __i915_request_add(rq, true);
627 } 634 }
628 635
629 return 0; 636 return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index a681c5b891ff..7854262ddfd9 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -38,8 +38,8 @@ struct drm_file;
38 38
39struct drm_i915_private; 39struct drm_i915_private;
40struct drm_i915_file_private; 40struct drm_i915_file_private;
41struct drm_i915_gem_request;
42struct i915_hw_ppgtt; 41struct i915_hw_ppgtt;
42struct i915_request;
43struct i915_vma; 43struct i915_vma;
44struct intel_ring; 44struct intel_ring;
45 45
@@ -276,7 +276,7 @@ int i915_gem_context_open(struct drm_i915_private *i915,
276 struct drm_file *file); 276 struct drm_file *file);
277void i915_gem_context_close(struct drm_file *file); 277void i915_gem_context_close(struct drm_file *file);
278 278
279int i915_switch_context(struct drm_i915_gem_request *req); 279int i915_switch_context(struct i915_request *rq);
280int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv); 280int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
281 281
282void i915_gem_context_release(struct kref *ctx_ref); 282void i915_gem_context_release(struct kref *ctx_ref);
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 60ca4f05ae94..54814a196ee4 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -168,7 +168,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
168 * retiring. 168 * retiring.
169 */ 169 */
170 if (!(flags & PIN_NONBLOCK)) 170 if (!(flags & PIN_NONBLOCK))
171 i915_gem_retire_requests(dev_priv); 171 i915_retire_requests(dev_priv);
172 else 172 else
173 phases[1] = NULL; 173 phases[1] = NULL;
174 174
@@ -293,7 +293,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
293 * retiring. 293 * retiring.
294 */ 294 */
295 if (!(flags & PIN_NONBLOCK)) 295 if (!(flags & PIN_NONBLOCK))
296 i915_gem_retire_requests(vm->i915); 296 i915_retire_requests(vm->i915);
297 297
298 check_color = vm->mm.color_adjust; 298 check_color = vm->mm.color_adjust;
299 if (check_color) { 299 if (check_color) {
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 4eb28e84fda4..8c170db8495d 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -200,7 +200,7 @@ struct i915_execbuffer {
200 struct i915_gem_context *ctx; /** context for building the request */ 200 struct i915_gem_context *ctx; /** context for building the request */
201 struct i915_address_space *vm; /** GTT and vma for the request */ 201 struct i915_address_space *vm; /** GTT and vma for the request */
202 202
203 struct drm_i915_gem_request *request; /** our request to build */ 203 struct i915_request *request; /** our request to build */
204 struct i915_vma *batch; /** identity of the batch obj/vma */ 204 struct i915_vma *batch; /** identity of the batch obj/vma */
205 205
206 /** actual size of execobj[] as we may extend it for the cmdparser */ 206 /** actual size of execobj[] as we may extend it for the cmdparser */
@@ -227,7 +227,7 @@ struct i915_execbuffer {
227 bool has_fence : 1; 227 bool has_fence : 1;
228 bool needs_unfenced : 1; 228 bool needs_unfenced : 1;
229 229
230 struct drm_i915_gem_request *rq; 230 struct i915_request *rq;
231 u32 *rq_cmd; 231 u32 *rq_cmd;
232 unsigned int rq_size; 232 unsigned int rq_size;
233 } reloc_cache; 233 } reloc_cache;
@@ -886,7 +886,7 @@ static void reloc_gpu_flush(struct reloc_cache *cache)
886 i915_gem_object_unpin_map(cache->rq->batch->obj); 886 i915_gem_object_unpin_map(cache->rq->batch->obj);
887 i915_gem_chipset_flush(cache->rq->i915); 887 i915_gem_chipset_flush(cache->rq->i915);
888 888
889 __i915_add_request(cache->rq, true); 889 __i915_request_add(cache->rq, true);
890 cache->rq = NULL; 890 cache->rq = NULL;
891} 891}
892 892
@@ -1070,7 +1070,7 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1070{ 1070{
1071 struct reloc_cache *cache = &eb->reloc_cache; 1071 struct reloc_cache *cache = &eb->reloc_cache;
1072 struct drm_i915_gem_object *obj; 1072 struct drm_i915_gem_object *obj;
1073 struct drm_i915_gem_request *rq; 1073 struct i915_request *rq;
1074 struct i915_vma *batch; 1074 struct i915_vma *batch;
1075 u32 *cmd; 1075 u32 *cmd;
1076 int err; 1076 int err;
@@ -1103,13 +1103,13 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1103 if (err) 1103 if (err)
1104 goto err_unmap; 1104 goto err_unmap;
1105 1105
1106 rq = i915_gem_request_alloc(eb->engine, eb->ctx); 1106 rq = i915_request_alloc(eb->engine, eb->ctx);
1107 if (IS_ERR(rq)) { 1107 if (IS_ERR(rq)) {
1108 err = PTR_ERR(rq); 1108 err = PTR_ERR(rq);
1109 goto err_unpin; 1109 goto err_unpin;
1110 } 1110 }
1111 1111
1112 err = i915_gem_request_await_object(rq, vma->obj, true); 1112 err = i915_request_await_object(rq, vma->obj, true);
1113 if (err) 1113 if (err)
1114 goto err_request; 1114 goto err_request;
1115 1115
@@ -1141,7 +1141,7 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1141 return 0; 1141 return 0;
1142 1142
1143err_request: 1143err_request:
1144 i915_add_request(rq); 1144 i915_request_add(rq);
1145err_unpin: 1145err_unpin:
1146 i915_vma_unpin(batch); 1146 i915_vma_unpin(batch);
1147err_unmap: 1147err_unmap:
@@ -1727,7 +1727,7 @@ slow:
1727} 1727}
1728 1728
1729static void eb_export_fence(struct i915_vma *vma, 1729static void eb_export_fence(struct i915_vma *vma,
1730 struct drm_i915_gem_request *req, 1730 struct i915_request *rq,
1731 unsigned int flags) 1731 unsigned int flags)
1732{ 1732{
1733 struct reservation_object *resv = vma->resv; 1733 struct reservation_object *resv = vma->resv;
@@ -1739,9 +1739,9 @@ static void eb_export_fence(struct i915_vma *vma,
1739 */ 1739 */
1740 reservation_object_lock(resv, NULL); 1740 reservation_object_lock(resv, NULL);
1741 if (flags & EXEC_OBJECT_WRITE) 1741 if (flags & EXEC_OBJECT_WRITE)
1742 reservation_object_add_excl_fence(resv, &req->fence); 1742 reservation_object_add_excl_fence(resv, &rq->fence);
1743 else if (reservation_object_reserve_shared(resv) == 0) 1743 else if (reservation_object_reserve_shared(resv) == 0)
1744 reservation_object_add_shared_fence(resv, &req->fence); 1744 reservation_object_add_shared_fence(resv, &rq->fence);
1745 reservation_object_unlock(resv); 1745 reservation_object_unlock(resv);
1746} 1746}
1747 1747
@@ -1757,7 +1757,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
1757 struct drm_i915_gem_object *obj = vma->obj; 1757 struct drm_i915_gem_object *obj = vma->obj;
1758 1758
1759 if (flags & EXEC_OBJECT_CAPTURE) { 1759 if (flags & EXEC_OBJECT_CAPTURE) {
1760 struct i915_gem_capture_list *capture; 1760 struct i915_capture_list *capture;
1761 1761
1762 capture = kmalloc(sizeof(*capture), GFP_KERNEL); 1762 capture = kmalloc(sizeof(*capture), GFP_KERNEL);
1763 if (unlikely(!capture)) 1763 if (unlikely(!capture))
@@ -1788,7 +1788,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
1788 if (flags & EXEC_OBJECT_ASYNC) 1788 if (flags & EXEC_OBJECT_ASYNC)
1789 continue; 1789 continue;
1790 1790
1791 err = i915_gem_request_await_object 1791 err = i915_request_await_object
1792 (eb->request, obj, flags & EXEC_OBJECT_WRITE); 1792 (eb->request, obj, flags & EXEC_OBJECT_WRITE);
1793 if (err) 1793 if (err)
1794 return err; 1794 return err;
@@ -1840,13 +1840,13 @@ static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1840} 1840}
1841 1841
1842void i915_vma_move_to_active(struct i915_vma *vma, 1842void i915_vma_move_to_active(struct i915_vma *vma,
1843 struct drm_i915_gem_request *req, 1843 struct i915_request *rq,
1844 unsigned int flags) 1844 unsigned int flags)
1845{ 1845{
1846 struct drm_i915_gem_object *obj = vma->obj; 1846 struct drm_i915_gem_object *obj = vma->obj;
1847 const unsigned int idx = req->engine->id; 1847 const unsigned int idx = rq->engine->id;
1848 1848
1849 lockdep_assert_held(&req->i915->drm.struct_mutex); 1849 lockdep_assert_held(&rq->i915->drm.struct_mutex);
1850 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1850 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1851 1851
1852 /* 1852 /*
@@ -1860,7 +1860,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,
1860 if (!i915_vma_is_active(vma)) 1860 if (!i915_vma_is_active(vma))
1861 obj->active_count++; 1861 obj->active_count++;
1862 i915_vma_set_active(vma, idx); 1862 i915_vma_set_active(vma, idx);
1863 i915_gem_active_set(&vma->last_read[idx], req); 1863 i915_gem_active_set(&vma->last_read[idx], rq);
1864 list_move_tail(&vma->vm_link, &vma->vm->active_list); 1864 list_move_tail(&vma->vm_link, &vma->vm->active_list);
1865 1865
1866 obj->write_domain = 0; 1866 obj->write_domain = 0;
@@ -1868,27 +1868,27 @@ void i915_vma_move_to_active(struct i915_vma *vma,
1868 obj->write_domain = I915_GEM_DOMAIN_RENDER; 1868 obj->write_domain = I915_GEM_DOMAIN_RENDER;
1869 1869
1870 if (intel_fb_obj_invalidate(obj, ORIGIN_CS)) 1870 if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
1871 i915_gem_active_set(&obj->frontbuffer_write, req); 1871 i915_gem_active_set(&obj->frontbuffer_write, rq);
1872 1872
1873 obj->read_domains = 0; 1873 obj->read_domains = 0;
1874 } 1874 }
1875 obj->read_domains |= I915_GEM_GPU_DOMAINS; 1875 obj->read_domains |= I915_GEM_GPU_DOMAINS;
1876 1876
1877 if (flags & EXEC_OBJECT_NEEDS_FENCE) 1877 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1878 i915_gem_active_set(&vma->last_fence, req); 1878 i915_gem_active_set(&vma->last_fence, rq);
1879} 1879}
1880 1880
1881static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req) 1881static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
1882{ 1882{
1883 u32 *cs; 1883 u32 *cs;
1884 int i; 1884 int i;
1885 1885
1886 if (!IS_GEN7(req->i915) || req->engine->id != RCS) { 1886 if (!IS_GEN7(rq->i915) || rq->engine->id != RCS) {
1887 DRM_DEBUG("sol reset is gen7/rcs only\n"); 1887 DRM_DEBUG("sol reset is gen7/rcs only\n");
1888 return -EINVAL; 1888 return -EINVAL;
1889 } 1889 }
1890 1890
1891 cs = intel_ring_begin(req, 4 * 2 + 2); 1891 cs = intel_ring_begin(rq, 4 * 2 + 2);
1892 if (IS_ERR(cs)) 1892 if (IS_ERR(cs))
1893 return PTR_ERR(cs); 1893 return PTR_ERR(cs);
1894 1894
@@ -1898,7 +1898,7 @@ static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1898 *cs++ = 0; 1898 *cs++ = 0;
1899 } 1899 }
1900 *cs++ = MI_NOOP; 1900 *cs++ = MI_NOOP;
1901 intel_ring_advance(req, cs); 1901 intel_ring_advance(rq, cs);
1902 1902
1903 return 0; 1903 return 0;
1904} 1904}
@@ -1944,10 +1944,10 @@ out:
1944} 1944}
1945 1945
1946static void 1946static void
1947add_to_client(struct drm_i915_gem_request *req, struct drm_file *file) 1947add_to_client(struct i915_request *rq, struct drm_file *file)
1948{ 1948{
1949 req->file_priv = file->driver_priv; 1949 rq->file_priv = file->driver_priv;
1950 list_add_tail(&req->client_link, &req->file_priv->mm.request_list); 1950 list_add_tail(&rq->client_link, &rq->file_priv->mm.request_list);
1951} 1951}
1952 1952
1953static int eb_submit(struct i915_execbuffer *eb) 1953static int eb_submit(struct i915_execbuffer *eb)
@@ -2151,7 +2151,7 @@ await_fence_array(struct i915_execbuffer *eb,
2151 if (!fence) 2151 if (!fence)
2152 return -EINVAL; 2152 return -EINVAL;
2153 2153
2154 err = i915_gem_request_await_dma_fence(eb->request, fence); 2154 err = i915_request_await_dma_fence(eb->request, fence);
2155 dma_fence_put(fence); 2155 dma_fence_put(fence);
2156 if (err < 0) 2156 if (err < 0)
2157 return err; 2157 return err;
@@ -2365,14 +2365,14 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2365 GEM_BUG_ON(eb.reloc_cache.rq); 2365 GEM_BUG_ON(eb.reloc_cache.rq);
2366 2366
2367 /* Allocate a request for this batch buffer nice and early. */ 2367 /* Allocate a request for this batch buffer nice and early. */
2368 eb.request = i915_gem_request_alloc(eb.engine, eb.ctx); 2368 eb.request = i915_request_alloc(eb.engine, eb.ctx);
2369 if (IS_ERR(eb.request)) { 2369 if (IS_ERR(eb.request)) {
2370 err = PTR_ERR(eb.request); 2370 err = PTR_ERR(eb.request);
2371 goto err_batch_unpin; 2371 goto err_batch_unpin;
2372 } 2372 }
2373 2373
2374 if (in_fence) { 2374 if (in_fence) {
2375 err = i915_gem_request_await_dma_fence(eb.request, in_fence); 2375 err = i915_request_await_dma_fence(eb.request, in_fence);
2376 if (err < 0) 2376 if (err < 0)
2377 goto err_request; 2377 goto err_request;
2378 } 2378 }
@@ -2400,10 +2400,10 @@ i915_gem_do_execbuffer(struct drm_device *dev,
2400 */ 2400 */
2401 eb.request->batch = eb.batch; 2401 eb.request->batch = eb.batch;
2402 2402
2403 trace_i915_gem_request_queue(eb.request, eb.batch_flags); 2403 trace_i915_request_queue(eb.request, eb.batch_flags);
2404 err = eb_submit(&eb); 2404 err = eb_submit(&eb);
2405err_request: 2405err_request:
2406 __i915_add_request(eb.request, err == 0); 2406 __i915_request_add(eb.request, err == 0);
2407 add_to_client(eb.request, file); 2407 add_to_client(eb.request, file);
2408 2408
2409 if (fences) 2409 if (fences)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index cd5984246bc3..21d72f695adb 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -765,16 +765,16 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
765} 765}
766 766
767/* Broadwell Page Directory Pointer Descriptors */ 767/* Broadwell Page Directory Pointer Descriptors */
768static int gen8_write_pdp(struct drm_i915_gem_request *req, 768static int gen8_write_pdp(struct i915_request *rq,
769 unsigned entry, 769 unsigned entry,
770 dma_addr_t addr) 770 dma_addr_t addr)
771{ 771{
772 struct intel_engine_cs *engine = req->engine; 772 struct intel_engine_cs *engine = rq->engine;
773 u32 *cs; 773 u32 *cs;
774 774
775 BUG_ON(entry >= 4); 775 BUG_ON(entry >= 4);
776 776
777 cs = intel_ring_begin(req, 6); 777 cs = intel_ring_begin(rq, 6);
778 if (IS_ERR(cs)) 778 if (IS_ERR(cs))
779 return PTR_ERR(cs); 779 return PTR_ERR(cs);
780 780
@@ -784,20 +784,20 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req,
784 *cs++ = MI_LOAD_REGISTER_IMM(1); 784 *cs++ = MI_LOAD_REGISTER_IMM(1);
785 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry)); 785 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, entry));
786 *cs++ = lower_32_bits(addr); 786 *cs++ = lower_32_bits(addr);
787 intel_ring_advance(req, cs); 787 intel_ring_advance(rq, cs);
788 788
789 return 0; 789 return 0;
790} 790}
791 791
792static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt, 792static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt,
793 struct drm_i915_gem_request *req) 793 struct i915_request *rq)
794{ 794{
795 int i, ret; 795 int i, ret;
796 796
797 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { 797 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
798 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 798 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
799 799
800 ret = gen8_write_pdp(req, i, pd_daddr); 800 ret = gen8_write_pdp(rq, i, pd_daddr);
801 if (ret) 801 if (ret)
802 return ret; 802 return ret;
803 } 803 }
@@ -806,9 +806,9 @@ static int gen8_mm_switch_3lvl(struct i915_hw_ppgtt *ppgtt,
806} 806}
807 807
808static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt, 808static int gen8_mm_switch_4lvl(struct i915_hw_ppgtt *ppgtt,
809 struct drm_i915_gem_request *req) 809 struct i915_request *rq)
810{ 810{
811 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); 811 return gen8_write_pdp(rq, 0, px_dma(&ppgtt->pml4));
812} 812}
813 813
814/* PDE TLBs are a pain to invalidate on GEN8+. When we modify 814/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
@@ -1732,13 +1732,13 @@ static inline u32 get_pd_offset(struct i915_hw_ppgtt *ppgtt)
1732} 1732}
1733 1733
1734static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, 1734static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1735 struct drm_i915_gem_request *req) 1735 struct i915_request *rq)
1736{ 1736{
1737 struct intel_engine_cs *engine = req->engine; 1737 struct intel_engine_cs *engine = rq->engine;
1738 u32 *cs; 1738 u32 *cs;
1739 1739
1740 /* NB: TLBs must be flushed and invalidated before a switch */ 1740 /* NB: TLBs must be flushed and invalidated before a switch */
1741 cs = intel_ring_begin(req, 6); 1741 cs = intel_ring_begin(rq, 6);
1742 if (IS_ERR(cs)) 1742 if (IS_ERR(cs))
1743 return PTR_ERR(cs); 1743 return PTR_ERR(cs);
1744 1744
@@ -1748,19 +1748,19 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1748 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); 1748 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1749 *cs++ = get_pd_offset(ppgtt); 1749 *cs++ = get_pd_offset(ppgtt);
1750 *cs++ = MI_NOOP; 1750 *cs++ = MI_NOOP;
1751 intel_ring_advance(req, cs); 1751 intel_ring_advance(rq, cs);
1752 1752
1753 return 0; 1753 return 0;
1754} 1754}
1755 1755
1756static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, 1756static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1757 struct drm_i915_gem_request *req) 1757 struct i915_request *rq)
1758{ 1758{
1759 struct intel_engine_cs *engine = req->engine; 1759 struct intel_engine_cs *engine = rq->engine;
1760 u32 *cs; 1760 u32 *cs;
1761 1761
1762 /* NB: TLBs must be flushed and invalidated before a switch */ 1762 /* NB: TLBs must be flushed and invalidated before a switch */
1763 cs = intel_ring_begin(req, 6); 1763 cs = intel_ring_begin(rq, 6);
1764 if (IS_ERR(cs)) 1764 if (IS_ERR(cs))
1765 return PTR_ERR(cs); 1765 return PTR_ERR(cs);
1766 1766
@@ -1770,16 +1770,16 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1770 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine)); 1770 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
1771 *cs++ = get_pd_offset(ppgtt); 1771 *cs++ = get_pd_offset(ppgtt);
1772 *cs++ = MI_NOOP; 1772 *cs++ = MI_NOOP;
1773 intel_ring_advance(req, cs); 1773 intel_ring_advance(rq, cs);
1774 1774
1775 return 0; 1775 return 0;
1776} 1776}
1777 1777
1778static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, 1778static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1779 struct drm_i915_gem_request *req) 1779 struct i915_request *rq)
1780{ 1780{
1781 struct intel_engine_cs *engine = req->engine; 1781 struct intel_engine_cs *engine = rq->engine;
1782 struct drm_i915_private *dev_priv = req->i915; 1782 struct drm_i915_private *dev_priv = rq->i915;
1783 1783
1784 I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); 1784 I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
1785 I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); 1785 I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index a42890d9af38..6efc017e8bb3 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -39,7 +39,8 @@
39#include <linux/pagevec.h> 39#include <linux/pagevec.h>
40 40
41#include "i915_gem_timeline.h" 41#include "i915_gem_timeline.h"
42#include "i915_gem_request.h" 42
43#include "i915_request.h"
43#include "i915_selftest.h" 44#include "i915_selftest.h"
44 45
45#define I915_GTT_PAGE_SIZE_4K BIT(12) 46#define I915_GTT_PAGE_SIZE_4K BIT(12)
@@ -398,7 +399,7 @@ struct i915_hw_ppgtt {
398 gen6_pte_t __iomem *pd_addr; 399 gen6_pte_t __iomem *pd_addr;
399 400
400 int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, 401 int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,
401 struct drm_i915_gem_request *req); 402 struct i915_request *rq);
402 void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); 403 void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
403}; 404};
404 405
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index ca2b3b62569d..54f00b350779 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -33,7 +33,7 @@
33 33
34#include <drm/i915_drm.h> 34#include <drm/i915_drm.h>
35 35
36#include "i915_gem_request.h" 36#include "i915_request.h"
37#include "i915_selftest.h" 37#include "i915_selftest.h"
38 38
39struct drm_i915_gem_object; 39struct drm_i915_gem_object;
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index f7fc0df251ac..1036e8686916 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -177,7 +177,7 @@ err:
177 177
178#undef OUT_BATCH 178#undef OUT_BATCH
179 179
180int i915_gem_render_state_emit(struct drm_i915_gem_request *rq) 180int i915_gem_render_state_emit(struct i915_request *rq)
181{ 181{
182 struct intel_engine_cs *engine = rq->engine; 182 struct intel_engine_cs *engine = rq->engine;
183 struct intel_render_state so = {}; /* keep the compiler happy */ 183 struct intel_render_state so = {}; /* keep the compiler happy */
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 86369520482e..112cda8fa1a8 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -24,8 +24,8 @@
24#ifndef _I915_GEM_RENDER_STATE_H_ 24#ifndef _I915_GEM_RENDER_STATE_H_
25#define _I915_GEM_RENDER_STATE_H_ 25#define _I915_GEM_RENDER_STATE_H_
26 26
27struct drm_i915_gem_request; 27struct i915_request;
28 28
29int i915_gem_render_state_emit(struct drm_i915_gem_request *rq); 29int i915_gem_render_state_emit(struct i915_request *rq);
30 30
31#endif /* _I915_GEM_RENDER_STATE_H_ */ 31#endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 0e158f9287c4..5757fb7c4b5a 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -175,7 +175,7 @@ i915_gem_shrink(struct drm_i915_private *i915,
175 i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED); 175 i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
176 176
177 trace_i915_gem_shrink(i915, target, flags); 177 trace_i915_gem_shrink(i915, target, flags);
178 i915_gem_retire_requests(i915); 178 i915_retire_requests(i915);
179 179
180 /* 180 /*
181 * Unbinding of objects will require HW access; Let us not wake the 181 * Unbinding of objects will require HW access; Let us not wake the
@@ -267,7 +267,7 @@ i915_gem_shrink(struct drm_i915_private *i915,
267 if (flags & I915_SHRINK_BOUND) 267 if (flags & I915_SHRINK_BOUND)
268 intel_runtime_pm_put(i915); 268 intel_runtime_pm_put(i915);
269 269
270 i915_gem_retire_requests(i915); 270 i915_retire_requests(i915);
271 271
272 shrinker_unlock(i915, unlock); 272 shrinker_unlock(i915, unlock);
273 273
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
index b5a22400a01f..33e01bf6aa36 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -27,9 +27,9 @@
27 27
28#include <linux/list.h> 28#include <linux/list.h>
29 29
30#include "i915_utils.h" 30#include "i915_request.h"
31#include "i915_gem_request.h"
32#include "i915_syncmap.h" 31#include "i915_syncmap.h"
32#include "i915_utils.h"
33 33
34struct i915_gem_timeline; 34struct i915_gem_timeline;
35 35
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 65c0bef73ee5..f89ac7a8f95f 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -586,6 +586,7 @@ static void err_print_capabilities(struct drm_i915_error_state_buf *m,
586 586
587 intel_device_info_dump_flags(info, &p); 587 intel_device_info_dump_flags(info, &p);
588 intel_driver_caps_print(caps, &p); 588 intel_driver_caps_print(caps, &p);
589 intel_device_info_dump_topology(&info->sseu, &p);
589} 590}
590 591
591static void err_print_params(struct drm_i915_error_state_buf *m, 592static void err_print_params(struct drm_i915_error_state_buf *m,
@@ -991,7 +992,7 @@ out:
991static inline uint32_t 992static inline uint32_t
992__active_get_seqno(struct i915_gem_active *active) 993__active_get_seqno(struct i915_gem_active *active)
993{ 994{
994 struct drm_i915_gem_request *request; 995 struct i915_request *request;
995 996
996 request = __i915_gem_active_peek(active); 997 request = __i915_gem_active_peek(active);
997 return request ? request->global_seqno : 0; 998 return request ? request->global_seqno : 0;
@@ -1000,7 +1001,7 @@ __active_get_seqno(struct i915_gem_active *active)
1000static inline int 1001static inline int
1001__active_get_engine_id(struct i915_gem_active *active) 1002__active_get_engine_id(struct i915_gem_active *active)
1002{ 1003{
1003 struct drm_i915_gem_request *request; 1004 struct i915_request *request;
1004 1005
1005 request = __i915_gem_active_peek(active); 1006 request = __i915_gem_active_peek(active);
1006 return request ? request->engine->id : -1; 1007 return request ? request->engine->id : -1;
@@ -1084,9 +1085,9 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
1084 return error_code; 1085 return error_code;
1085} 1086}
1086 1087
1087static void i915_gem_record_fences(struct drm_i915_private *dev_priv, 1088static void gem_record_fences(struct i915_gpu_state *error)
1088 struct i915_gpu_state *error)
1089{ 1089{
1090 struct drm_i915_private *dev_priv = error->i915;
1090 int i; 1091 int i;
1091 1092
1092 if (INTEL_GEN(dev_priv) >= 6) { 1093 if (INTEL_GEN(dev_priv) >= 6) {
@@ -1102,27 +1103,6 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
1102 error->nfence = i; 1103 error->nfence = i;
1103} 1104}
1104 1105
1105static inline u32
1106gen8_engine_sync_index(struct intel_engine_cs *engine,
1107 struct intel_engine_cs *other)
1108{
1109 int idx;
1110
1111 /*
1112 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
1113 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
1114 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
1115 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
1116 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
1117 */
1118
1119 idx = (other - engine) - 1;
1120 if (idx < 0)
1121 idx += I915_NUM_ENGINES;
1122
1123 return idx;
1124}
1125
1126static void gen6_record_semaphore_state(struct intel_engine_cs *engine, 1106static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
1127 struct drm_i915_error_engine *ee) 1107 struct drm_i915_error_engine *ee)
1128{ 1108{
@@ -1293,7 +1273,7 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
1293 } 1273 }
1294} 1274}
1295 1275
1296static void record_request(struct drm_i915_gem_request *request, 1276static void record_request(struct i915_request *request,
1297 struct drm_i915_error_request *erq) 1277 struct drm_i915_error_request *erq)
1298{ 1278{
1299 erq->context = request->ctx->hw_id; 1279 erq->context = request->ctx->hw_id;
@@ -1310,10 +1290,10 @@ static void record_request(struct drm_i915_gem_request *request,
1310} 1290}
1311 1291
1312static void engine_record_requests(struct intel_engine_cs *engine, 1292static void engine_record_requests(struct intel_engine_cs *engine,
1313 struct drm_i915_gem_request *first, 1293 struct i915_request *first,
1314 struct drm_i915_error_engine *ee) 1294 struct drm_i915_error_engine *ee)
1315{ 1295{
1316 struct drm_i915_gem_request *request; 1296 struct i915_request *request;
1317 int count; 1297 int count;
1318 1298
1319 count = 0; 1299 count = 0;
@@ -1363,7 +1343,7 @@ static void error_record_engine_execlists(struct intel_engine_cs *engine,
1363 unsigned int n; 1343 unsigned int n;
1364 1344
1365 for (n = 0; n < execlists_num_ports(execlists); n++) { 1345 for (n = 0; n < execlists_num_ports(execlists); n++) {
1366 struct drm_i915_gem_request *rq = port_request(&execlists->port[n]); 1346 struct i915_request *rq = port_request(&execlists->port[n]);
1367 1347
1368 if (!rq) 1348 if (!rq)
1369 break; 1349 break;
@@ -1398,10 +1378,10 @@ static void record_context(struct drm_i915_error_context *e,
1398 e->active = atomic_read(&ctx->active_count); 1378 e->active = atomic_read(&ctx->active_count);
1399} 1379}
1400 1380
1401static void request_record_user_bo(struct drm_i915_gem_request *request, 1381static void request_record_user_bo(struct i915_request *request,
1402 struct drm_i915_error_engine *ee) 1382 struct drm_i915_error_engine *ee)
1403{ 1383{
1404 struct i915_gem_capture_list *c; 1384 struct i915_capture_list *c;
1405 struct drm_i915_error_object **bo; 1385 struct drm_i915_error_object **bo;
1406 long count; 1386 long count;
1407 1387
@@ -1445,16 +1425,16 @@ capture_object(struct drm_i915_private *dev_priv,
1445 } 1425 }
1446} 1426}
1447 1427
1448static void i915_gem_record_rings(struct drm_i915_private *dev_priv, 1428static void gem_record_rings(struct i915_gpu_state *error)
1449 struct i915_gpu_state *error)
1450{ 1429{
1451 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1430 struct drm_i915_private *i915 = error->i915;
1431 struct i915_ggtt *ggtt = &i915->ggtt;
1452 int i; 1432 int i;
1453 1433
1454 for (i = 0; i < I915_NUM_ENGINES; i++) { 1434 for (i = 0; i < I915_NUM_ENGINES; i++) {
1455 struct intel_engine_cs *engine = dev_priv->engine[i]; 1435 struct intel_engine_cs *engine = i915->engine[i];
1456 struct drm_i915_error_engine *ee = &error->engine[i]; 1436 struct drm_i915_error_engine *ee = &error->engine[i];
1457 struct drm_i915_gem_request *request; 1437 struct i915_request *request;
1458 1438
1459 ee->engine_id = -1; 1439 ee->engine_id = -1;
1460 1440
@@ -1481,17 +1461,16 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
1481 * by userspace. 1461 * by userspace.
1482 */ 1462 */
1483 ee->batchbuffer = 1463 ee->batchbuffer =
1484 i915_error_object_create(dev_priv, 1464 i915_error_object_create(i915, request->batch);
1485 request->batch);
1486 1465
1487 if (HAS_BROKEN_CS_TLB(dev_priv)) 1466 if (HAS_BROKEN_CS_TLB(i915))
1488 ee->wa_batchbuffer = 1467 ee->wa_batchbuffer =
1489 i915_error_object_create(dev_priv, 1468 i915_error_object_create(i915,
1490 engine->scratch); 1469 engine->scratch);
1491 request_record_user_bo(request, ee); 1470 request_record_user_bo(request, ee);
1492 1471
1493 ee->ctx = 1472 ee->ctx =
1494 i915_error_object_create(dev_priv, 1473 i915_error_object_create(i915,
1495 request->ctx->engine[i].state); 1474 request->ctx->engine[i].state);
1496 1475
1497 error->simulated |= 1476 error->simulated |=
@@ -1505,27 +1484,24 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
1505 ee->cpu_ring_head = ring->head; 1484 ee->cpu_ring_head = ring->head;
1506 ee->cpu_ring_tail = ring->tail; 1485 ee->cpu_ring_tail = ring->tail;
1507 ee->ringbuffer = 1486 ee->ringbuffer =
1508 i915_error_object_create(dev_priv, ring->vma); 1487 i915_error_object_create(i915, ring->vma);
1509 1488
1510 engine_record_requests(engine, request, ee); 1489 engine_record_requests(engine, request, ee);
1511 } 1490 }
1512 1491
1513 ee->hws_page = 1492 ee->hws_page =
1514 i915_error_object_create(dev_priv, 1493 i915_error_object_create(i915,
1515 engine->status_page.vma); 1494 engine->status_page.vma);
1516 1495
1517 ee->wa_ctx = 1496 ee->wa_ctx = i915_error_object_create(i915, engine->wa_ctx.vma);
1518 i915_error_object_create(dev_priv, engine->wa_ctx.vma);
1519 1497
1520 ee->default_state = 1498 ee->default_state = capture_object(i915, engine->default_state);
1521 capture_object(dev_priv, engine->default_state);
1522 } 1499 }
1523} 1500}
1524 1501
1525static void i915_gem_capture_vm(struct drm_i915_private *dev_priv, 1502static void gem_capture_vm(struct i915_gpu_state *error,
1526 struct i915_gpu_state *error, 1503 struct i915_address_space *vm,
1527 struct i915_address_space *vm, 1504 int idx)
1528 int idx)
1529{ 1505{
1530 struct drm_i915_error_buffer *active_bo; 1506 struct drm_i915_error_buffer *active_bo;
1531 struct i915_vma *vma; 1507 struct i915_vma *vma;
@@ -1548,8 +1524,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
1548 error->active_bo_count[idx] = count; 1524 error->active_bo_count[idx] = count;
1549} 1525}
1550 1526
1551static void i915_capture_active_buffers(struct drm_i915_private *dev_priv, 1527static void capture_active_buffers(struct i915_gpu_state *error)
1552 struct i915_gpu_state *error)
1553{ 1528{
1554 int cnt = 0, i, j; 1529 int cnt = 0, i, j;
1555 1530
@@ -1569,14 +1544,13 @@ static void i915_capture_active_buffers(struct drm_i915_private *dev_priv,
1569 for (j = 0; j < i && !found; j++) 1544 for (j = 0; j < i && !found; j++)
1570 found = error->engine[j].vm == ee->vm; 1545 found = error->engine[j].vm == ee->vm;
1571 if (!found) 1546 if (!found)
1572 i915_gem_capture_vm(dev_priv, error, ee->vm, cnt++); 1547 gem_capture_vm(error, ee->vm, cnt++);
1573 } 1548 }
1574} 1549}
1575 1550
1576static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv, 1551static void capture_pinned_buffers(struct i915_gpu_state *error)
1577 struct i915_gpu_state *error)
1578{ 1552{
1579 struct i915_address_space *vm = &dev_priv->ggtt.base; 1553 struct i915_address_space *vm = &error->i915->ggtt.base;
1580 struct drm_i915_error_buffer *bo; 1554 struct drm_i915_error_buffer *bo;
1581 struct i915_vma *vma; 1555 struct i915_vma *vma;
1582 int count_inactive, count_active; 1556 int count_inactive, count_active;
@@ -1626,9 +1600,9 @@ static void capture_uc_state(struct i915_gpu_state *error)
1626} 1600}
1627 1601
1628/* Capture all registers which don't fit into another category. */ 1602/* Capture all registers which don't fit into another category. */
1629static void i915_capture_reg_state(struct drm_i915_private *dev_priv, 1603static void capture_reg_state(struct i915_gpu_state *error)
1630 struct i915_gpu_state *error)
1631{ 1604{
1605 struct drm_i915_private *dev_priv = error->i915;
1632 int i; 1606 int i;
1633 1607
1634 /* General organization 1608 /* General organization
@@ -1725,24 +1699,25 @@ static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
1725 engine_mask ? "reset" : "continue"); 1699 engine_mask ? "reset" : "continue");
1726} 1700}
1727 1701
1728static void i915_capture_gen_state(struct drm_i915_private *dev_priv, 1702static void capture_gen_state(struct i915_gpu_state *error)
1729 struct i915_gpu_state *error)
1730{ 1703{
1731 error->awake = dev_priv->gt.awake; 1704 struct drm_i915_private *i915 = error->i915;
1732 error->wakelock = atomic_read(&dev_priv->runtime_pm.wakeref_count); 1705
1733 error->suspended = dev_priv->runtime_pm.suspended; 1706 error->awake = i915->gt.awake;
1707 error->wakelock = atomic_read(&i915->runtime_pm.wakeref_count);
1708 error->suspended = i915->runtime_pm.suspended;
1734 1709
1735 error->iommu = -1; 1710 error->iommu = -1;
1736#ifdef CONFIG_INTEL_IOMMU 1711#ifdef CONFIG_INTEL_IOMMU
1737 error->iommu = intel_iommu_gfx_mapped; 1712 error->iommu = intel_iommu_gfx_mapped;
1738#endif 1713#endif
1739 error->reset_count = i915_reset_count(&dev_priv->gpu_error); 1714 error->reset_count = i915_reset_count(&i915->gpu_error);
1740 error->suspend_count = dev_priv->suspend_count; 1715 error->suspend_count = i915->suspend_count;
1741 1716
1742 memcpy(&error->device_info, 1717 memcpy(&error->device_info,
1743 INTEL_INFO(dev_priv), 1718 INTEL_INFO(i915),
1744 sizeof(error->device_info)); 1719 sizeof(error->device_info));
1745 error->driver_caps = dev_priv->caps; 1720 error->driver_caps = i915->caps;
1746} 1721}
1747 1722
1748static __always_inline void dup_param(const char *type, void *x) 1723static __always_inline void dup_param(const char *type, void *x)
@@ -1769,14 +1744,13 @@ static int capture(void *data)
1769 error->i915->gt.last_init_time); 1744 error->i915->gt.last_init_time);
1770 1745
1771 capture_params(error); 1746 capture_params(error);
1747 capture_gen_state(error);
1772 capture_uc_state(error); 1748 capture_uc_state(error);
1773 1749 capture_reg_state(error);
1774 i915_capture_gen_state(error->i915, error); 1750 gem_record_fences(error);
1775 i915_capture_reg_state(error->i915, error); 1751 gem_record_rings(error);
1776 i915_gem_record_fences(error->i915, error); 1752 capture_active_buffers(error);
1777 i915_gem_record_rings(error->i915, error); 1753 capture_pinned_buffers(error);
1778 i915_capture_active_buffers(error->i915, error);
1779 i915_capture_pinned_buffers(error->i915, error);
1780 1754
1781 error->overlay = intel_overlay_capture_error_state(error->i915); 1755 error->overlay = intel_overlay_capture_error_state(error->i915);
1782 error->display = intel_display_capture_error_state(error->i915); 1756 error->display = intel_display_capture_error_state(error->i915);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 17de6cef2a30..633c18785c1e 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -415,6 +415,9 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
415 if (READ_ONCE(rps->interrupts_enabled)) 415 if (READ_ONCE(rps->interrupts_enabled))
416 return; 416 return;
417 417
418 if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
419 return;
420
418 spin_lock_irq(&dev_priv->irq_lock); 421 spin_lock_irq(&dev_priv->irq_lock);
419 WARN_ON_ONCE(rps->pm_iir); 422 WARN_ON_ONCE(rps->pm_iir);
420 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); 423 WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
@@ -431,6 +434,9 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
431 if (!READ_ONCE(rps->interrupts_enabled)) 434 if (!READ_ONCE(rps->interrupts_enabled))
432 return; 435 return;
433 436
437 if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
438 return;
439
434 spin_lock_irq(&dev_priv->irq_lock); 440 spin_lock_irq(&dev_priv->irq_lock);
435 rps->interrupts_enabled = false; 441 rps->interrupts_enabled = false;
436 442
@@ -1071,7 +1077,7 @@ static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
1071 1077
1072static void notify_ring(struct intel_engine_cs *engine) 1078static void notify_ring(struct intel_engine_cs *engine)
1073{ 1079{
1074 struct drm_i915_gem_request *rq = NULL; 1080 struct i915_request *rq = NULL;
1075 struct intel_wait *wait; 1081 struct intel_wait *wait;
1076 1082
1077 if (!engine->breadcrumbs.irq_armed) 1083 if (!engine->breadcrumbs.irq_armed)
@@ -1098,13 +1104,13 @@ static void notify_ring(struct intel_engine_cs *engine)
1098 */ 1104 */
1099 if (i915_seqno_passed(intel_engine_get_seqno(engine), 1105 if (i915_seqno_passed(intel_engine_get_seqno(engine),
1100 wait->seqno)) { 1106 wait->seqno)) {
1101 struct drm_i915_gem_request *waiter = wait->request; 1107 struct i915_request *waiter = wait->request;
1102 1108
1103 wakeup = true; 1109 wakeup = true;
1104 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1110 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1105 &waiter->fence.flags) && 1111 &waiter->fence.flags) &&
1106 intel_wait_check_request(wait, waiter)) 1112 intel_wait_check_request(wait, waiter))
1107 rq = i915_gem_request_get(waiter); 1113 rq = i915_request_get(waiter);
1108 } 1114 }
1109 1115
1110 if (wakeup) 1116 if (wakeup)
@@ -1117,7 +1123,8 @@ static void notify_ring(struct intel_engine_cs *engine)
1117 1123
1118 if (rq) { 1124 if (rq) {
1119 dma_fence_signal(&rq->fence); 1125 dma_fence_signal(&rq->fence);
1120 i915_gem_request_put(rq); 1126 GEM_BUG_ON(!i915_request_completed(rq));
1127 i915_request_put(rq);
1121 } 1128 }
1122 1129
1123 trace_intel_engine_notify(engine, wait); 1130 trace_intel_engine_notify(engine, wait);
@@ -2755,6 +2762,156 @@ static void __fini_wedge(struct wedge_me *w)
2755 (W)->i915; \ 2762 (W)->i915; \
2756 __fini_wedge((W))) 2763 __fini_wedge((W)))
2757 2764
2765static __always_inline void
2766gen11_cs_irq_handler(struct intel_engine_cs * const engine, const u32 iir)
2767{
2768 gen8_cs_irq_handler(engine, iir, 0);
2769}
2770
2771static void
2772gen11_gt_engine_irq_handler(struct drm_i915_private * const i915,
2773 const unsigned int bank,
2774 const unsigned int engine_n,
2775 const u16 iir)
2776{
2777 struct intel_engine_cs ** const engine = i915->engine;
2778
2779 switch (bank) {
2780 case 0:
2781 switch (engine_n) {
2782
2783 case GEN11_RCS0:
2784 return gen11_cs_irq_handler(engine[RCS], iir);
2785
2786 case GEN11_BCS:
2787 return gen11_cs_irq_handler(engine[BCS], iir);
2788 }
2789 case 1:
2790 switch (engine_n) {
2791
2792 case GEN11_VCS(0):
2793 return gen11_cs_irq_handler(engine[_VCS(0)], iir);
2794 case GEN11_VCS(1):
2795 return gen11_cs_irq_handler(engine[_VCS(1)], iir);
2796 case GEN11_VCS(2):
2797 return gen11_cs_irq_handler(engine[_VCS(2)], iir);
2798 case GEN11_VCS(3):
2799 return gen11_cs_irq_handler(engine[_VCS(3)], iir);
2800
2801 case GEN11_VECS(0):
2802 return gen11_cs_irq_handler(engine[_VECS(0)], iir);
2803 case GEN11_VECS(1):
2804 return gen11_cs_irq_handler(engine[_VECS(1)], iir);
2805 }
2806 }
2807}
2808
2809static u32
2810gen11_gt_engine_intr(struct drm_i915_private * const i915,
2811 const unsigned int bank, const unsigned int bit)
2812{
2813 void __iomem * const regs = i915->regs;
2814 u32 timeout_ts;
2815 u32 ident;
2816
2817 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
2818
2819 /*
2820 * NB: Specs do not specify how long to spin wait,
2821 * so we do ~100us as an educated guess.
2822 */
2823 timeout_ts = (local_clock() >> 10) + 100;
2824 do {
2825 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
2826 } while (!(ident & GEN11_INTR_DATA_VALID) &&
2827 !time_after32(local_clock() >> 10, timeout_ts));
2828
2829 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
2830 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
2831 bank, bit, ident);
2832 return 0;
2833 }
2834
2835 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
2836 GEN11_INTR_DATA_VALID);
2837
2838 return ident & GEN11_INTR_ENGINE_MASK;
2839}
2840
2841static void
2842gen11_gt_irq_handler(struct drm_i915_private * const i915,
2843 const u32 master_ctl)
2844{
2845 void __iomem * const regs = i915->regs;
2846 unsigned int bank;
2847
2848 for (bank = 0; bank < 2; bank++) {
2849 unsigned long intr_dw;
2850 unsigned int bit;
2851
2852 if (!(master_ctl & GEN11_GT_DW_IRQ(bank)))
2853 continue;
2854
2855 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
2856
2857 if (unlikely(!intr_dw)) {
2858 DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
2859 continue;
2860 }
2861
2862 for_each_set_bit(bit, &intr_dw, 32) {
2863 const u16 iir = gen11_gt_engine_intr(i915, bank, bit);
2864
2865 if (unlikely(!iir))
2866 continue;
2867
2868 gen11_gt_engine_irq_handler(i915, bank, bit, iir);
2869 }
2870
2871 /* Clear must be after shared has been served for engine */
2872 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
2873 }
2874}
2875
2876static irqreturn_t gen11_irq_handler(int irq, void *arg)
2877{
2878 struct drm_i915_private * const i915 = to_i915(arg);
2879 void __iomem * const regs = i915->regs;
2880 u32 master_ctl;
2881
2882 if (!intel_irqs_enabled(i915))
2883 return IRQ_NONE;
2884
2885 master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ);
2886 master_ctl &= ~GEN11_MASTER_IRQ;
2887 if (!master_ctl)
2888 return IRQ_NONE;
2889
2890 /* Disable interrupts. */
2891 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, 0);
2892
2893 /* Find, clear, then process each source of interrupt. */
2894 gen11_gt_irq_handler(i915, master_ctl);
2895
2896 /* IRQs are synced during runtime_suspend, we don't require a wakeref */
2897 if (master_ctl & GEN11_DISPLAY_IRQ) {
2898 const u32 disp_ctl = raw_reg_read(regs, GEN11_DISPLAY_INT_CTL);
2899
2900 disable_rpm_wakeref_asserts(i915);
2901 /*
2902 * GEN11_DISPLAY_INT_CTL has same format as GEN8_MASTER_IRQ
2903 * for the display related bits.
2904 */
2905 gen8_de_irq_handler(i915, disp_ctl);
2906 enable_rpm_wakeref_asserts(i915);
2907 }
2908
2909 /* Acknowledge and enable interrupts. */
2910 raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ | master_ctl);
2911
2912 return IRQ_HANDLED;
2913}
2914
2758/** 2915/**
2759 * i915_reset_device - do process context error handling work 2916 * i915_reset_device - do process context error handling work
2760 * @dev_priv: i915 device private 2917 * @dev_priv: i915 device private
@@ -3180,6 +3337,42 @@ static void gen8_irq_reset(struct drm_device *dev)
3180 ibx_irq_reset(dev_priv); 3337 ibx_irq_reset(dev_priv);
3181} 3338}
3182 3339
3340static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
3341{
3342 /* Disable RCS, BCS, VCS and VECS class engines. */
3343 I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, 0);
3344 I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, 0);
3345
3346 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
3347 I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~0);
3348 I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~0);
3349 I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~0);
3350 I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~0);
3351 I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~0);
3352}
3353
3354static void gen11_irq_reset(struct drm_device *dev)
3355{
3356 struct drm_i915_private *dev_priv = dev->dev_private;
3357 int pipe;
3358
3359 I915_WRITE(GEN11_GFX_MSTR_IRQ, 0);
3360 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3361
3362 gen11_gt_irq_reset(dev_priv);
3363
3364 I915_WRITE(GEN11_DISPLAY_INT_CTL, 0);
3365
3366 for_each_pipe(dev_priv, pipe)
3367 if (intel_display_power_is_enabled(dev_priv,
3368 POWER_DOMAIN_PIPE(pipe)))
3369 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3370
3371 GEN3_IRQ_RESET(GEN8_DE_PORT_);
3372 GEN3_IRQ_RESET(GEN8_DE_MISC_);
3373 GEN3_IRQ_RESET(GEN8_PCU_);
3374}
3375
3183void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, 3376void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3184 u8 pipe_mask) 3377 u8 pipe_mask)
3185{ 3378{
@@ -3677,6 +3870,41 @@ static int gen8_irq_postinstall(struct drm_device *dev)
3677 return 0; 3870 return 0;
3678} 3871}
3679 3872
3873static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3874{
3875 const u32 irqs = GT_RENDER_USER_INTERRUPT | GT_CONTEXT_SWITCH_INTERRUPT;
3876
3877 BUILD_BUG_ON(irqs & 0xffff0000);
3878
3879 /* Enable RCS, BCS, VCS and VECS class interrupts. */
3880 I915_WRITE(GEN11_RENDER_COPY_INTR_ENABLE, irqs << 16 | irqs);
3881 I915_WRITE(GEN11_VCS_VECS_INTR_ENABLE, irqs << 16 | irqs);
3882
3883 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
3884 I915_WRITE(GEN11_RCS0_RSVD_INTR_MASK, ~(irqs << 16));
3885 I915_WRITE(GEN11_BCS_RSVD_INTR_MASK, ~(irqs << 16));
3886 I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK, ~(irqs | irqs << 16));
3887 I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK, ~(irqs | irqs << 16));
3888 I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~(irqs | irqs << 16));
3889
3890 dev_priv->pm_imr = 0xffffffff; /* TODO */
3891}
3892
3893static int gen11_irq_postinstall(struct drm_device *dev)
3894{
3895 struct drm_i915_private *dev_priv = dev->dev_private;
3896
3897 gen11_gt_irq_postinstall(dev_priv);
3898 gen8_de_irq_postinstall(dev_priv);
3899
3900 I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
3901
3902 I915_WRITE(GEN11_GFX_MSTR_IRQ, GEN11_MASTER_IRQ);
3903 POSTING_READ(GEN11_GFX_MSTR_IRQ);
3904
3905 return 0;
3906}
3907
3680static int cherryview_irq_postinstall(struct drm_device *dev) 3908static int cherryview_irq_postinstall(struct drm_device *dev)
3681{ 3909{
3682 struct drm_i915_private *dev_priv = to_i915(dev); 3910 struct drm_i915_private *dev_priv = to_i915(dev);
@@ -4125,6 +4353,14 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
4125 dev->driver->enable_vblank = i965_enable_vblank; 4353 dev->driver->enable_vblank = i965_enable_vblank;
4126 dev->driver->disable_vblank = i965_disable_vblank; 4354 dev->driver->disable_vblank = i965_disable_vblank;
4127 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup; 4355 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4356 } else if (INTEL_GEN(dev_priv) >= 11) {
4357 dev->driver->irq_handler = gen11_irq_handler;
4358 dev->driver->irq_preinstall = gen11_irq_reset;
4359 dev->driver->irq_postinstall = gen11_irq_postinstall;
4360 dev->driver->irq_uninstall = gen11_irq_reset;
4361 dev->driver->enable_vblank = gen8_enable_vblank;
4362 dev->driver->disable_vblank = gen8_disable_vblank;
4363 dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
4128 } else if (INTEL_GEN(dev_priv) >= 8) { 4364 } else if (INTEL_GEN(dev_priv) >= 8) {
4129 dev->driver->irq_handler = gen8_irq_handler; 4365 dev->driver->irq_handler = gen8_irq_handler;
4130 dev->driver->irq_preinstall = gen8_irq_reset; 4366 dev->driver->irq_preinstall = gen8_irq_reset;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 1eaabf28d7b7..062e91b39085 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -594,7 +594,8 @@ static const struct intel_device_info intel_cannonlake_info = {
594 GEN10_FEATURES, \ 594 GEN10_FEATURES, \
595 GEN(11), \ 595 GEN(11), \
596 .ddb_size = 2048, \ 596 .ddb_size = 2048, \
597 .has_csr = 0 597 .has_csr = 0, \
598 .has_logical_ring_elsq = 1
598 599
599static const struct intel_device_info intel_icelake_11_info = { 600static const struct intel_device_info intel_icelake_11_info = {
600 GEN11_FEATURES, 601 GEN11_FEATURES,
@@ -664,6 +665,7 @@ static const struct pci_device_id pciidlist[] = {
664 INTEL_CFL_U_GT2_IDS(&intel_coffeelake_gt2_info), 665 INTEL_CFL_U_GT2_IDS(&intel_coffeelake_gt2_info),
665 INTEL_CFL_U_GT3_IDS(&intel_coffeelake_gt3_info), 666 INTEL_CFL_U_GT3_IDS(&intel_coffeelake_gt3_info),
666 INTEL_CNL_IDS(&intel_cannonlake_info), 667 INTEL_CNL_IDS(&intel_cannonlake_info),
668 INTEL_ICL_11_IDS(&intel_icelake_11_info),
667 {0, 0, 0} 669 {0, 0, 0}
668}; 670};
669MODULE_DEVICE_TABLE(pci, pciidlist); 671MODULE_DEVICE_TABLE(pci, pciidlist);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 0be50e43507d..abaca6edeb71 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1303,9 +1303,8 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
1303 */ 1303 */
1304 mutex_lock(&dev_priv->drm.struct_mutex); 1304 mutex_lock(&dev_priv->drm.struct_mutex);
1305 dev_priv->perf.oa.exclusive_stream = NULL; 1305 dev_priv->perf.oa.exclusive_stream = NULL;
1306 mutex_unlock(&dev_priv->drm.struct_mutex);
1307
1308 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); 1306 dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
1307 mutex_unlock(&dev_priv->drm.struct_mutex);
1309 1308
1310 free_oa_buffer(dev_priv); 1309 free_oa_buffer(dev_priv);
1311 1310
@@ -1630,10 +1629,10 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
1630 * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This 1629 * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This
1631 * is only used by the kernel context. 1630 * is only used by the kernel context.
1632 */ 1631 */
1633static int gen8_emit_oa_config(struct drm_i915_gem_request *req, 1632static int gen8_emit_oa_config(struct i915_request *rq,
1634 const struct i915_oa_config *oa_config) 1633 const struct i915_oa_config *oa_config)
1635{ 1634{
1636 struct drm_i915_private *dev_priv = req->i915; 1635 struct drm_i915_private *dev_priv = rq->i915;
1637 /* The MMIO offsets for Flex EU registers aren't contiguous */ 1636 /* The MMIO offsets for Flex EU registers aren't contiguous */
1638 u32 flex_mmio[] = { 1637 u32 flex_mmio[] = {
1639 i915_mmio_reg_offset(EU_PERF_CNTL0), 1638 i915_mmio_reg_offset(EU_PERF_CNTL0),
@@ -1647,7 +1646,7 @@ static int gen8_emit_oa_config(struct drm_i915_gem_request *req,
1647 u32 *cs; 1646 u32 *cs;
1648 int i; 1647 int i;
1649 1648
1650 cs = intel_ring_begin(req, ARRAY_SIZE(flex_mmio) * 2 + 4); 1649 cs = intel_ring_begin(rq, ARRAY_SIZE(flex_mmio) * 2 + 4);
1651 if (IS_ERR(cs)) 1650 if (IS_ERR(cs))
1652 return PTR_ERR(cs); 1651 return PTR_ERR(cs);
1653 1652
@@ -1685,7 +1684,7 @@ static int gen8_emit_oa_config(struct drm_i915_gem_request *req,
1685 } 1684 }
1686 1685
1687 *cs++ = MI_NOOP; 1686 *cs++ = MI_NOOP;
1688 intel_ring_advance(req, cs); 1687 intel_ring_advance(rq, cs);
1689 1688
1690 return 0; 1689 return 0;
1691} 1690}
@@ -1695,38 +1694,38 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
1695{ 1694{
1696 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 1695 struct intel_engine_cs *engine = dev_priv->engine[RCS];
1697 struct i915_gem_timeline *timeline; 1696 struct i915_gem_timeline *timeline;
1698 struct drm_i915_gem_request *req; 1697 struct i915_request *rq;
1699 int ret; 1698 int ret;
1700 1699
1701 lockdep_assert_held(&dev_priv->drm.struct_mutex); 1700 lockdep_assert_held(&dev_priv->drm.struct_mutex);
1702 1701
1703 i915_gem_retire_requests(dev_priv); 1702 i915_retire_requests(dev_priv);
1704 1703
1705 req = i915_gem_request_alloc(engine, dev_priv->kernel_context); 1704 rq = i915_request_alloc(engine, dev_priv->kernel_context);
1706 if (IS_ERR(req)) 1705 if (IS_ERR(rq))
1707 return PTR_ERR(req); 1706 return PTR_ERR(rq);
1708 1707
1709 ret = gen8_emit_oa_config(req, oa_config); 1708 ret = gen8_emit_oa_config(rq, oa_config);
1710 if (ret) { 1709 if (ret) {
1711 i915_add_request(req); 1710 i915_request_add(rq);
1712 return ret; 1711 return ret;
1713 } 1712 }
1714 1713
1715 /* Queue this switch after all other activity */ 1714 /* Queue this switch after all other activity */
1716 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { 1715 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
1717 struct drm_i915_gem_request *prev; 1716 struct i915_request *prev;
1718 struct intel_timeline *tl; 1717 struct intel_timeline *tl;
1719 1718
1720 tl = &timeline->engine[engine->id]; 1719 tl = &timeline->engine[engine->id];
1721 prev = i915_gem_active_raw(&tl->last_request, 1720 prev = i915_gem_active_raw(&tl->last_request,
1722 &dev_priv->drm.struct_mutex); 1721 &dev_priv->drm.struct_mutex);
1723 if (prev) 1722 if (prev)
1724 i915_sw_fence_await_sw_fence_gfp(&req->submit, 1723 i915_sw_fence_await_sw_fence_gfp(&rq->submit,
1725 &prev->submit, 1724 &prev->submit,
1726 GFP_KERNEL); 1725 GFP_KERNEL);
1727 } 1726 }
1728 1727
1729 i915_add_request(req); 1728 i915_request_add(rq);
1730 1729
1731 return 0; 1730 return 0;
1732} 1731}
@@ -1756,22 +1755,13 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
1756 * Note: it's only the RCS/Render context that has any OA state. 1755 * Note: it's only the RCS/Render context that has any OA state.
1757 */ 1756 */
1758static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv, 1757static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
1759 const struct i915_oa_config *oa_config, 1758 const struct i915_oa_config *oa_config)
1760 bool interruptible)
1761{ 1759{
1762 struct i915_gem_context *ctx; 1760 struct i915_gem_context *ctx;
1763 int ret; 1761 int ret;
1764 unsigned int wait_flags = I915_WAIT_LOCKED; 1762 unsigned int wait_flags = I915_WAIT_LOCKED;
1765 1763
1766 if (interruptible) { 1764 lockdep_assert_held(&dev_priv->drm.struct_mutex);
1767 ret = i915_mutex_lock_interruptible(&dev_priv->drm);
1768 if (ret)
1769 return ret;
1770
1771 wait_flags |= I915_WAIT_INTERRUPTIBLE;
1772 } else {
1773 mutex_lock(&dev_priv->drm.struct_mutex);
1774 }
1775 1765
1776 /* Switch away from any user context. */ 1766 /* Switch away from any user context. */
1777 ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config); 1767 ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config);
@@ -1819,8 +1809,6 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
1819 } 1809 }
1820 1810
1821 out: 1811 out:
1822 mutex_unlock(&dev_priv->drm.struct_mutex);
1823
1824 return ret; 1812 return ret;
1825} 1813}
1826 1814
@@ -1863,7 +1851,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
1863 * to make sure all slices/subslices are ON before writing to NOA 1851 * to make sure all slices/subslices are ON before writing to NOA
1864 * registers. 1852 * registers.
1865 */ 1853 */
1866 ret = gen8_configure_all_contexts(dev_priv, oa_config, true); 1854 ret = gen8_configure_all_contexts(dev_priv, oa_config);
1867 if (ret) 1855 if (ret)
1868 return ret; 1856 return ret;
1869 1857
@@ -1878,7 +1866,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
1878static void gen8_disable_metric_set(struct drm_i915_private *dev_priv) 1866static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
1879{ 1867{
1880 /* Reset all contexts' slices/subslices configurations. */ 1868 /* Reset all contexts' slices/subslices configurations. */
1881 gen8_configure_all_contexts(dev_priv, NULL, false); 1869 gen8_configure_all_contexts(dev_priv, NULL);
1882 1870
1883 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & 1871 I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
1884 ~GT_NOA_ENABLE)); 1872 ~GT_NOA_ENABLE));
@@ -1888,7 +1876,7 @@ static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
1888static void gen10_disable_metric_set(struct drm_i915_private *dev_priv) 1876static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
1889{ 1877{
1890 /* Reset all contexts' slices/subslices configurations. */ 1878 /* Reset all contexts' slices/subslices configurations. */
1891 gen8_configure_all_contexts(dev_priv, NULL, false); 1879 gen8_configure_all_contexts(dev_priv, NULL);
1892 1880
1893 /* Make sure we disable noa to save power. */ 1881 /* Make sure we disable noa to save power. */
1894 I915_WRITE(RPM_CONFIG1, 1882 I915_WRITE(RPM_CONFIG1,
@@ -2138,6 +2126,10 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
2138 if (ret) 2126 if (ret)
2139 goto err_oa_buf_alloc; 2127 goto err_oa_buf_alloc;
2140 2128
2129 ret = i915_mutex_lock_interruptible(&dev_priv->drm);
2130 if (ret)
2131 goto err_lock;
2132
2141 ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv, 2133 ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv,
2142 stream->oa_config); 2134 stream->oa_config);
2143 if (ret) 2135 if (ret)
@@ -2145,23 +2137,17 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
2145 2137
2146 stream->ops = &i915_oa_stream_ops; 2138 stream->ops = &i915_oa_stream_ops;
2147 2139
2148 /* Lock device for exclusive_stream access late because
2149 * enable_metric_set() might lock as well on gen8+.
2150 */
2151 ret = i915_mutex_lock_interruptible(&dev_priv->drm);
2152 if (ret)
2153 goto err_lock;
2154
2155 dev_priv->perf.oa.exclusive_stream = stream; 2140 dev_priv->perf.oa.exclusive_stream = stream;
2156 2141
2157 mutex_unlock(&dev_priv->drm.struct_mutex); 2142 mutex_unlock(&dev_priv->drm.struct_mutex);
2158 2143
2159 return 0; 2144 return 0;
2160 2145
2161err_lock: 2146err_enable:
2162 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); 2147 dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
2148 mutex_unlock(&dev_priv->drm.struct_mutex);
2163 2149
2164err_enable: 2150err_lock:
2165 free_oa_buffer(dev_priv); 2151 free_oa_buffer(dev_priv);
2166 2152
2167err_oa_buf_alloc: 2153err_oa_buf_alloc:
diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c
new file mode 100644
index 000000000000..3ace929dd90f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_query.c
@@ -0,0 +1,125 @@
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7#include "i915_drv.h"
8#include "i915_query.h"
9#include <uapi/drm/i915_drm.h>
10
11static int query_topology_info(struct drm_i915_private *dev_priv,
12 struct drm_i915_query_item *query_item)
13{
14 const struct sseu_dev_info *sseu = &INTEL_INFO(dev_priv)->sseu;
15 struct drm_i915_query_topology_info topo;
16 u32 slice_length, subslice_length, eu_length, total_length;
17
18 if (query_item->flags != 0)
19 return -EINVAL;
20
21 if (sseu->max_slices == 0)
22 return -ENODEV;
23
24 BUILD_BUG_ON(sizeof(u8) != sizeof(sseu->slice_mask));
25
26 slice_length = sizeof(sseu->slice_mask);
27 subslice_length = sseu->max_slices *
28 DIV_ROUND_UP(sseu->max_subslices,
29 sizeof(sseu->subslice_mask[0]) * BITS_PER_BYTE);
30 eu_length = sseu->max_slices * sseu->max_subslices *
31 DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE);
32
33 total_length = sizeof(topo) + slice_length + subslice_length + eu_length;
34
35 if (query_item->length == 0)
36 return total_length;
37
38 if (query_item->length < total_length)
39 return -EINVAL;
40
41 if (copy_from_user(&topo, u64_to_user_ptr(query_item->data_ptr),
42 sizeof(topo)))
43 return -EFAULT;
44
45 if (topo.flags != 0)
46 return -EINVAL;
47
48 if (!access_ok(VERIFY_WRITE, u64_to_user_ptr(query_item->data_ptr),
49 total_length))
50 return -EFAULT;
51
52 memset(&topo, 0, sizeof(topo));
53 topo.max_slices = sseu->max_slices;
54 topo.max_subslices = sseu->max_subslices;
55 topo.max_eus_per_subslice = sseu->max_eus_per_subslice;
56
57 topo.subslice_offset = slice_length;
58 topo.subslice_stride = DIV_ROUND_UP(sseu->max_subslices, BITS_PER_BYTE);
59 topo.eu_offset = slice_length + subslice_length;
60 topo.eu_stride =
61 DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE);
62
63 if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr),
64 &topo, sizeof(topo)))
65 return -EFAULT;
66
67 if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)),
68 &sseu->slice_mask, slice_length))
69 return -EFAULT;
70
71 if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr +
72 sizeof(topo) + slice_length),
73 sseu->subslice_mask, subslice_length))
74 return -EFAULT;
75
76 if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr +
77 sizeof(topo) +
78 slice_length + subslice_length),
79 sseu->eu_mask, eu_length))
80 return -EFAULT;
81
82 return total_length;
83}
84
85static int (* const i915_query_funcs[])(struct drm_i915_private *dev_priv,
86 struct drm_i915_query_item *query_item) = {
87 query_topology_info,
88};
89
90int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
91{
92 struct drm_i915_private *dev_priv = to_i915(dev);
93 struct drm_i915_query *args = data;
94 struct drm_i915_query_item __user *user_item_ptr =
95 u64_to_user_ptr(args->items_ptr);
96 u32 i;
97
98 if (args->flags != 0)
99 return -EINVAL;
100
101 for (i = 0; i < args->num_items; i++, user_item_ptr++) {
102 struct drm_i915_query_item item;
103 u64 func_idx;
104 int ret;
105
106 if (copy_from_user(&item, user_item_ptr, sizeof(item)))
107 return -EFAULT;
108
109 if (item.query_id == 0)
110 return -EINVAL;
111
112 func_idx = item.query_id - 1;
113
114 if (func_idx < ARRAY_SIZE(i915_query_funcs))
115 ret = i915_query_funcs[func_idx](dev_priv, &item);
116 else
117 ret = -EINVAL;
118
119 /* Only write the length back to userspace if they differ. */
120 if (ret != item.length && put_user(ret, &user_item_ptr->length))
121 return -EFAULT;
122 }
123
124 return 0;
125}
diff --git a/drivers/gpu/drm/i915/i915_query.h b/drivers/gpu/drm/i915/i915_query.h
new file mode 100644
index 000000000000..31dcef181f63
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_query.h
@@ -0,0 +1,15 @@
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7#ifndef _I915_QUERY_H_
8#define _I915_QUERY_H_
9
10struct drm_device;
11struct drm_file;
12
13int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
14
15#endif
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 47ca8b97ce50..e6a8c0ee7df1 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -178,6 +178,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
178#define BCS_HW 2 178#define BCS_HW 2
179#define VECS_HW 3 179#define VECS_HW 3
180#define VCS2_HW 4 180#define VCS2_HW 4
181#define VCS3_HW 6
182#define VCS4_HW 7
183#define VECS2_HW 12
181 184
182/* Engine class */ 185/* Engine class */
183 186
@@ -188,7 +191,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
188#define OTHER_CLASS 4 191#define OTHER_CLASS 4
189#define MAX_ENGINE_CLASS 4 192#define MAX_ENGINE_CLASS 4
190 193
191#define MAX_ENGINE_INSTANCE 1 194#define MAX_ENGINE_INSTANCE 3
192 195
193/* PCI config space */ 196/* PCI config space */
194 197
@@ -2342,7 +2345,13 @@ enum i915_power_well_id {
2342#define BSD_RING_BASE 0x04000 2345#define BSD_RING_BASE 0x04000
2343#define GEN6_BSD_RING_BASE 0x12000 2346#define GEN6_BSD_RING_BASE 0x12000
2344#define GEN8_BSD2_RING_BASE 0x1c000 2347#define GEN8_BSD2_RING_BASE 0x1c000
2348#define GEN11_BSD_RING_BASE 0x1c0000
2349#define GEN11_BSD2_RING_BASE 0x1c4000
2350#define GEN11_BSD3_RING_BASE 0x1d0000
2351#define GEN11_BSD4_RING_BASE 0x1d4000
2345#define VEBOX_RING_BASE 0x1a000 2352#define VEBOX_RING_BASE 0x1a000
2353#define GEN11_VEBOX_RING_BASE 0x1c8000
2354#define GEN11_VEBOX2_RING_BASE 0x1d8000
2346#define BLT_RING_BASE 0x22000 2355#define BLT_RING_BASE 0x22000
2347#define RING_TAIL(base) _MMIO((base)+0x30) 2356#define RING_TAIL(base) _MMIO((base)+0x30)
2348#define RING_HEAD(base) _MMIO((base)+0x34) 2357#define RING_HEAD(base) _MMIO((base)+0x34)
@@ -2807,6 +2816,13 @@ enum i915_power_well_id {
2807#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4) 2816#define GEN9_RCS_FE_FSM2 _MMIO(0x22a4)
2808 2817
2809/* Fuse readout registers for GT */ 2818/* Fuse readout registers for GT */
2819#define HSW_PAVP_FUSE1 _MMIO(0x911C)
2820#define HSW_F1_EU_DIS_SHIFT 16
2821#define HSW_F1_EU_DIS_MASK (0x3 << HSW_F1_EU_DIS_SHIFT)
2822#define HSW_F1_EU_DIS_10EUS 0
2823#define HSW_F1_EU_DIS_8EUS 1
2824#define HSW_F1_EU_DIS_6EUS 2
2825
2810#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168) 2826#define CHV_FUSE_GT _MMIO(VLV_DISPLAY_BASE + 0x2168)
2811#define CHV_FGT_DISABLE_SS0 (1 << 10) 2827#define CHV_FGT_DISABLE_SS0 (1 << 10)
2812#define CHV_FGT_DISABLE_SS1 (1 << 11) 2828#define CHV_FGT_DISABLE_SS1 (1 << 11)
@@ -3896,6 +3912,12 @@ enum {
3896 3912
3897#define GEN8_CTX_ID_SHIFT 32 3913#define GEN8_CTX_ID_SHIFT 32
3898#define GEN8_CTX_ID_WIDTH 21 3914#define GEN8_CTX_ID_WIDTH 21
3915#define GEN11_SW_CTX_ID_SHIFT 37
3916#define GEN11_SW_CTX_ID_WIDTH 11
3917#define GEN11_ENGINE_CLASS_SHIFT 61
3918#define GEN11_ENGINE_CLASS_WIDTH 3
3919#define GEN11_ENGINE_INSTANCE_SHIFT 48
3920#define GEN11_ENGINE_INSTANCE_WIDTH 6
3899 3921
3900#define CHV_CLK_CTL1 _MMIO(0x101100) 3922#define CHV_CLK_CTL1 _MMIO(0x101100)
3901#define VLV_CLK_CTL2 _MMIO(0x101104) 3923#define VLV_CLK_CTL2 _MMIO(0x101104)
@@ -3943,6 +3965,9 @@ enum {
3943#define SARBUNIT_CLKGATE_DIS (1 << 5) 3965#define SARBUNIT_CLKGATE_DIS (1 << 5)
3944#define RCCUNIT_CLKGATE_DIS (1 << 7) 3966#define RCCUNIT_CLKGATE_DIS (1 << 7)
3945 3967
3968#define SUBSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9524)
3969#define GWUNIT_CLKGATE_DIS (1 << 16)
3970
3946#define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434) 3971#define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434)
3947#define VFUNIT_CLKGATE_DIS (1 << 20) 3972#define VFUNIT_CLKGATE_DIS (1 << 20)
3948 3973
@@ -5347,8 +5372,8 @@ enum {
5347#define _DPF_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64520) 5372#define _DPF_AUX_CH_DATA4 (dev_priv->info.display_mmio_offset + 0x64520)
5348#define _DPF_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64524) 5373#define _DPF_AUX_CH_DATA5 (dev_priv->info.display_mmio_offset + 0x64524)
5349 5374
5350#define DP_AUX_CH_CTL(port) _MMIO_PORT(port, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL) 5375#define DP_AUX_CH_CTL(aux_ch) _MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL)
5351#define DP_AUX_CH_DATA(port, i) _MMIO(_PORT(port, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ 5376#define DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
5352 5377
5353#define DP_AUX_CH_CTL_SEND_BUSY (1 << 31) 5378#define DP_AUX_CH_CTL_SEND_BUSY (1 << 31)
5354#define DP_AUX_CH_CTL_DONE (1 << 30) 5379#define DP_AUX_CH_CTL_DONE (1 << 30)
@@ -7897,8 +7922,8 @@ enum {
7897#define _PCH_DPD_AUX_CH_DATA4 0xe4320 7922#define _PCH_DPD_AUX_CH_DATA4 0xe4320
7898#define _PCH_DPD_AUX_CH_DATA5 0xe4324 7923#define _PCH_DPD_AUX_CH_DATA5 0xe4324
7899 7924
7900#define PCH_DP_AUX_CH_CTL(port) _MMIO_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_CTL, _PCH_DPC_AUX_CH_CTL) 7925#define PCH_DP_AUX_CH_CTL(aux_ch) _MMIO_PORT((aux_ch) - AUX_CH_B, _PCH_DPB_AUX_CH_CTL, _PCH_DPC_AUX_CH_CTL)
7901#define PCH_DP_AUX_CH_DATA(port, i) _MMIO(_PORT((port) - PORT_B, _PCH_DPB_AUX_CH_DATA1, _PCH_DPC_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ 7926#define PCH_DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT((aux_ch) - AUX_CH_B, _PCH_DPB_AUX_CH_DATA1, _PCH_DPC_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
7902 7927
7903/* CPT */ 7928/* CPT */
7904#define PORT_TRANS_A_SEL_CPT 0 7929#define PORT_TRANS_A_SEL_CPT 0
@@ -7998,9 +8023,13 @@ enum {
7998#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7) 8023#define VLV_GTLC_PW_RENDER_STATUS_MASK (1 << 7)
7999#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */ 8024#define FORCEWAKE_MT _MMIO(0xa188) /* multi-threaded */
8000#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270) 8025#define FORCEWAKE_MEDIA_GEN9 _MMIO(0xa270)
8026#define FORCEWAKE_MEDIA_VDBOX_GEN11(n) _MMIO(0xa540 + (n) * 4)
8027#define FORCEWAKE_MEDIA_VEBOX_GEN11(n) _MMIO(0xa560 + (n) * 4)
8001#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278) 8028#define FORCEWAKE_RENDER_GEN9 _MMIO(0xa278)
8002#define FORCEWAKE_BLITTER_GEN9 _MMIO(0xa188) 8029#define FORCEWAKE_BLITTER_GEN9 _MMIO(0xa188)
8003#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0x0D88) 8030#define FORCEWAKE_ACK_MEDIA_GEN9 _MMIO(0x0D88)
8031#define FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(n) _MMIO(0x0D50 + (n) * 4)
8032#define FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(n) _MMIO(0x0D70 + (n) * 4)
8004#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0x0D84) 8033#define FORCEWAKE_ACK_RENDER_GEN9 _MMIO(0x0D84)
8005#define FORCEWAKE_ACK_BLITTER_GEN9 _MMIO(0x130044) 8034#define FORCEWAKE_ACK_BLITTER_GEN9 _MMIO(0x130044)
8006#define FORCEWAKE_KERNEL BIT(0) 8035#define FORCEWAKE_KERNEL BIT(0)
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_request.c
index 8bc7c50b8418..d437beac3969 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -37,7 +37,8 @@ static const char *i915_fence_get_driver_name(struct dma_fence *fence)
37 37
38static const char *i915_fence_get_timeline_name(struct dma_fence *fence) 38static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
39{ 39{
40 /* The timeline struct (as part of the ppgtt underneath a context) 40 /*
41 * The timeline struct (as part of the ppgtt underneath a context)
41 * may be freed when the request is no longer in use by the GPU. 42 * may be freed when the request is no longer in use by the GPU.
42 * We could extend the life of a context to beyond that of all 43 * We could extend the life of a context to beyond that of all
43 * fences, possibly keeping the hw resource around indefinitely, 44 * fences, possibly keeping the hw resource around indefinitely,
@@ -53,7 +54,7 @@ static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
53 54
54static bool i915_fence_signaled(struct dma_fence *fence) 55static bool i915_fence_signaled(struct dma_fence *fence)
55{ 56{
56 return i915_gem_request_completed(to_request(fence)); 57 return i915_request_completed(to_request(fence));
57} 58}
58 59
59static bool i915_fence_enable_signaling(struct dma_fence *fence) 60static bool i915_fence_enable_signaling(struct dma_fence *fence)
@@ -69,22 +70,23 @@ static signed long i915_fence_wait(struct dma_fence *fence,
69 bool interruptible, 70 bool interruptible,
70 signed long timeout) 71 signed long timeout)
71{ 72{
72 return i915_wait_request(to_request(fence), interruptible, timeout); 73 return i915_request_wait(to_request(fence), interruptible, timeout);
73} 74}
74 75
75static void i915_fence_release(struct dma_fence *fence) 76static void i915_fence_release(struct dma_fence *fence)
76{ 77{
77 struct drm_i915_gem_request *req = to_request(fence); 78 struct i915_request *rq = to_request(fence);
78 79
79 /* The request is put onto a RCU freelist (i.e. the address 80 /*
81 * The request is put onto a RCU freelist (i.e. the address
80 * is immediately reused), mark the fences as being freed now. 82 * is immediately reused), mark the fences as being freed now.
81 * Otherwise the debugobjects for the fences are only marked as 83 * Otherwise the debugobjects for the fences are only marked as
82 * freed when the slab cache itself is freed, and so we would get 84 * freed when the slab cache itself is freed, and so we would get
83 * caught trying to reuse dead objects. 85 * caught trying to reuse dead objects.
84 */ 86 */
85 i915_sw_fence_fini(&req->submit); 87 i915_sw_fence_fini(&rq->submit);
86 88
87 kmem_cache_free(req->i915->requests, req); 89 kmem_cache_free(rq->i915->requests, rq);
88} 90}
89 91
90const struct dma_fence_ops i915_fence_ops = { 92const struct dma_fence_ops i915_fence_ops = {
@@ -97,7 +99,7 @@ const struct dma_fence_ops i915_fence_ops = {
97}; 99};
98 100
99static inline void 101static inline void
100i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 102i915_request_remove_from_client(struct i915_request *request)
101{ 103{
102 struct drm_i915_file_private *file_priv; 104 struct drm_i915_file_private *file_priv;
103 105
@@ -215,9 +217,9 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
215 struct intel_timeline *tl = engine->timeline; 217 struct intel_timeline *tl = engine->timeline;
216 218
217 if (!i915_seqno_passed(seqno, tl->seqno)) { 219 if (!i915_seqno_passed(seqno, tl->seqno)) {
218 /* spin until threads are complete */ 220 /* Flush any waiters before we reuse the seqno */
219 while (intel_breadcrumbs_busy(engine)) 221 intel_engine_disarm_breadcrumbs(engine);
220 cond_resched(); 222 GEM_BUG_ON(!list_empty(&engine->breadcrumbs.signals));
221 } 223 }
222 224
223 /* Check we are idle before we fiddle with hw state! */ 225 /* Check we are idle before we fiddle with hw state! */
@@ -238,17 +240,15 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
238 240
239int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) 241int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
240{ 242{
241 struct drm_i915_private *dev_priv = to_i915(dev); 243 struct drm_i915_private *i915 = to_i915(dev);
242 244
243 lockdep_assert_held(&dev_priv->drm.struct_mutex); 245 lockdep_assert_held(&i915->drm.struct_mutex);
244 246
245 if (seqno == 0) 247 if (seqno == 0)
246 return -EINVAL; 248 return -EINVAL;
247 249
248 /* HWS page needs to be set less than what we 250 /* HWS page needs to be set less than what we will inject to ring */
249 * will inject to ring 251 return reset_all_global_seqno(i915, seqno - 1);
250 */
251 return reset_all_global_seqno(dev_priv, seqno - 1);
252} 252}
253 253
254static void mark_busy(struct drm_i915_private *i915) 254static void mark_busy(struct drm_i915_private *i915)
@@ -331,16 +331,17 @@ static void unreserve_engine(struct intel_engine_cs *engine)
331} 331}
332 332
333void i915_gem_retire_noop(struct i915_gem_active *active, 333void i915_gem_retire_noop(struct i915_gem_active *active,
334 struct drm_i915_gem_request *request) 334 struct i915_request *request)
335{ 335{
336 /* Space left intentionally blank */ 336 /* Space left intentionally blank */
337} 337}
338 338
339static void advance_ring(struct drm_i915_gem_request *request) 339static void advance_ring(struct i915_request *request)
340{ 340{
341 unsigned int tail; 341 unsigned int tail;
342 342
343 /* We know the GPU must have read the request to have 343 /*
344 * We know the GPU must have read the request to have
344 * sent us the seqno + interrupt, so use the position 345 * sent us the seqno + interrupt, so use the position
345 * of tail of the request to update the last known position 346 * of tail of the request to update the last known position
346 * of the GPU head. 347 * of the GPU head.
@@ -349,7 +350,8 @@ static void advance_ring(struct drm_i915_gem_request *request)
349 * completion order. 350 * completion order.
350 */ 351 */
351 if (list_is_last(&request->ring_link, &request->ring->request_list)) { 352 if (list_is_last(&request->ring_link, &request->ring->request_list)) {
352 /* We may race here with execlists resubmitting this request 353 /*
354 * We may race here with execlists resubmitting this request
353 * as we retire it. The resubmission will move the ring->tail 355 * as we retire it. The resubmission will move the ring->tail
354 * forwards (to request->wa_tail). We either read the 356 * forwards (to request->wa_tail). We either read the
355 * current value that was written to hw, or the value that 357 * current value that was written to hw, or the value that
@@ -365,30 +367,30 @@ static void advance_ring(struct drm_i915_gem_request *request)
365 request->ring->head = tail; 367 request->ring->head = tail;
366} 368}
367 369
368static void free_capture_list(struct drm_i915_gem_request *request) 370static void free_capture_list(struct i915_request *request)
369{ 371{
370 struct i915_gem_capture_list *capture; 372 struct i915_capture_list *capture;
371 373
372 capture = request->capture_list; 374 capture = request->capture_list;
373 while (capture) { 375 while (capture) {
374 struct i915_gem_capture_list *next = capture->next; 376 struct i915_capture_list *next = capture->next;
375 377
376 kfree(capture); 378 kfree(capture);
377 capture = next; 379 capture = next;
378 } 380 }
379} 381}
380 382
381static void i915_gem_request_retire(struct drm_i915_gem_request *request) 383static void i915_request_retire(struct i915_request *request)
382{ 384{
383 struct intel_engine_cs *engine = request->engine; 385 struct intel_engine_cs *engine = request->engine;
384 struct i915_gem_active *active, *next; 386 struct i915_gem_active *active, *next;
385 387
386 lockdep_assert_held(&request->i915->drm.struct_mutex); 388 lockdep_assert_held(&request->i915->drm.struct_mutex);
387 GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit)); 389 GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
388 GEM_BUG_ON(!i915_gem_request_completed(request)); 390 GEM_BUG_ON(!i915_request_completed(request));
389 GEM_BUG_ON(!request->i915->gt.active_requests); 391 GEM_BUG_ON(!request->i915->gt.active_requests);
390 392
391 trace_i915_gem_request_retire(request); 393 trace_i915_request_retire(request);
392 394
393 spin_lock_irq(&engine->timeline->lock); 395 spin_lock_irq(&engine->timeline->lock);
394 list_del_init(&request->link); 396 list_del_init(&request->link);
@@ -399,7 +401,8 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
399 401
400 free_capture_list(request); 402 free_capture_list(request);
401 403
402 /* Walk through the active list, calling retire on each. This allows 404 /*
405 * Walk through the active list, calling retire on each. This allows
403 * objects to track their GPU activity and mark themselves as idle 406 * objects to track their GPU activity and mark themselves as idle
404 * when their *last* active request is completed (updating state 407 * when their *last* active request is completed (updating state
405 * tracking lists for eviction, active references for GEM, etc). 408 * tracking lists for eviction, active references for GEM, etc).
@@ -409,7 +412,8 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
409 * the node after the callback). 412 * the node after the callback).
410 */ 413 */
411 list_for_each_entry_safe(active, next, &request->active_list, link) { 414 list_for_each_entry_safe(active, next, &request->active_list, link) {
412 /* In microbenchmarks or focusing upon time inside the kernel, 415 /*
416 * In microbenchmarks or focusing upon time inside the kernel,
413 * we may spend an inordinate amount of time simply handling 417 * we may spend an inordinate amount of time simply handling
414 * the retirement of requests and processing their callbacks. 418 * the retirement of requests and processing their callbacks.
415 * Of which, this loop itself is particularly hot due to the 419 * Of which, this loop itself is particularly hot due to the
@@ -426,15 +430,16 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
426 active->retire(active, request); 430 active->retire(active, request);
427 } 431 }
428 432
429 i915_gem_request_remove_from_client(request); 433 i915_request_remove_from_client(request);
430 434
431 /* Retirement decays the ban score as it is a sign of ctx progress */ 435 /* Retirement decays the ban score as it is a sign of ctx progress */
432 atomic_dec_if_positive(&request->ctx->ban_score); 436 atomic_dec_if_positive(&request->ctx->ban_score);
433 437
434 /* The backing object for the context is done after switching to the 438 /*
439 * The backing object for the context is done after switching to the
435 * *next* context. Therefore we cannot retire the previous context until 440 * *next* context. Therefore we cannot retire the previous context until
436 * the next context has already started running. However, since we 441 * the next context has already started running. However, since we
437 * cannot take the required locks at i915_gem_request_submit() we 442 * cannot take the required locks at i915_request_submit() we
438 * defer the unpinning of the active context to now, retirement of 443 * defer the unpinning of the active context to now, retirement of
439 * the subsequent request. 444 * the subsequent request.
440 */ 445 */
@@ -454,26 +459,26 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
454 spin_unlock_irq(&request->lock); 459 spin_unlock_irq(&request->lock);
455 460
456 i915_priotree_fini(request->i915, &request->priotree); 461 i915_priotree_fini(request->i915, &request->priotree);
457 i915_gem_request_put(request); 462 i915_request_put(request);
458} 463}
459 464
460void i915_gem_request_retire_upto(struct drm_i915_gem_request *req) 465void i915_request_retire_upto(struct i915_request *rq)
461{ 466{
462 struct intel_engine_cs *engine = req->engine; 467 struct intel_engine_cs *engine = rq->engine;
463 struct drm_i915_gem_request *tmp; 468 struct i915_request *tmp;
464 469
465 lockdep_assert_held(&req->i915->drm.struct_mutex); 470 lockdep_assert_held(&rq->i915->drm.struct_mutex);
466 GEM_BUG_ON(!i915_gem_request_completed(req)); 471 GEM_BUG_ON(!i915_request_completed(rq));
467 472
468 if (list_empty(&req->link)) 473 if (list_empty(&rq->link))
469 return; 474 return;
470 475
471 do { 476 do {
472 tmp = list_first_entry(&engine->timeline->requests, 477 tmp = list_first_entry(&engine->timeline->requests,
473 typeof(*tmp), link); 478 typeof(*tmp), link);
474 479
475 i915_gem_request_retire(tmp); 480 i915_request_retire(tmp);
476 } while (tmp != req); 481 } while (tmp != rq);
477} 482}
478 483
479static u32 timeline_get_seqno(struct intel_timeline *tl) 484static u32 timeline_get_seqno(struct intel_timeline *tl)
@@ -481,7 +486,7 @@ static u32 timeline_get_seqno(struct intel_timeline *tl)
481 return ++tl->seqno; 486 return ++tl->seqno;
482} 487}
483 488
484void __i915_gem_request_submit(struct drm_i915_gem_request *request) 489void __i915_request_submit(struct i915_request *request)
485{ 490{
486 struct intel_engine_cs *engine = request->engine; 491 struct intel_engine_cs *engine = request->engine;
487 struct intel_timeline *timeline; 492 struct intel_timeline *timeline;
@@ -490,8 +495,6 @@ void __i915_gem_request_submit(struct drm_i915_gem_request *request)
490 GEM_BUG_ON(!irqs_disabled()); 495 GEM_BUG_ON(!irqs_disabled());
491 lockdep_assert_held(&engine->timeline->lock); 496 lockdep_assert_held(&engine->timeline->lock);
492 497
493 trace_i915_gem_request_execute(request);
494
495 /* Transfer from per-context onto the global per-engine timeline */ 498 /* Transfer from per-context onto the global per-engine timeline */
496 timeline = engine->timeline; 499 timeline = engine->timeline;
497 GEM_BUG_ON(timeline == request->timeline); 500 GEM_BUG_ON(timeline == request->timeline);
@@ -515,10 +518,12 @@ void __i915_gem_request_submit(struct drm_i915_gem_request *request)
515 list_move_tail(&request->link, &timeline->requests); 518 list_move_tail(&request->link, &timeline->requests);
516 spin_unlock(&request->timeline->lock); 519 spin_unlock(&request->timeline->lock);
517 520
521 trace_i915_request_execute(request);
522
518 wake_up_all(&request->execute); 523 wake_up_all(&request->execute);
519} 524}
520 525
521void i915_gem_request_submit(struct drm_i915_gem_request *request) 526void i915_request_submit(struct i915_request *request)
522{ 527{
523 struct intel_engine_cs *engine = request->engine; 528 struct intel_engine_cs *engine = request->engine;
524 unsigned long flags; 529 unsigned long flags;
@@ -526,12 +531,12 @@ void i915_gem_request_submit(struct drm_i915_gem_request *request)
526 /* Will be called from irq-context when using foreign fences. */ 531 /* Will be called from irq-context when using foreign fences. */
527 spin_lock_irqsave(&engine->timeline->lock, flags); 532 spin_lock_irqsave(&engine->timeline->lock, flags);
528 533
529 __i915_gem_request_submit(request); 534 __i915_request_submit(request);
530 535
531 spin_unlock_irqrestore(&engine->timeline->lock, flags); 536 spin_unlock_irqrestore(&engine->timeline->lock, flags);
532} 537}
533 538
534void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request) 539void __i915_request_unsubmit(struct i915_request *request)
535{ 540{
536 struct intel_engine_cs *engine = request->engine; 541 struct intel_engine_cs *engine = request->engine;
537 struct intel_timeline *timeline; 542 struct intel_timeline *timeline;
@@ -539,7 +544,8 @@ void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request)
539 GEM_BUG_ON(!irqs_disabled()); 544 GEM_BUG_ON(!irqs_disabled());
540 lockdep_assert_held(&engine->timeline->lock); 545 lockdep_assert_held(&engine->timeline->lock);
541 546
542 /* Only unwind in reverse order, required so that the per-context list 547 /*
548 * Only unwind in reverse order, required so that the per-context list
543 * is kept in seqno/ring order. 549 * is kept in seqno/ring order.
544 */ 550 */
545 GEM_BUG_ON(!request->global_seqno); 551 GEM_BUG_ON(!request->global_seqno);
@@ -563,15 +569,16 @@ void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request)
563 list_move(&request->link, &timeline->requests); 569 list_move(&request->link, &timeline->requests);
564 spin_unlock(&timeline->lock); 570 spin_unlock(&timeline->lock);
565 571
566 /* We don't need to wake_up any waiters on request->execute, they 572 /*
573 * We don't need to wake_up any waiters on request->execute, they
567 * will get woken by any other event or us re-adding this request 574 * will get woken by any other event or us re-adding this request
568 * to the engine timeline (__i915_gem_request_submit()). The waiters 575 * to the engine timeline (__i915_request_submit()). The waiters
569 * should be quite adapt at finding that the request now has a new 576 * should be quite adapt at finding that the request now has a new
570 * global_seqno to the one they went to sleep on. 577 * global_seqno to the one they went to sleep on.
571 */ 578 */
572} 579}
573 580
574void i915_gem_request_unsubmit(struct drm_i915_gem_request *request) 581void i915_request_unsubmit(struct i915_request *request)
575{ 582{
576 struct intel_engine_cs *engine = request->engine; 583 struct intel_engine_cs *engine = request->engine;
577 unsigned long flags; 584 unsigned long flags;
@@ -579,7 +586,7 @@ void i915_gem_request_unsubmit(struct drm_i915_gem_request *request)
579 /* Will be called from irq-context when using foreign fences. */ 586 /* Will be called from irq-context when using foreign fences. */
580 spin_lock_irqsave(&engine->timeline->lock, flags); 587 spin_lock_irqsave(&engine->timeline->lock, flags);
581 588
582 __i915_gem_request_unsubmit(request); 589 __i915_request_unsubmit(request);
583 590
584 spin_unlock_irqrestore(&engine->timeline->lock, flags); 591 spin_unlock_irqrestore(&engine->timeline->lock, flags);
585} 592}
@@ -587,18 +594,19 @@ void i915_gem_request_unsubmit(struct drm_i915_gem_request *request)
587static int __i915_sw_fence_call 594static int __i915_sw_fence_call
588submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 595submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
589{ 596{
590 struct drm_i915_gem_request *request = 597 struct i915_request *request =
591 container_of(fence, typeof(*request), submit); 598 container_of(fence, typeof(*request), submit);
592 599
593 switch (state) { 600 switch (state) {
594 case FENCE_COMPLETE: 601 case FENCE_COMPLETE:
595 trace_i915_gem_request_submit(request); 602 trace_i915_request_submit(request);
596 /* 603 /*
597 * We need to serialize use of the submit_request() callback with its 604 * We need to serialize use of the submit_request() callback
598 * hotplugging performed during an emergency i915_gem_set_wedged(). 605 * with its hotplugging performed during an emergency
599 * We use the RCU mechanism to mark the critical section in order to 606 * i915_gem_set_wedged(). We use the RCU mechanism to mark the
600 * force i915_gem_set_wedged() to wait until the submit_request() is 607 * critical section in order to force i915_gem_set_wedged() to
601 * completed before proceeding. 608 * wait until the submit_request() is completed before
609 * proceeding.
602 */ 610 */
603 rcu_read_lock(); 611 rcu_read_lock();
604 request->engine->submit_request(request); 612 request->engine->submit_request(request);
@@ -606,7 +614,7 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
606 break; 614 break;
607 615
608 case FENCE_FREE: 616 case FENCE_FREE:
609 i915_gem_request_put(request); 617 i915_request_put(request);
610 break; 618 break;
611 } 619 }
612 620
@@ -614,7 +622,7 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
614} 622}
615 623
616/** 624/**
617 * i915_gem_request_alloc - allocate a request structure 625 * i915_request_alloc - allocate a request structure
618 * 626 *
619 * @engine: engine that we wish to issue the request on. 627 * @engine: engine that we wish to issue the request on.
620 * @ctx: context that the request will be associated with. 628 * @ctx: context that the request will be associated with.
@@ -622,31 +630,32 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
622 * Returns a pointer to the allocated request if successful, 630 * Returns a pointer to the allocated request if successful,
623 * or an error code if not. 631 * or an error code if not.
624 */ 632 */
625struct drm_i915_gem_request * 633struct i915_request *
626i915_gem_request_alloc(struct intel_engine_cs *engine, 634i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
627 struct i915_gem_context *ctx)
628{ 635{
629 struct drm_i915_private *dev_priv = engine->i915; 636 struct drm_i915_private *i915 = engine->i915;
630 struct drm_i915_gem_request *req; 637 struct i915_request *rq;
631 struct intel_ring *ring; 638 struct intel_ring *ring;
632 int ret; 639 int ret;
633 640
634 lockdep_assert_held(&dev_priv->drm.struct_mutex); 641 lockdep_assert_held(&i915->drm.struct_mutex);
635 642
636 /* 643 /*
637 * Preempt contexts are reserved for exclusive use to inject a 644 * Preempt contexts are reserved for exclusive use to inject a
638 * preemption context switch. They are never to be used for any trivial 645 * preemption context switch. They are never to be used for any trivial
639 * request! 646 * request!
640 */ 647 */
641 GEM_BUG_ON(ctx == dev_priv->preempt_context); 648 GEM_BUG_ON(ctx == i915->preempt_context);
642 649
643 /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report 650 /*
651 * ABI: Before userspace accesses the GPU (e.g. execbuffer), report
644 * EIO if the GPU is already wedged. 652 * EIO if the GPU is already wedged.
645 */ 653 */
646 if (i915_terminally_wedged(&dev_priv->gpu_error)) 654 if (i915_terminally_wedged(&i915->gpu_error))
647 return ERR_PTR(-EIO); 655 return ERR_PTR(-EIO);
648 656
649 /* Pinning the contexts may generate requests in order to acquire 657 /*
658 * Pinning the contexts may generate requests in order to acquire
650 * GGTT space, so do this first before we reserve a seqno for 659 * GGTT space, so do this first before we reserve a seqno for
651 * ourselves. 660 * ourselves.
652 */ 661 */
@@ -664,12 +673,13 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
664 goto err_unreserve; 673 goto err_unreserve;
665 674
666 /* Move the oldest request to the slab-cache (if not in use!) */ 675 /* Move the oldest request to the slab-cache (if not in use!) */
667 req = list_first_entry_or_null(&engine->timeline->requests, 676 rq = list_first_entry_or_null(&engine->timeline->requests,
668 typeof(*req), link); 677 typeof(*rq), link);
669 if (req && i915_gem_request_completed(req)) 678 if (rq && i915_request_completed(rq))
670 i915_gem_request_retire(req); 679 i915_request_retire(rq);
671 680
672 /* Beware: Dragons be flying overhead. 681 /*
682 * Beware: Dragons be flying overhead.
673 * 683 *
674 * We use RCU to look up requests in flight. The lookups may 684 * We use RCU to look up requests in flight. The lookups may
675 * race with the request being allocated from the slab freelist. 685 * race with the request being allocated from the slab freelist.
@@ -697,11 +707,11 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
697 * 707 *
698 * Do not use kmem_cache_zalloc() here! 708 * Do not use kmem_cache_zalloc() here!
699 */ 709 */
700 req = kmem_cache_alloc(dev_priv->requests, 710 rq = kmem_cache_alloc(i915->requests,
701 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 711 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
702 if (unlikely(!req)) { 712 if (unlikely(!rq)) {
703 /* Ratelimit ourselves to prevent oom from malicious clients */ 713 /* Ratelimit ourselves to prevent oom from malicious clients */
704 ret = i915_gem_wait_for_idle(dev_priv, 714 ret = i915_gem_wait_for_idle(i915,
705 I915_WAIT_LOCKED | 715 I915_WAIT_LOCKED |
706 I915_WAIT_INTERRUPTIBLE); 716 I915_WAIT_INTERRUPTIBLE);
707 if (ret) 717 if (ret)
@@ -715,55 +725,55 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
715 * Having already penalized the client to stall, we spend 725 * Having already penalized the client to stall, we spend
716 * a little extra time to re-optimise page allocation. 726 * a little extra time to re-optimise page allocation.
717 */ 727 */
718 kmem_cache_shrink(dev_priv->requests); 728 kmem_cache_shrink(i915->requests);
719 rcu_barrier(); /* Recover the TYPESAFE_BY_RCU pages */ 729 rcu_barrier(); /* Recover the TYPESAFE_BY_RCU pages */
720 730
721 req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL); 731 rq = kmem_cache_alloc(i915->requests, GFP_KERNEL);
722 if (!req) { 732 if (!rq) {
723 ret = -ENOMEM; 733 ret = -ENOMEM;
724 goto err_unreserve; 734 goto err_unreserve;
725 } 735 }
726 } 736 }
727 737
728 req->timeline = i915_gem_context_lookup_timeline(ctx, engine); 738 rq->timeline = i915_gem_context_lookup_timeline(ctx, engine);
729 GEM_BUG_ON(req->timeline == engine->timeline); 739 GEM_BUG_ON(rq->timeline == engine->timeline);
730 740
731 spin_lock_init(&req->lock); 741 spin_lock_init(&rq->lock);
732 dma_fence_init(&req->fence, 742 dma_fence_init(&rq->fence,
733 &i915_fence_ops, 743 &i915_fence_ops,
734 &req->lock, 744 &rq->lock,
735 req->timeline->fence_context, 745 rq->timeline->fence_context,
736 timeline_get_seqno(req->timeline)); 746 timeline_get_seqno(rq->timeline));
737 747
738 /* We bump the ref for the fence chain */ 748 /* We bump the ref for the fence chain */
739 i915_sw_fence_init(&i915_gem_request_get(req)->submit, submit_notify); 749 i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify);
740 init_waitqueue_head(&req->execute); 750 init_waitqueue_head(&rq->execute);
741 751
742 i915_priotree_init(&req->priotree); 752 i915_priotree_init(&rq->priotree);
743 753
744 INIT_LIST_HEAD(&req->active_list); 754 INIT_LIST_HEAD(&rq->active_list);
745 req->i915 = dev_priv; 755 rq->i915 = i915;
746 req->engine = engine; 756 rq->engine = engine;
747 req->ctx = ctx; 757 rq->ctx = ctx;
748 req->ring = ring; 758 rq->ring = ring;
749 759
750 /* No zalloc, must clear what we need by hand */ 760 /* No zalloc, must clear what we need by hand */
751 req->global_seqno = 0; 761 rq->global_seqno = 0;
752 req->signaling.wait.seqno = 0; 762 rq->signaling.wait.seqno = 0;
753 req->file_priv = NULL; 763 rq->file_priv = NULL;
754 req->batch = NULL; 764 rq->batch = NULL;
755 req->capture_list = NULL; 765 rq->capture_list = NULL;
756 req->waitboost = false; 766 rq->waitboost = false;
757 767
758 /* 768 /*
759 * Reserve space in the ring buffer for all the commands required to 769 * Reserve space in the ring buffer for all the commands required to
760 * eventually emit this request. This is to guarantee that the 770 * eventually emit this request. This is to guarantee that the
761 * i915_add_request() call can't fail. Note that the reserve may need 771 * i915_request_add() call can't fail. Note that the reserve may need
762 * to be redone if the request is not actually submitted straight 772 * to be redone if the request is not actually submitted straight
763 * away, e.g. because a GPU scheduler has deferred it. 773 * away, e.g. because a GPU scheduler has deferred it.
764 */ 774 */
765 req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; 775 rq->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
766 GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz); 776 GEM_BUG_ON(rq->reserved_space < engine->emit_breadcrumb_sz);
767 777
768 /* 778 /*
769 * Record the position of the start of the request so that 779 * Record the position of the start of the request so that
@@ -771,30 +781,30 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
771 * GPU processing the request, we never over-estimate the 781 * GPU processing the request, we never over-estimate the
772 * position of the head. 782 * position of the head.
773 */ 783 */
774 req->head = req->ring->emit; 784 rq->head = rq->ring->emit;
775 785
776 /* Unconditionally invalidate GPU caches and TLBs. */ 786 /* Unconditionally invalidate GPU caches and TLBs. */
777 ret = engine->emit_flush(req, EMIT_INVALIDATE); 787 ret = engine->emit_flush(rq, EMIT_INVALIDATE);
778 if (ret) 788 if (ret)
779 goto err_unwind; 789 goto err_unwind;
780 790
781 ret = engine->request_alloc(req); 791 ret = engine->request_alloc(rq);
782 if (ret) 792 if (ret)
783 goto err_unwind; 793 goto err_unwind;
784 794
785 /* Check that we didn't interrupt ourselves with a new request */ 795 /* Check that we didn't interrupt ourselves with a new request */
786 GEM_BUG_ON(req->timeline->seqno != req->fence.seqno); 796 GEM_BUG_ON(rq->timeline->seqno != rq->fence.seqno);
787 return req; 797 return rq;
788 798
789err_unwind: 799err_unwind:
790 req->ring->emit = req->head; 800 rq->ring->emit = rq->head;
791 801
792 /* Make sure we didn't add ourselves to external state before freeing */ 802 /* Make sure we didn't add ourselves to external state before freeing */
793 GEM_BUG_ON(!list_empty(&req->active_list)); 803 GEM_BUG_ON(!list_empty(&rq->active_list));
794 GEM_BUG_ON(!list_empty(&req->priotree.signalers_list)); 804 GEM_BUG_ON(!list_empty(&rq->priotree.signalers_list));
795 GEM_BUG_ON(!list_empty(&req->priotree.waiters_list)); 805 GEM_BUG_ON(!list_empty(&rq->priotree.waiters_list));
796 806
797 kmem_cache_free(dev_priv->requests, req); 807 kmem_cache_free(i915->requests, rq);
798err_unreserve: 808err_unreserve:
799 unreserve_engine(engine); 809 unreserve_engine(engine);
800err_unpin: 810err_unpin:
@@ -803,15 +813,14 @@ err_unpin:
803} 813}
804 814
805static int 815static int
806i915_gem_request_await_request(struct drm_i915_gem_request *to, 816i915_request_await_request(struct i915_request *to, struct i915_request *from)
807 struct drm_i915_gem_request *from)
808{ 817{
809 int ret; 818 int ret;
810 819
811 GEM_BUG_ON(to == from); 820 GEM_BUG_ON(to == from);
812 GEM_BUG_ON(to->timeline == from->timeline); 821 GEM_BUG_ON(to->timeline == from->timeline);
813 822
814 if (i915_gem_request_completed(from)) 823 if (i915_request_completed(from))
815 return 0; 824 return 0;
816 825
817 if (to->engine->schedule) { 826 if (to->engine->schedule) {
@@ -834,7 +843,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
834 843
835 GEM_BUG_ON(!from->engine->semaphore.signal); 844 GEM_BUG_ON(!from->engine->semaphore.signal);
836 845
837 seqno = i915_gem_request_global_seqno(from); 846 seqno = i915_request_global_seqno(from);
838 if (!seqno) 847 if (!seqno)
839 goto await_dma_fence; 848 goto await_dma_fence;
840 849
@@ -858,14 +867,14 @@ await_dma_fence:
858} 867}
859 868
860int 869int
861i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, 870i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
862 struct dma_fence *fence)
863{ 871{
864 struct dma_fence **child = &fence; 872 struct dma_fence **child = &fence;
865 unsigned int nchild = 1; 873 unsigned int nchild = 1;
866 int ret; 874 int ret;
867 875
868 /* Note that if the fence-array was created in signal-on-any mode, 876 /*
877 * Note that if the fence-array was created in signal-on-any mode,
869 * we should *not* decompose it into its individual fences. However, 878 * we should *not* decompose it into its individual fences. However,
870 * we don't currently store which mode the fence-array is operating 879 * we don't currently store which mode the fence-array is operating
871 * in. Fortunately, the only user of signal-on-any is private to 880 * in. Fortunately, the only user of signal-on-any is private to
@@ -887,37 +896,36 @@ i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
887 896
888 /* 897 /*
889 * Requests on the same timeline are explicitly ordered, along 898 * Requests on the same timeline are explicitly ordered, along
890 * with their dependencies, by i915_add_request() which ensures 899 * with their dependencies, by i915_request_add() which ensures
891 * that requests are submitted in-order through each ring. 900 * that requests are submitted in-order through each ring.
892 */ 901 */
893 if (fence->context == req->fence.context) 902 if (fence->context == rq->fence.context)
894 continue; 903 continue;
895 904
896 /* Squash repeated waits to the same timelines */ 905 /* Squash repeated waits to the same timelines */
897 if (fence->context != req->i915->mm.unordered_timeline && 906 if (fence->context != rq->i915->mm.unordered_timeline &&
898 intel_timeline_sync_is_later(req->timeline, fence)) 907 intel_timeline_sync_is_later(rq->timeline, fence))
899 continue; 908 continue;
900 909
901 if (dma_fence_is_i915(fence)) 910 if (dma_fence_is_i915(fence))
902 ret = i915_gem_request_await_request(req, 911 ret = i915_request_await_request(rq, to_request(fence));
903 to_request(fence));
904 else 912 else
905 ret = i915_sw_fence_await_dma_fence(&req->submit, fence, 913 ret = i915_sw_fence_await_dma_fence(&rq->submit, fence,
906 I915_FENCE_TIMEOUT, 914 I915_FENCE_TIMEOUT,
907 I915_FENCE_GFP); 915 I915_FENCE_GFP);
908 if (ret < 0) 916 if (ret < 0)
909 return ret; 917 return ret;
910 918
911 /* Record the latest fence used against each timeline */ 919 /* Record the latest fence used against each timeline */
912 if (fence->context != req->i915->mm.unordered_timeline) 920 if (fence->context != rq->i915->mm.unordered_timeline)
913 intel_timeline_sync_set(req->timeline, fence); 921 intel_timeline_sync_set(rq->timeline, fence);
914 } while (--nchild); 922 } while (--nchild);
915 923
916 return 0; 924 return 0;
917} 925}
918 926
919/** 927/**
920 * i915_gem_request_await_object - set this request to (async) wait upon a bo 928 * i915_request_await_object - set this request to (async) wait upon a bo
921 * @to: request we are wishing to use 929 * @to: request we are wishing to use
922 * @obj: object which may be in use on another ring. 930 * @obj: object which may be in use on another ring.
923 * @write: whether the wait is on behalf of a writer 931 * @write: whether the wait is on behalf of a writer
@@ -937,9 +945,9 @@ i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
937 * Returns 0 if successful, else propagates up the lower layer error. 945 * Returns 0 if successful, else propagates up the lower layer error.
938 */ 946 */
939int 947int
940i915_gem_request_await_object(struct drm_i915_gem_request *to, 948i915_request_await_object(struct i915_request *to,
941 struct drm_i915_gem_object *obj, 949 struct drm_i915_gem_object *obj,
942 bool write) 950 bool write)
943{ 951{
944 struct dma_fence *excl; 952 struct dma_fence *excl;
945 int ret = 0; 953 int ret = 0;
@@ -954,7 +962,7 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to,
954 return ret; 962 return ret;
955 963
956 for (i = 0; i < count; i++) { 964 for (i = 0; i < count; i++) {
957 ret = i915_gem_request_await_dma_fence(to, shared[i]); 965 ret = i915_request_await_dma_fence(to, shared[i]);
958 if (ret) 966 if (ret)
959 break; 967 break;
960 968
@@ -970,7 +978,7 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to,
970 978
971 if (excl) { 979 if (excl) {
972 if (ret == 0) 980 if (ret == 0)
973 ret = i915_gem_request_await_dma_fence(to, excl); 981 ret = i915_request_await_dma_fence(to, excl);
974 982
975 dma_fence_put(excl); 983 dma_fence_put(excl);
976 } 984 }
@@ -983,21 +991,21 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to,
983 * request is not being tracked for completion but the work itself is 991 * request is not being tracked for completion but the work itself is
984 * going to happen on the hardware. This would be a Bad Thing(tm). 992 * going to happen on the hardware. This would be a Bad Thing(tm).
985 */ 993 */
986void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) 994void __i915_request_add(struct i915_request *request, bool flush_caches)
987{ 995{
988 struct intel_engine_cs *engine = request->engine; 996 struct intel_engine_cs *engine = request->engine;
989 struct intel_ring *ring = request->ring; 997 struct intel_ring *ring = request->ring;
990 struct intel_timeline *timeline = request->timeline; 998 struct intel_timeline *timeline = request->timeline;
991 struct drm_i915_gem_request *prev; 999 struct i915_request *prev;
992 u32 *cs; 1000 u32 *cs;
993 int err; 1001 int err;
994 1002
995 lockdep_assert_held(&request->i915->drm.struct_mutex); 1003 lockdep_assert_held(&request->i915->drm.struct_mutex);
996 trace_i915_gem_request_add(request); 1004 trace_i915_request_add(request);
997 1005
998 /* 1006 /*
999 * Make sure that no request gazumped us - if it was allocated after 1007 * Make sure that no request gazumped us - if it was allocated after
1000 * our i915_gem_request_alloc() and called __i915_add_request() before 1008 * our i915_request_alloc() and called __i915_request_add() before
1001 * us, the timeline will hold its seqno which is later than ours. 1009 * us, the timeline will hold its seqno which is later than ours.
1002 */ 1010 */
1003 GEM_BUG_ON(timeline->seqno != request->fence.seqno); 1011 GEM_BUG_ON(timeline->seqno != request->fence.seqno);
@@ -1042,7 +1050,7 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
1042 1050
1043 prev = i915_gem_active_raw(&timeline->last_request, 1051 prev = i915_gem_active_raw(&timeline->last_request,
1044 &request->i915->drm.struct_mutex); 1052 &request->i915->drm.struct_mutex);
1045 if (prev && !i915_gem_request_completed(prev)) { 1053 if (prev && !i915_request_completed(prev)) {
1046 i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, 1054 i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
1047 &request->submitq); 1055 &request->submitq);
1048 if (engine->schedule) 1056 if (engine->schedule)
@@ -1097,15 +1105,16 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
1097 * work on behalf of others -- but instead we should benefit from 1105 * work on behalf of others -- but instead we should benefit from
1098 * improved resource management. (Well, that's the theory at least.) 1106 * improved resource management. (Well, that's the theory at least.)
1099 */ 1107 */
1100 if (prev && i915_gem_request_completed(prev)) 1108 if (prev && i915_request_completed(prev))
1101 i915_gem_request_retire_upto(prev); 1109 i915_request_retire_upto(prev);
1102} 1110}
1103 1111
1104static unsigned long local_clock_us(unsigned int *cpu) 1112static unsigned long local_clock_us(unsigned int *cpu)
1105{ 1113{
1106 unsigned long t; 1114 unsigned long t;
1107 1115
1108 /* Cheaply and approximately convert from nanoseconds to microseconds. 1116 /*
1117 * Cheaply and approximately convert from nanoseconds to microseconds.
1109 * The result and subsequent calculations are also defined in the same 1118 * The result and subsequent calculations are also defined in the same
1110 * approximate microseconds units. The principal source of timing 1119 * approximate microseconds units. The principal source of timing
1111 * error here is from the simple truncation. 1120 * error here is from the simple truncation.
@@ -1133,10 +1142,10 @@ static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1133 return this_cpu != cpu; 1142 return this_cpu != cpu;
1134} 1143}
1135 1144
1136static bool __i915_spin_request(const struct drm_i915_gem_request *req, 1145static bool __i915_spin_request(const struct i915_request *rq,
1137 u32 seqno, int state, unsigned long timeout_us) 1146 u32 seqno, int state, unsigned long timeout_us)
1138{ 1147{
1139 struct intel_engine_cs *engine = req->engine; 1148 struct intel_engine_cs *engine = rq->engine;
1140 unsigned int irq, cpu; 1149 unsigned int irq, cpu;
1141 1150
1142 GEM_BUG_ON(!seqno); 1151 GEM_BUG_ON(!seqno);
@@ -1155,7 +1164,8 @@ static bool __i915_spin_request(const struct drm_i915_gem_request *req,
1155 if (!i915_seqno_passed(intel_engine_get_seqno(engine), seqno - 1)) 1164 if (!i915_seqno_passed(intel_engine_get_seqno(engine), seqno - 1))
1156 return false; 1165 return false;
1157 1166
1158 /* When waiting for high frequency requests, e.g. during synchronous 1167 /*
1168 * When waiting for high frequency requests, e.g. during synchronous
1159 * rendering split between the CPU and GPU, the finite amount of time 1169 * rendering split between the CPU and GPU, the finite amount of time
1160 * required to set up the irq and wait upon it limits the response 1170 * required to set up the irq and wait upon it limits the response
1161 * rate. By busywaiting on the request completion for a short while we 1171 * rate. By busywaiting on the request completion for a short while we
@@ -1169,9 +1179,10 @@ static bool __i915_spin_request(const struct drm_i915_gem_request *req,
1169 timeout_us += local_clock_us(&cpu); 1179 timeout_us += local_clock_us(&cpu);
1170 do { 1180 do {
1171 if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno)) 1181 if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno))
1172 return seqno == i915_gem_request_global_seqno(req); 1182 return seqno == i915_request_global_seqno(rq);
1173 1183
1174 /* Seqno are meant to be ordered *before* the interrupt. If 1184 /*
1185 * Seqno are meant to be ordered *before* the interrupt. If
1175 * we see an interrupt without a corresponding seqno advance, 1186 * we see an interrupt without a corresponding seqno advance,
1176 * assume we won't see one in the near future but require 1187 * assume we won't see one in the near future but require
1177 * the engine->seqno_barrier() to fixup coherency. 1188 * the engine->seqno_barrier() to fixup coherency.
@@ -1191,7 +1202,7 @@ static bool __i915_spin_request(const struct drm_i915_gem_request *req,
1191 return false; 1202 return false;
1192} 1203}
1193 1204
1194static bool __i915_wait_request_check_and_reset(struct drm_i915_gem_request *request) 1205static bool __i915_wait_request_check_and_reset(struct i915_request *request)
1195{ 1206{
1196 if (likely(!i915_reset_handoff(&request->i915->gpu_error))) 1207 if (likely(!i915_reset_handoff(&request->i915->gpu_error)))
1197 return false; 1208 return false;
@@ -1202,12 +1213,12 @@ static bool __i915_wait_request_check_and_reset(struct drm_i915_gem_request *req
1202} 1213}
1203 1214
1204/** 1215/**
1205 * i915_wait_request - wait until execution of request has finished 1216 * i915_request_wait - wait until execution of request has finished
1206 * @req: the request to wait upon 1217 * @rq: the request to wait upon
1207 * @flags: how to wait 1218 * @flags: how to wait
1208 * @timeout: how long to wait in jiffies 1219 * @timeout: how long to wait in jiffies
1209 * 1220 *
1210 * i915_wait_request() waits for the request to be completed, for a 1221 * i915_request_wait() waits for the request to be completed, for a
1211 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an 1222 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
1212 * unbounded wait). 1223 * unbounded wait).
1213 * 1224 *
@@ -1220,13 +1231,13 @@ static bool __i915_wait_request_check_and_reset(struct drm_i915_gem_request *req
1220 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is 1231 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
1221 * pending before the request completes. 1232 * pending before the request completes.
1222 */ 1233 */
1223long i915_wait_request(struct drm_i915_gem_request *req, 1234long i915_request_wait(struct i915_request *rq,
1224 unsigned int flags, 1235 unsigned int flags,
1225 long timeout) 1236 long timeout)
1226{ 1237{
1227 const int state = flags & I915_WAIT_INTERRUPTIBLE ? 1238 const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1228 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; 1239 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1229 wait_queue_head_t *errq = &req->i915->gpu_error.wait_queue; 1240 wait_queue_head_t *errq = &rq->i915->gpu_error.wait_queue;
1230 DEFINE_WAIT_FUNC(reset, default_wake_function); 1241 DEFINE_WAIT_FUNC(reset, default_wake_function);
1231 DEFINE_WAIT_FUNC(exec, default_wake_function); 1242 DEFINE_WAIT_FUNC(exec, default_wake_function);
1232 struct intel_wait wait; 1243 struct intel_wait wait;
@@ -1234,33 +1245,33 @@ long i915_wait_request(struct drm_i915_gem_request *req,
1234 might_sleep(); 1245 might_sleep();
1235#if IS_ENABLED(CONFIG_LOCKDEP) 1246#if IS_ENABLED(CONFIG_LOCKDEP)
1236 GEM_BUG_ON(debug_locks && 1247 GEM_BUG_ON(debug_locks &&
1237 !!lockdep_is_held(&req->i915->drm.struct_mutex) != 1248 !!lockdep_is_held(&rq->i915->drm.struct_mutex) !=
1238 !!(flags & I915_WAIT_LOCKED)); 1249 !!(flags & I915_WAIT_LOCKED));
1239#endif 1250#endif
1240 GEM_BUG_ON(timeout < 0); 1251 GEM_BUG_ON(timeout < 0);
1241 1252
1242 if (i915_gem_request_completed(req)) 1253 if (i915_request_completed(rq))
1243 return timeout; 1254 return timeout;
1244 1255
1245 if (!timeout) 1256 if (!timeout)
1246 return -ETIME; 1257 return -ETIME;
1247 1258
1248 trace_i915_gem_request_wait_begin(req, flags); 1259 trace_i915_request_wait_begin(rq, flags);
1249 1260
1250 add_wait_queue(&req->execute, &exec); 1261 add_wait_queue(&rq->execute, &exec);
1251 if (flags & I915_WAIT_LOCKED) 1262 if (flags & I915_WAIT_LOCKED)
1252 add_wait_queue(errq, &reset); 1263 add_wait_queue(errq, &reset);
1253 1264
1254 intel_wait_init(&wait, req); 1265 intel_wait_init(&wait, rq);
1255 1266
1256restart: 1267restart:
1257 do { 1268 do {
1258 set_current_state(state); 1269 set_current_state(state);
1259 if (intel_wait_update_request(&wait, req)) 1270 if (intel_wait_update_request(&wait, rq))
1260 break; 1271 break;
1261 1272
1262 if (flags & I915_WAIT_LOCKED && 1273 if (flags & I915_WAIT_LOCKED &&
1263 __i915_wait_request_check_and_reset(req)) 1274 __i915_wait_request_check_and_reset(rq))
1264 continue; 1275 continue;
1265 1276
1266 if (signal_pending_state(state, current)) { 1277 if (signal_pending_state(state, current)) {
@@ -1277,22 +1288,23 @@ restart:
1277 } while (1); 1288 } while (1);
1278 1289
1279 GEM_BUG_ON(!intel_wait_has_seqno(&wait)); 1290 GEM_BUG_ON(!intel_wait_has_seqno(&wait));
1280 GEM_BUG_ON(!i915_sw_fence_signaled(&req->submit)); 1291 GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
1281 1292
1282 /* Optimistic short spin before touching IRQs */ 1293 /* Optimistic short spin before touching IRQs */
1283 if (__i915_spin_request(req, wait.seqno, state, 5)) 1294 if (__i915_spin_request(rq, wait.seqno, state, 5))
1284 goto complete; 1295 goto complete;
1285 1296
1286 set_current_state(state); 1297 set_current_state(state);
1287 if (intel_engine_add_wait(req->engine, &wait)) 1298 if (intel_engine_add_wait(rq->engine, &wait))
1288 /* In order to check that we haven't missed the interrupt 1299 /*
1300 * In order to check that we haven't missed the interrupt
1289 * as we enabled it, we need to kick ourselves to do a 1301 * as we enabled it, we need to kick ourselves to do a
1290 * coherent check on the seqno before we sleep. 1302 * coherent check on the seqno before we sleep.
1291 */ 1303 */
1292 goto wakeup; 1304 goto wakeup;
1293 1305
1294 if (flags & I915_WAIT_LOCKED) 1306 if (flags & I915_WAIT_LOCKED)
1295 __i915_wait_request_check_and_reset(req); 1307 __i915_wait_request_check_and_reset(rq);
1296 1308
1297 for (;;) { 1309 for (;;) {
1298 if (signal_pending_state(state, current)) { 1310 if (signal_pending_state(state, current)) {
@@ -1308,21 +1320,23 @@ restart:
1308 timeout = io_schedule_timeout(timeout); 1320 timeout = io_schedule_timeout(timeout);
1309 1321
1310 if (intel_wait_complete(&wait) && 1322 if (intel_wait_complete(&wait) &&
1311 intel_wait_check_request(&wait, req)) 1323 intel_wait_check_request(&wait, rq))
1312 break; 1324 break;
1313 1325
1314 set_current_state(state); 1326 set_current_state(state);
1315 1327
1316wakeup: 1328wakeup:
1317 /* Carefully check if the request is complete, giving time 1329 /*
1330 * Carefully check if the request is complete, giving time
1318 * for the seqno to be visible following the interrupt. 1331 * for the seqno to be visible following the interrupt.
1319 * We also have to check in case we are kicked by the GPU 1332 * We also have to check in case we are kicked by the GPU
1320 * reset in order to drop the struct_mutex. 1333 * reset in order to drop the struct_mutex.
1321 */ 1334 */
1322 if (__i915_request_irq_complete(req)) 1335 if (__i915_request_irq_complete(rq))
1323 break; 1336 break;
1324 1337
1325 /* If the GPU is hung, and we hold the lock, reset the GPU 1338 /*
1339 * If the GPU is hung, and we hold the lock, reset the GPU
1326 * and then check for completion. On a full reset, the engine's 1340 * and then check for completion. On a full reset, the engine's
1327 * HW seqno will be advanced passed us and we are complete. 1341 * HW seqno will be advanced passed us and we are complete.
1328 * If we do a partial reset, we have to wait for the GPU to 1342 * If we do a partial reset, we have to wait for the GPU to
@@ -1333,33 +1347,33 @@ wakeup:
1333 * itself, or indirectly by recovering the GPU). 1347 * itself, or indirectly by recovering the GPU).
1334 */ 1348 */
1335 if (flags & I915_WAIT_LOCKED && 1349 if (flags & I915_WAIT_LOCKED &&
1336 __i915_wait_request_check_and_reset(req)) 1350 __i915_wait_request_check_and_reset(rq))
1337 continue; 1351 continue;
1338 1352
1339 /* Only spin if we know the GPU is processing this request */ 1353 /* Only spin if we know the GPU is processing this request */
1340 if (__i915_spin_request(req, wait.seqno, state, 2)) 1354 if (__i915_spin_request(rq, wait.seqno, state, 2))
1341 break; 1355 break;
1342 1356
1343 if (!intel_wait_check_request(&wait, req)) { 1357 if (!intel_wait_check_request(&wait, rq)) {
1344 intel_engine_remove_wait(req->engine, &wait); 1358 intel_engine_remove_wait(rq->engine, &wait);
1345 goto restart; 1359 goto restart;
1346 } 1360 }
1347 } 1361 }
1348 1362
1349 intel_engine_remove_wait(req->engine, &wait); 1363 intel_engine_remove_wait(rq->engine, &wait);
1350complete: 1364complete:
1351 __set_current_state(TASK_RUNNING); 1365 __set_current_state(TASK_RUNNING);
1352 if (flags & I915_WAIT_LOCKED) 1366 if (flags & I915_WAIT_LOCKED)
1353 remove_wait_queue(errq, &reset); 1367 remove_wait_queue(errq, &reset);
1354 remove_wait_queue(&req->execute, &exec); 1368 remove_wait_queue(&rq->execute, &exec);
1355 trace_i915_gem_request_wait_end(req); 1369 trace_i915_request_wait_end(rq);
1356 1370
1357 return timeout; 1371 return timeout;
1358} 1372}
1359 1373
1360static void engine_retire_requests(struct intel_engine_cs *engine) 1374static void engine_retire_requests(struct intel_engine_cs *engine)
1361{ 1375{
1362 struct drm_i915_gem_request *request, *next; 1376 struct i915_request *request, *next;
1363 u32 seqno = intel_engine_get_seqno(engine); 1377 u32 seqno = intel_engine_get_seqno(engine);
1364 LIST_HEAD(retire); 1378 LIST_HEAD(retire);
1365 1379
@@ -1374,24 +1388,24 @@ static void engine_retire_requests(struct intel_engine_cs *engine)
1374 spin_unlock_irq(&engine->timeline->lock); 1388 spin_unlock_irq(&engine->timeline->lock);
1375 1389
1376 list_for_each_entry_safe(request, next, &retire, link) 1390 list_for_each_entry_safe(request, next, &retire, link)
1377 i915_gem_request_retire(request); 1391 i915_request_retire(request);
1378} 1392}
1379 1393
1380void i915_gem_retire_requests(struct drm_i915_private *dev_priv) 1394void i915_retire_requests(struct drm_i915_private *i915)
1381{ 1395{
1382 struct intel_engine_cs *engine; 1396 struct intel_engine_cs *engine;
1383 enum intel_engine_id id; 1397 enum intel_engine_id id;
1384 1398
1385 lockdep_assert_held(&dev_priv->drm.struct_mutex); 1399 lockdep_assert_held(&i915->drm.struct_mutex);
1386 1400
1387 if (!dev_priv->gt.active_requests) 1401 if (!i915->gt.active_requests)
1388 return; 1402 return;
1389 1403
1390 for_each_engine(engine, dev_priv, id) 1404 for_each_engine(engine, i915, id)
1391 engine_retire_requests(engine); 1405 engine_retire_requests(engine);
1392} 1406}
1393 1407
1394#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1408#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1395#include "selftests/mock_request.c" 1409#include "selftests/mock_request.c"
1396#include "selftests/i915_gem_request.c" 1410#include "selftests/i915_request.c"
1397#endif 1411#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_request.h
index 2236e9188c5c..7d6eb82eeb91 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright © 2008-2015 Intel Corporation 2 * Copyright © 2008-2018 Intel Corporation
3 * 3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a 4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"), 5 * copy of this software and associated documentation files (the "Software"),
@@ -22,8 +22,8 @@
22 * 22 *
23 */ 23 */
24 24
25#ifndef I915_GEM_REQUEST_H 25#ifndef I915_REQUEST_H
26#define I915_GEM_REQUEST_H 26#define I915_REQUEST_H
27 27
28#include <linux/dma-fence.h> 28#include <linux/dma-fence.h>
29 29
@@ -34,18 +34,18 @@
34 34
35struct drm_file; 35struct drm_file;
36struct drm_i915_gem_object; 36struct drm_i915_gem_object;
37struct drm_i915_gem_request; 37struct i915_request;
38 38
39struct intel_wait { 39struct intel_wait {
40 struct rb_node node; 40 struct rb_node node;
41 struct task_struct *tsk; 41 struct task_struct *tsk;
42 struct drm_i915_gem_request *request; 42 struct i915_request *request;
43 u32 seqno; 43 u32 seqno;
44}; 44};
45 45
46struct intel_signal_node { 46struct intel_signal_node {
47 struct rb_node node;
48 struct intel_wait wait; 47 struct intel_wait wait;
48 struct list_head link;
49}; 49};
50 50
51struct i915_dependency { 51struct i915_dependency {
@@ -57,7 +57,12 @@ struct i915_dependency {
57#define I915_DEPENDENCY_ALLOC BIT(0) 57#define I915_DEPENDENCY_ALLOC BIT(0)
58}; 58};
59 59
60/* Requests exist in a complex web of interdependencies. Each request 60/*
61 * "People assume that time is a strict progression of cause to effect, but
62 * actually, from a nonlinear, non-subjective viewpoint, it's more like a big
63 * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015
64 *
65 * Requests exist in a complex web of interdependencies. Each request
61 * has to wait for some other request to complete before it is ready to be run 66 * has to wait for some other request to complete before it is ready to be run
62 * (e.g. we have to wait until the pixels have been rendering into a texture 67 * (e.g. we have to wait until the pixels have been rendering into a texture
63 * before we can copy from it). We track the readiness of a request in terms 68 * before we can copy from it). We track the readiness of a request in terms
@@ -81,8 +86,8 @@ enum {
81 I915_PRIORITY_INVALID = INT_MIN 86 I915_PRIORITY_INVALID = INT_MIN
82}; 87};
83 88
84struct i915_gem_capture_list { 89struct i915_capture_list {
85 struct i915_gem_capture_list *next; 90 struct i915_capture_list *next;
86 struct i915_vma *vma; 91 struct i915_vma *vma;
87}; 92};
88 93
@@ -106,7 +111,7 @@ struct i915_gem_capture_list {
106 * 111 *
107 * The requests are reference counted. 112 * The requests are reference counted.
108 */ 113 */
109struct drm_i915_gem_request { 114struct i915_request {
110 struct dma_fence fence; 115 struct dma_fence fence;
111 spinlock_t lock; 116 spinlock_t lock;
112 117
@@ -120,7 +125,7 @@ struct drm_i915_gem_request {
120 * it persists while any request is linked to it. Requests themselves 125 * it persists while any request is linked to it. Requests themselves
121 * are also refcounted, so the request will only be freed when the last 126 * are also refcounted, so the request will only be freed when the last
122 * reference to it is dismissed, and the code in 127 * reference to it is dismissed, and the code in
123 * i915_gem_request_free() will then decrement the refcount on the 128 * i915_request_free() will then decrement the refcount on the
124 * context. 129 * context.
125 */ 130 */
126 struct i915_gem_context *ctx; 131 struct i915_gem_context *ctx;
@@ -129,7 +134,8 @@ struct drm_i915_gem_request {
129 struct intel_timeline *timeline; 134 struct intel_timeline *timeline;
130 struct intel_signal_node signaling; 135 struct intel_signal_node signaling;
131 136
132 /* Fences for the various phases in the request's lifetime. 137 /*
138 * Fences for the various phases in the request's lifetime.
133 * 139 *
134 * The submit fence is used to await upon all of the request's 140 * The submit fence is used to await upon all of the request's
135 * dependencies. When it is signaled, the request is ready to run. 141 * dependencies. When it is signaled, the request is ready to run.
@@ -139,7 +145,8 @@ struct drm_i915_gem_request {
139 wait_queue_entry_t submitq; 145 wait_queue_entry_t submitq;
140 wait_queue_head_t execute; 146 wait_queue_head_t execute;
141 147
142 /* A list of everyone we wait upon, and everyone who waits upon us. 148 /*
149 * A list of everyone we wait upon, and everyone who waits upon us.
143 * Even though we will not be submitted to the hardware before the 150 * Even though we will not be submitted to the hardware before the
144 * submit fence is signaled (it waits for all external events as well 151 * submit fence is signaled (it waits for all external events as well
145 * as our own requests), the scheduler still needs to know the 152 * as our own requests), the scheduler still needs to know the
@@ -150,7 +157,8 @@ struct drm_i915_gem_request {
150 struct i915_priotree priotree; 157 struct i915_priotree priotree;
151 struct i915_dependency dep; 158 struct i915_dependency dep;
152 159
153 /** GEM sequence number associated with this request on the 160 /**
161 * GEM sequence number associated with this request on the
154 * global execution timeline. It is zero when the request is not 162 * global execution timeline. It is zero when the request is not
155 * on the HW queue (i.e. not on the engine timeline list). 163 * on the HW queue (i.e. not on the engine timeline list).
156 * Its value is guarded by the timeline spinlock. 164 * Its value is guarded by the timeline spinlock.
@@ -180,12 +188,13 @@ struct drm_i915_gem_request {
180 * error state dump only). 188 * error state dump only).
181 */ 189 */
182 struct i915_vma *batch; 190 struct i915_vma *batch;
183 /** Additional buffers requested by userspace to be captured upon 191 /**
192 * Additional buffers requested by userspace to be captured upon
184 * a GPU hang. The vma/obj on this list are protected by their 193 * a GPU hang. The vma/obj on this list are protected by their
185 * active reference - all objects on this list must also be 194 * active reference - all objects on this list must also be
186 * on the active_list (of their final request). 195 * on the active_list (of their final request).
187 */ 196 */
188 struct i915_gem_capture_list *capture_list; 197 struct i915_capture_list *capture_list;
189 struct list_head active_list; 198 struct list_head active_list;
190 199
191 /** Time at which this request was emitted, in jiffies. */ 200 /** Time at which this request was emitted, in jiffies. */
@@ -213,40 +222,40 @@ static inline bool dma_fence_is_i915(const struct dma_fence *fence)
213 return fence->ops == &i915_fence_ops; 222 return fence->ops == &i915_fence_ops;
214} 223}
215 224
216struct drm_i915_gem_request * __must_check 225struct i915_request * __must_check
217i915_gem_request_alloc(struct intel_engine_cs *engine, 226i915_request_alloc(struct intel_engine_cs *engine,
218 struct i915_gem_context *ctx); 227 struct i915_gem_context *ctx);
219void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); 228void i915_request_retire_upto(struct i915_request *rq);
220 229
221static inline struct drm_i915_gem_request * 230static inline struct i915_request *
222to_request(struct dma_fence *fence) 231to_request(struct dma_fence *fence)
223{ 232{
224 /* We assume that NULL fence/request are interoperable */ 233 /* We assume that NULL fence/request are interoperable */
225 BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); 234 BUILD_BUG_ON(offsetof(struct i915_request, fence) != 0);
226 GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); 235 GEM_BUG_ON(fence && !dma_fence_is_i915(fence));
227 return container_of(fence, struct drm_i915_gem_request, fence); 236 return container_of(fence, struct i915_request, fence);
228} 237}
229 238
230static inline struct drm_i915_gem_request * 239static inline struct i915_request *
231i915_gem_request_get(struct drm_i915_gem_request *req) 240i915_request_get(struct i915_request *rq)
232{ 241{
233 return to_request(dma_fence_get(&req->fence)); 242 return to_request(dma_fence_get(&rq->fence));
234} 243}
235 244
236static inline struct drm_i915_gem_request * 245static inline struct i915_request *
237i915_gem_request_get_rcu(struct drm_i915_gem_request *req) 246i915_request_get_rcu(struct i915_request *rq)
238{ 247{
239 return to_request(dma_fence_get_rcu(&req->fence)); 248 return to_request(dma_fence_get_rcu(&rq->fence));
240} 249}
241 250
242static inline void 251static inline void
243i915_gem_request_put(struct drm_i915_gem_request *req) 252i915_request_put(struct i915_request *rq)
244{ 253{
245 dma_fence_put(&req->fence); 254 dma_fence_put(&rq->fence);
246} 255}
247 256
248/** 257/**
249 * i915_gem_request_global_seqno - report the current global seqno 258 * i915_request_global_seqno - report the current global seqno
250 * @request - the request 259 * @request - the request
251 * 260 *
252 * A request is assigned a global seqno only when it is on the hardware 261 * A request is assigned a global seqno only when it is on the hardware
@@ -264,34 +273,28 @@ i915_gem_request_put(struct drm_i915_gem_request *req)
264 * after the read, it is indeed complete). 273 * after the read, it is indeed complete).
265 */ 274 */
266static u32 275static u32
267i915_gem_request_global_seqno(const struct drm_i915_gem_request *request) 276i915_request_global_seqno(const struct i915_request *request)
268{ 277{
269 return READ_ONCE(request->global_seqno); 278 return READ_ONCE(request->global_seqno);
270} 279}
271 280
272int 281int i915_request_await_object(struct i915_request *to,
273i915_gem_request_await_object(struct drm_i915_gem_request *to,
274 struct drm_i915_gem_object *obj, 282 struct drm_i915_gem_object *obj,
275 bool write); 283 bool write);
276int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, 284int i915_request_await_dma_fence(struct i915_request *rq,
277 struct dma_fence *fence); 285 struct dma_fence *fence);
278 286
279void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches); 287void __i915_request_add(struct i915_request *rq, bool flush_caches);
280#define i915_add_request(req) \ 288#define i915_request_add(rq) \
281 __i915_add_request(req, false) 289 __i915_request_add(rq, false)
282 290
283void __i915_gem_request_submit(struct drm_i915_gem_request *request); 291void __i915_request_submit(struct i915_request *request);
284void i915_gem_request_submit(struct drm_i915_gem_request *request); 292void i915_request_submit(struct i915_request *request);
285 293
286void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request); 294void __i915_request_unsubmit(struct i915_request *request);
287void i915_gem_request_unsubmit(struct drm_i915_gem_request *request); 295void i915_request_unsubmit(struct i915_request *request);
288 296
289struct intel_rps_client; 297long i915_request_wait(struct i915_request *rq,
290#define NO_WAITBOOST ERR_PTR(-1)
291#define IS_RPS_CLIENT(p) (!IS_ERR(p))
292#define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p))
293
294long i915_wait_request(struct drm_i915_gem_request *req,
295 unsigned int flags, 298 unsigned int flags,
296 long timeout) 299 long timeout)
297 __attribute__((nonnull(1))); 300 __attribute__((nonnull(1)));
@@ -310,47 +313,48 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
310} 313}
311 314
312static inline bool 315static inline bool
313__i915_gem_request_completed(const struct drm_i915_gem_request *req, u32 seqno) 316__i915_request_completed(const struct i915_request *rq, u32 seqno)
314{ 317{
315 GEM_BUG_ON(!seqno); 318 GEM_BUG_ON(!seqno);
316 return i915_seqno_passed(intel_engine_get_seqno(req->engine), seqno) && 319 return i915_seqno_passed(intel_engine_get_seqno(rq->engine), seqno) &&
317 seqno == i915_gem_request_global_seqno(req); 320 seqno == i915_request_global_seqno(rq);
318} 321}
319 322
320static inline bool 323static inline bool i915_request_completed(const struct i915_request *rq)
321i915_gem_request_completed(const struct drm_i915_gem_request *req)
322{ 324{
323 u32 seqno; 325 u32 seqno;
324 326
325 seqno = i915_gem_request_global_seqno(req); 327 seqno = i915_request_global_seqno(rq);
326 if (!seqno) 328 if (!seqno)
327 return false; 329 return false;
328 330
329 return __i915_gem_request_completed(req, seqno); 331 return __i915_request_completed(rq, seqno);
330} 332}
331 333
332static inline bool 334static inline bool i915_request_started(const struct i915_request *rq)
333i915_gem_request_started(const struct drm_i915_gem_request *req)
334{ 335{
335 u32 seqno; 336 u32 seqno;
336 337
337 seqno = i915_gem_request_global_seqno(req); 338 seqno = i915_request_global_seqno(rq);
338 if (!seqno) 339 if (!seqno)
339 return false; 340 return false;
340 341
341 return i915_seqno_passed(intel_engine_get_seqno(req->engine), 342 return i915_seqno_passed(intel_engine_get_seqno(rq->engine),
342 seqno - 1); 343 seqno - 1);
343} 344}
344 345
345static inline bool i915_priotree_signaled(const struct i915_priotree *pt) 346static inline bool i915_priotree_signaled(const struct i915_priotree *pt)
346{ 347{
347 const struct drm_i915_gem_request *rq = 348 const struct i915_request *rq =
348 container_of(pt, const struct drm_i915_gem_request, priotree); 349 container_of(pt, const struct i915_request, priotree);
349 350
350 return i915_gem_request_completed(rq); 351 return i915_request_completed(rq);
351} 352}
352 353
353/* We treat requests as fences. This is not be to confused with our 354void i915_retire_requests(struct drm_i915_private *i915);
355
356/*
357 * We treat requests as fences. This is not be to confused with our
354 * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. 358 * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
355 * We use the fences to synchronize access from the CPU with activity on the 359 * We use the fences to synchronize access from the CPU with activity on the
356 * GPU, for example, we should not rewrite an object's PTE whilst the GPU 360 * GPU, for example, we should not rewrite an object's PTE whilst the GPU
@@ -380,16 +384,16 @@ static inline bool i915_priotree_signaled(const struct i915_priotree *pt)
380struct i915_gem_active; 384struct i915_gem_active;
381 385
382typedef void (*i915_gem_retire_fn)(struct i915_gem_active *, 386typedef void (*i915_gem_retire_fn)(struct i915_gem_active *,
383 struct drm_i915_gem_request *); 387 struct i915_request *);
384 388
385struct i915_gem_active { 389struct i915_gem_active {
386 struct drm_i915_gem_request __rcu *request; 390 struct i915_request __rcu *request;
387 struct list_head link; 391 struct list_head link;
388 i915_gem_retire_fn retire; 392 i915_gem_retire_fn retire;
389}; 393};
390 394
391void i915_gem_retire_noop(struct i915_gem_active *, 395void i915_gem_retire_noop(struct i915_gem_active *,
392 struct drm_i915_gem_request *request); 396 struct i915_request *request);
393 397
394/** 398/**
395 * init_request_active - prepares the activity tracker for use 399 * init_request_active - prepares the activity tracker for use
@@ -421,7 +425,7 @@ init_request_active(struct i915_gem_active *active,
421 */ 425 */
422static inline void 426static inline void
423i915_gem_active_set(struct i915_gem_active *active, 427i915_gem_active_set(struct i915_gem_active *active,
424 struct drm_i915_gem_request *request) 428 struct i915_request *request)
425{ 429{
426 list_move(&active->link, &request->active_list); 430 list_move(&active->link, &request->active_list);
427 rcu_assign_pointer(active->request, request); 431 rcu_assign_pointer(active->request, request);
@@ -446,10 +450,11 @@ i915_gem_active_set_retire_fn(struct i915_gem_active *active,
446 active->retire = fn ?: i915_gem_retire_noop; 450 active->retire = fn ?: i915_gem_retire_noop;
447} 451}
448 452
449static inline struct drm_i915_gem_request * 453static inline struct i915_request *
450__i915_gem_active_peek(const struct i915_gem_active *active) 454__i915_gem_active_peek(const struct i915_gem_active *active)
451{ 455{
452 /* Inside the error capture (running with the driver in an unknown 456 /*
457 * Inside the error capture (running with the driver in an unknown
453 * state), we want to bend the rules slightly (a lot). 458 * state), we want to bend the rules slightly (a lot).
454 * 459 *
455 * Work is in progress to make it safer, in the meantime this keeps 460 * Work is in progress to make it safer, in the meantime this keeps
@@ -466,7 +471,7 @@ __i915_gem_active_peek(const struct i915_gem_active *active)
466 * It does not obtain a reference on the request for the caller, so the caller 471 * It does not obtain a reference on the request for the caller, so the caller
467 * must hold struct_mutex. 472 * must hold struct_mutex.
468 */ 473 */
469static inline struct drm_i915_gem_request * 474static inline struct i915_request *
470i915_gem_active_raw(const struct i915_gem_active *active, struct mutex *mutex) 475i915_gem_active_raw(const struct i915_gem_active *active, struct mutex *mutex)
471{ 476{
472 return rcu_dereference_protected(active->request, 477 return rcu_dereference_protected(active->request,
@@ -481,13 +486,13 @@ i915_gem_active_raw(const struct i915_gem_active *active, struct mutex *mutex)
481 * still active, or NULL. It does not obtain a reference on the request 486 * still active, or NULL. It does not obtain a reference on the request
482 * for the caller, so the caller must hold struct_mutex. 487 * for the caller, so the caller must hold struct_mutex.
483 */ 488 */
484static inline struct drm_i915_gem_request * 489static inline struct i915_request *
485i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex) 490i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex)
486{ 491{
487 struct drm_i915_gem_request *request; 492 struct i915_request *request;
488 493
489 request = i915_gem_active_raw(active, mutex); 494 request = i915_gem_active_raw(active, mutex);
490 if (!request || i915_gem_request_completed(request)) 495 if (!request || i915_request_completed(request))
491 return NULL; 496 return NULL;
492 497
493 return request; 498 return request;
@@ -500,10 +505,10 @@ i915_gem_active_peek(const struct i915_gem_active *active, struct mutex *mutex)
500 * i915_gem_active_get() returns a reference to the active request, or NULL 505 * i915_gem_active_get() returns a reference to the active request, or NULL
501 * if the active tracker is idle. The caller must hold struct_mutex. 506 * if the active tracker is idle. The caller must hold struct_mutex.
502 */ 507 */
503static inline struct drm_i915_gem_request * 508static inline struct i915_request *
504i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex) 509i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex)
505{ 510{
506 return i915_gem_request_get(i915_gem_active_peek(active, mutex)); 511 return i915_request_get(i915_gem_active_peek(active, mutex));
507} 512}
508 513
509/** 514/**
@@ -514,10 +519,11 @@ i915_gem_active_get(const struct i915_gem_active *active, struct mutex *mutex)
514 * if the active tracker is idle. The caller must hold the RCU read lock, but 519 * if the active tracker is idle. The caller must hold the RCU read lock, but
515 * the returned pointer is safe to use outside of RCU. 520 * the returned pointer is safe to use outside of RCU.
516 */ 521 */
517static inline struct drm_i915_gem_request * 522static inline struct i915_request *
518__i915_gem_active_get_rcu(const struct i915_gem_active *active) 523__i915_gem_active_get_rcu(const struct i915_gem_active *active)
519{ 524{
520 /* Performing a lockless retrieval of the active request is super 525 /*
526 * Performing a lockless retrieval of the active request is super
521 * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing 527 * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing
522 * slab of request objects will not be freed whilst we hold the 528 * slab of request objects will not be freed whilst we hold the
523 * RCU read lock. It does not guarantee that the request itself 529 * RCU read lock. It does not guarantee that the request itself
@@ -525,13 +531,13 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
525 * 531 *
526 * Thread A Thread B 532 * Thread A Thread B
527 * 533 *
528 * req = active.request 534 * rq = active.request
529 * retire(req) -> free(req); 535 * retire(rq) -> free(rq);
530 * (req is now first on the slab freelist) 536 * (rq is now first on the slab freelist)
531 * active.request = NULL 537 * active.request = NULL
532 * 538 *
533 * req = new submission on a new object 539 * rq = new submission on a new object
534 * ref(req) 540 * ref(rq)
535 * 541 *
536 * To prevent the request from being reused whilst the caller 542 * To prevent the request from being reused whilst the caller
537 * uses it, we take a reference like normal. Whilst acquiring 543 * uses it, we take a reference like normal. Whilst acquiring
@@ -560,32 +566,34 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
560 * 566 *
561 * It is then imperative that we do not zero the request on 567 * It is then imperative that we do not zero the request on
562 * reallocation, so that we can chase the dangling pointers! 568 * reallocation, so that we can chase the dangling pointers!
563 * See i915_gem_request_alloc(). 569 * See i915_request_alloc().
564 */ 570 */
565 do { 571 do {
566 struct drm_i915_gem_request *request; 572 struct i915_request *request;
567 573
568 request = rcu_dereference(active->request); 574 request = rcu_dereference(active->request);
569 if (!request || i915_gem_request_completed(request)) 575 if (!request || i915_request_completed(request))
570 return NULL; 576 return NULL;
571 577
572 /* An especially silly compiler could decide to recompute the 578 /*
573 * result of i915_gem_request_completed, more specifically 579 * An especially silly compiler could decide to recompute the
580 * result of i915_request_completed, more specifically
574 * re-emit the load for request->fence.seqno. A race would catch 581 * re-emit the load for request->fence.seqno. A race would catch
575 * a later seqno value, which could flip the result from true to 582 * a later seqno value, which could flip the result from true to
576 * false. Which means part of the instructions below might not 583 * false. Which means part of the instructions below might not
577 * be executed, while later on instructions are executed. Due to 584 * be executed, while later on instructions are executed. Due to
578 * barriers within the refcounting the inconsistency can't reach 585 * barriers within the refcounting the inconsistency can't reach
579 * past the call to i915_gem_request_get_rcu, but not executing 586 * past the call to i915_request_get_rcu, but not executing
580 * that while still executing i915_gem_request_put() creates 587 * that while still executing i915_request_put() creates
581 * havoc enough. Prevent this with a compiler barrier. 588 * havoc enough. Prevent this with a compiler barrier.
582 */ 589 */
583 barrier(); 590 barrier();
584 591
585 request = i915_gem_request_get_rcu(request); 592 request = i915_request_get_rcu(request);
586 593
587 /* What stops the following rcu_access_pointer() from occurring 594 /*
588 * before the above i915_gem_request_get_rcu()? If we were 595 * What stops the following rcu_access_pointer() from occurring
596 * before the above i915_request_get_rcu()? If we were
589 * to read the value before pausing to get the reference to 597 * to read the value before pausing to get the reference to
590 * the request, we may not notice a change in the active 598 * the request, we may not notice a change in the active
591 * tracker. 599 * tracker.
@@ -599,9 +607,9 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
599 * compiler. 607 * compiler.
600 * 608 *
601 * The atomic operation at the heart of 609 * The atomic operation at the heart of
602 * i915_gem_request_get_rcu(), see dma_fence_get_rcu(), is 610 * i915_request_get_rcu(), see dma_fence_get_rcu(), is
603 * atomic_inc_not_zero() which is only a full memory barrier 611 * atomic_inc_not_zero() which is only a full memory barrier
604 * when successful. That is, if i915_gem_request_get_rcu() 612 * when successful. That is, if i915_request_get_rcu()
605 * returns the request (and so with the reference counted 613 * returns the request (and so with the reference counted
606 * incremented) then the following read for rcu_access_pointer() 614 * incremented) then the following read for rcu_access_pointer()
607 * must occur after the atomic operation and so confirm 615 * must occur after the atomic operation and so confirm
@@ -613,7 +621,7 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
613 if (!request || request == rcu_access_pointer(active->request)) 621 if (!request || request == rcu_access_pointer(active->request))
614 return rcu_pointer_handoff(request); 622 return rcu_pointer_handoff(request);
615 623
616 i915_gem_request_put(request); 624 i915_request_put(request);
617 } while (1); 625 } while (1);
618} 626}
619 627
@@ -625,12 +633,12 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
625 * or NULL if the active tracker is idle. The reference is obtained under RCU, 633 * or NULL if the active tracker is idle. The reference is obtained under RCU,
626 * so no locking is required by the caller. 634 * so no locking is required by the caller.
627 * 635 *
628 * The reference should be freed with i915_gem_request_put(). 636 * The reference should be freed with i915_request_put().
629 */ 637 */
630static inline struct drm_i915_gem_request * 638static inline struct i915_request *
631i915_gem_active_get_unlocked(const struct i915_gem_active *active) 639i915_gem_active_get_unlocked(const struct i915_gem_active *active)
632{ 640{
633 struct drm_i915_gem_request *request; 641 struct i915_request *request;
634 642
635 rcu_read_lock(); 643 rcu_read_lock();
636 request = __i915_gem_active_get_rcu(active); 644 request = __i915_gem_active_get_rcu(active);
@@ -670,7 +678,7 @@ i915_gem_active_isset(const struct i915_gem_active *active)
670 * can then wait upon the request, and afterwards release our reference, 678 * can then wait upon the request, and afterwards release our reference,
671 * free of any locking. 679 * free of any locking.
672 * 680 *
673 * This function wraps i915_wait_request(), see it for the full details on 681 * This function wraps i915_request_wait(), see it for the full details on
674 * the arguments. 682 * the arguments.
675 * 683 *
676 * Returns 0 if successful, or a negative error code. 684 * Returns 0 if successful, or a negative error code.
@@ -678,13 +686,13 @@ i915_gem_active_isset(const struct i915_gem_active *active)
678static inline int 686static inline int
679i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags) 687i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags)
680{ 688{
681 struct drm_i915_gem_request *request; 689 struct i915_request *request;
682 long ret = 0; 690 long ret = 0;
683 691
684 request = i915_gem_active_get_unlocked(active); 692 request = i915_gem_active_get_unlocked(active);
685 if (request) { 693 if (request) {
686 ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT); 694 ret = i915_request_wait(request, flags, MAX_SCHEDULE_TIMEOUT);
687 i915_gem_request_put(request); 695 i915_request_put(request);
688 } 696 }
689 697
690 return ret < 0 ? ret : 0; 698 return ret < 0 ? ret : 0;
@@ -703,14 +711,14 @@ static inline int __must_check
703i915_gem_active_retire(struct i915_gem_active *active, 711i915_gem_active_retire(struct i915_gem_active *active,
704 struct mutex *mutex) 712 struct mutex *mutex)
705{ 713{
706 struct drm_i915_gem_request *request; 714 struct i915_request *request;
707 long ret; 715 long ret;
708 716
709 request = i915_gem_active_raw(active, mutex); 717 request = i915_gem_active_raw(active, mutex);
710 if (!request) 718 if (!request)
711 return 0; 719 return 0;
712 720
713 ret = i915_wait_request(request, 721 ret = i915_request_wait(request,
714 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, 722 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
715 MAX_SCHEDULE_TIMEOUT); 723 MAX_SCHEDULE_TIMEOUT);
716 if (ret < 0) 724 if (ret < 0)
@@ -727,4 +735,4 @@ i915_gem_active_retire(struct i915_gem_active *active,
727#define for_each_active(mask, idx) \ 735#define for_each_active(mask, idx) \
728 for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) 736 for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx))
729 737
730#endif /* I915_GEM_REQUEST_H */ 738#endif /* I915_REQUEST_H */
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index e1169c02eb2b..408827bf5d96 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -586,8 +586,7 @@ TRACE_EVENT(i915_gem_evict_vm,
586); 586);
587 587
588TRACE_EVENT(i915_gem_ring_sync_to, 588TRACE_EVENT(i915_gem_ring_sync_to,
589 TP_PROTO(struct drm_i915_gem_request *to, 589 TP_PROTO(struct i915_request *to, struct i915_request *from),
590 struct drm_i915_gem_request *from),
591 TP_ARGS(to, from), 590 TP_ARGS(to, from),
592 591
593 TP_STRUCT__entry( 592 TP_STRUCT__entry(
@@ -610,9 +609,9 @@ TRACE_EVENT(i915_gem_ring_sync_to,
610 __entry->seqno) 609 __entry->seqno)
611); 610);
612 611
613TRACE_EVENT(i915_gem_request_queue, 612TRACE_EVENT(i915_request_queue,
614 TP_PROTO(struct drm_i915_gem_request *req, u32 flags), 613 TP_PROTO(struct i915_request *rq, u32 flags),
615 TP_ARGS(req, flags), 614 TP_ARGS(rq, flags),
616 615
617 TP_STRUCT__entry( 616 TP_STRUCT__entry(
618 __field(u32, dev) 617 __field(u32, dev)
@@ -624,11 +623,11 @@ TRACE_EVENT(i915_gem_request_queue,
624 ), 623 ),
625 624
626 TP_fast_assign( 625 TP_fast_assign(
627 __entry->dev = req->i915->drm.primary->index; 626 __entry->dev = rq->i915->drm.primary->index;
628 __entry->hw_id = req->ctx->hw_id; 627 __entry->hw_id = rq->ctx->hw_id;
629 __entry->ring = req->engine->id; 628 __entry->ring = rq->engine->id;
630 __entry->ctx = req->fence.context; 629 __entry->ctx = rq->fence.context;
631 __entry->seqno = req->fence.seqno; 630 __entry->seqno = rq->fence.seqno;
632 __entry->flags = flags; 631 __entry->flags = flags;
633 ), 632 ),
634 633
@@ -637,9 +636,9 @@ TRACE_EVENT(i915_gem_request_queue,
637 __entry->seqno, __entry->flags) 636 __entry->seqno, __entry->flags)
638); 637);
639 638
640DECLARE_EVENT_CLASS(i915_gem_request, 639DECLARE_EVENT_CLASS(i915_request,
641 TP_PROTO(struct drm_i915_gem_request *req), 640 TP_PROTO(struct i915_request *rq),
642 TP_ARGS(req), 641 TP_ARGS(rq),
643 642
644 TP_STRUCT__entry( 643 TP_STRUCT__entry(
645 __field(u32, dev) 644 __field(u32, dev)
@@ -651,12 +650,12 @@ DECLARE_EVENT_CLASS(i915_gem_request,
651 ), 650 ),
652 651
653 TP_fast_assign( 652 TP_fast_assign(
654 __entry->dev = req->i915->drm.primary->index; 653 __entry->dev = rq->i915->drm.primary->index;
655 __entry->hw_id = req->ctx->hw_id; 654 __entry->hw_id = rq->ctx->hw_id;
656 __entry->ring = req->engine->id; 655 __entry->ring = rq->engine->id;
657 __entry->ctx = req->fence.context; 656 __entry->ctx = rq->fence.context;
658 __entry->seqno = req->fence.seqno; 657 __entry->seqno = rq->fence.seqno;
659 __entry->global = req->global_seqno; 658 __entry->global = rq->global_seqno;
660 ), 659 ),
661 660
662 TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u", 661 TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u",
@@ -664,26 +663,25 @@ DECLARE_EVENT_CLASS(i915_gem_request,
664 __entry->seqno, __entry->global) 663 __entry->seqno, __entry->global)
665); 664);
666 665
667DEFINE_EVENT(i915_gem_request, i915_gem_request_add, 666DEFINE_EVENT(i915_request, i915_request_add,
668 TP_PROTO(struct drm_i915_gem_request *req), 667 TP_PROTO(struct i915_request *rq),
669 TP_ARGS(req) 668 TP_ARGS(rq)
670); 669);
671 670
672#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS) 671#if defined(CONFIG_DRM_I915_LOW_LEVEL_TRACEPOINTS)
673DEFINE_EVENT(i915_gem_request, i915_gem_request_submit, 672DEFINE_EVENT(i915_request, i915_request_submit,
674 TP_PROTO(struct drm_i915_gem_request *req), 673 TP_PROTO(struct i915_request *rq),
675 TP_ARGS(req) 674 TP_ARGS(rq)
676); 675);
677 676
678DEFINE_EVENT(i915_gem_request, i915_gem_request_execute, 677DEFINE_EVENT(i915_request, i915_request_execute,
679 TP_PROTO(struct drm_i915_gem_request *req), 678 TP_PROTO(struct i915_request *rq),
680 TP_ARGS(req) 679 TP_ARGS(rq)
681); 680);
682 681
683DECLARE_EVENT_CLASS(i915_gem_request_hw, 682DECLARE_EVENT_CLASS(i915_request_hw,
684 TP_PROTO(struct drm_i915_gem_request *req, 683 TP_PROTO(struct i915_request *rq, unsigned int port),
685 unsigned int port), 684 TP_ARGS(rq, port),
686 TP_ARGS(req, port),
687 685
688 TP_STRUCT__entry( 686 TP_STRUCT__entry(
689 __field(u32, dev) 687 __field(u32, dev)
@@ -696,14 +694,14 @@ DECLARE_EVENT_CLASS(i915_gem_request_hw,
696 ), 694 ),
697 695
698 TP_fast_assign( 696 TP_fast_assign(
699 __entry->dev = req->i915->drm.primary->index; 697 __entry->dev = rq->i915->drm.primary->index;
700 __entry->hw_id = req->ctx->hw_id; 698 __entry->hw_id = rq->ctx->hw_id;
701 __entry->ring = req->engine->id; 699 __entry->ring = rq->engine->id;
702 __entry->ctx = req->fence.context; 700 __entry->ctx = rq->fence.context;
703 __entry->seqno = req->fence.seqno; 701 __entry->seqno = rq->fence.seqno;
704 __entry->global_seqno = req->global_seqno; 702 __entry->global_seqno = rq->global_seqno;
705 __entry->port = port; 703 __entry->port = port;
706 ), 704 ),
707 705
708 TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, port=%u", 706 TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, port=%u",
709 __entry->dev, __entry->hw_id, __entry->ring, 707 __entry->dev, __entry->hw_id, __entry->ring,
@@ -711,34 +709,34 @@ DECLARE_EVENT_CLASS(i915_gem_request_hw,
711 __entry->global_seqno, __entry->port) 709 __entry->global_seqno, __entry->port)
712); 710);
713 711
714DEFINE_EVENT(i915_gem_request_hw, i915_gem_request_in, 712DEFINE_EVENT(i915_request_hw, i915_request_in,
715 TP_PROTO(struct drm_i915_gem_request *req, unsigned int port), 713 TP_PROTO(struct i915_request *rq, unsigned int port),
716 TP_ARGS(req, port) 714 TP_ARGS(rq, port)
717); 715);
718 716
719DEFINE_EVENT(i915_gem_request, i915_gem_request_out, 717DEFINE_EVENT(i915_request, i915_request_out,
720 TP_PROTO(struct drm_i915_gem_request *req), 718 TP_PROTO(struct i915_request *rq),
721 TP_ARGS(req) 719 TP_ARGS(rq)
722); 720);
723#else 721#else
724#if !defined(TRACE_HEADER_MULTI_READ) 722#if !defined(TRACE_HEADER_MULTI_READ)
725static inline void 723static inline void
726trace_i915_gem_request_submit(struct drm_i915_gem_request *req) 724trace_i915_request_submit(struct i915_request *rq)
727{ 725{
728} 726}
729 727
730static inline void 728static inline void
731trace_i915_gem_request_execute(struct drm_i915_gem_request *req) 729trace_i915_request_execute(struct i915_request *rq)
732{ 730{
733} 731}
734 732
735static inline void 733static inline void
736trace_i915_gem_request_in(struct drm_i915_gem_request *req, unsigned int port) 734trace_i915_request_in(struct i915_request *rq, unsigned int port)
737{ 735{
738} 736}
739 737
740static inline void 738static inline void
741trace_i915_gem_request_out(struct drm_i915_gem_request *req) 739trace_i915_request_out(struct i915_request *rq)
742{ 740{
743} 741}
744#endif 742#endif
@@ -767,14 +765,14 @@ TRACE_EVENT(intel_engine_notify,
767 __entry->waiters) 765 __entry->waiters)
768); 766);
769 767
770DEFINE_EVENT(i915_gem_request, i915_gem_request_retire, 768DEFINE_EVENT(i915_request, i915_request_retire,
771 TP_PROTO(struct drm_i915_gem_request *req), 769 TP_PROTO(struct i915_request *rq),
772 TP_ARGS(req) 770 TP_ARGS(rq)
773); 771);
774 772
775TRACE_EVENT(i915_gem_request_wait_begin, 773TRACE_EVENT(i915_request_wait_begin,
776 TP_PROTO(struct drm_i915_gem_request *req, unsigned int flags), 774 TP_PROTO(struct i915_request *rq, unsigned int flags),
777 TP_ARGS(req, flags), 775 TP_ARGS(rq, flags),
778 776
779 TP_STRUCT__entry( 777 TP_STRUCT__entry(
780 __field(u32, dev) 778 __field(u32, dev)
@@ -793,12 +791,12 @@ TRACE_EVENT(i915_gem_request_wait_begin,
793 * less desirable. 791 * less desirable.
794 */ 792 */
795 TP_fast_assign( 793 TP_fast_assign(
796 __entry->dev = req->i915->drm.primary->index; 794 __entry->dev = rq->i915->drm.primary->index;
797 __entry->hw_id = req->ctx->hw_id; 795 __entry->hw_id = rq->ctx->hw_id;
798 __entry->ring = req->engine->id; 796 __entry->ring = rq->engine->id;
799 __entry->ctx = req->fence.context; 797 __entry->ctx = rq->fence.context;
800 __entry->seqno = req->fence.seqno; 798 __entry->seqno = rq->fence.seqno;
801 __entry->global = req->global_seqno; 799 __entry->global = rq->global_seqno;
802 __entry->flags = flags; 800 __entry->flags = flags;
803 ), 801 ),
804 802
@@ -808,9 +806,9 @@ TRACE_EVENT(i915_gem_request_wait_begin,
808 !!(__entry->flags & I915_WAIT_LOCKED), __entry->flags) 806 !!(__entry->flags & I915_WAIT_LOCKED), __entry->flags)
809); 807);
810 808
811DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end, 809DEFINE_EVENT(i915_request, i915_request_wait_end,
812 TP_PROTO(struct drm_i915_gem_request *req), 810 TP_PROTO(struct i915_request *rq),
813 TP_ARGS(req) 811 TP_ARGS(rq)
814); 812);
815 813
816TRACE_EVENT(i915_flip_request, 814TRACE_EVENT(i915_flip_request,
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index e0e7c48f45dc..4bda3bd29bf5 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -31,8 +31,7 @@
31#include <drm/drm_gem.h> 31#include <drm/drm_gem.h>
32 32
33static void 33static void
34i915_vma_retire(struct i915_gem_active *active, 34i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
35 struct drm_i915_gem_request *rq)
36{ 35{
37 const unsigned int idx = rq->engine->id; 36 const unsigned int idx = rq->engine->id;
38 struct i915_vma *vma = 37 struct i915_vma *vma =
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index fd5b84904f7c..8c5022095418 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -32,8 +32,8 @@
32#include "i915_gem_gtt.h" 32#include "i915_gem_gtt.h"
33#include "i915_gem_fence_reg.h" 33#include "i915_gem_fence_reg.h"
34#include "i915_gem_object.h" 34#include "i915_gem_object.h"
35#include "i915_gem_request.h"
36 35
36#include "i915_request.h"
37 37
38enum i915_cache_level; 38enum i915_cache_level;
39 39
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index b955f7d7bd0f..1f79e7a47433 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -168,17 +168,21 @@ static void irq_enable(struct intel_engine_cs *engine)
168 set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted); 168 set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
169 169
170 /* Caller disables interrupts */ 170 /* Caller disables interrupts */
171 spin_lock(&engine->i915->irq_lock); 171 if (engine->irq_enable) {
172 engine->irq_enable(engine); 172 spin_lock(&engine->i915->irq_lock);
173 spin_unlock(&engine->i915->irq_lock); 173 engine->irq_enable(engine);
174 spin_unlock(&engine->i915->irq_lock);
175 }
174} 176}
175 177
176static void irq_disable(struct intel_engine_cs *engine) 178static void irq_disable(struct intel_engine_cs *engine)
177{ 179{
178 /* Caller disables interrupts */ 180 /* Caller disables interrupts */
179 spin_lock(&engine->i915->irq_lock); 181 if (engine->irq_disable) {
180 engine->irq_disable(engine); 182 spin_lock(&engine->i915->irq_lock);
181 spin_unlock(&engine->i915->irq_lock); 183 engine->irq_disable(engine);
184 spin_unlock(&engine->i915->irq_lock);
185 }
182} 186}
183 187
184void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 188void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
@@ -243,6 +247,8 @@ void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
243 spin_unlock(&b->irq_lock); 247 spin_unlock(&b->irq_lock);
244 248
245 rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) { 249 rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {
250 GEM_BUG_ON(!i915_seqno_passed(intel_engine_get_seqno(engine),
251 wait->seqno));
246 RB_CLEAR_NODE(&wait->node); 252 RB_CLEAR_NODE(&wait->node);
247 wake_up_process(wait->tsk); 253 wake_up_process(wait->tsk);
248 } 254 }
@@ -336,7 +342,8 @@ static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,
336 lockdep_assert_held(&b->rb_lock); 342 lockdep_assert_held(&b->rb_lock);
337 GEM_BUG_ON(b->irq_wait == wait); 343 GEM_BUG_ON(b->irq_wait == wait);
338 344
339 /* This request is completed, so remove it from the tree, mark it as 345 /*
346 * This request is completed, so remove it from the tree, mark it as
340 * complete, and *then* wake up the associated task. N.B. when the 347 * complete, and *then* wake up the associated task. N.B. when the
341 * task wakes up, it will find the empty rb_node, discern that it 348 * task wakes up, it will find the empty rb_node, discern that it
342 * has already been removed from the tree and skip the serialisation 349 * has already been removed from the tree and skip the serialisation
@@ -347,7 +354,8 @@ static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,
347 rb_erase(&wait->node, &b->waiters); 354 rb_erase(&wait->node, &b->waiters);
348 RB_CLEAR_NODE(&wait->node); 355 RB_CLEAR_NODE(&wait->node);
349 356
350 wake_up_process(wait->tsk); /* implicit smp_wmb() */ 357 if (wait->tsk->state != TASK_RUNNING)
358 wake_up_process(wait->tsk); /* implicit smp_wmb() */
351} 359}
352 360
353static inline void __intel_breadcrumbs_next(struct intel_engine_cs *engine, 361static inline void __intel_breadcrumbs_next(struct intel_engine_cs *engine,
@@ -588,23 +596,6 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
588 spin_unlock_irq(&b->rb_lock); 596 spin_unlock_irq(&b->rb_lock);
589} 597}
590 598
591static bool signal_complete(const struct drm_i915_gem_request *request)
592{
593 if (!request)
594 return false;
595
596 /*
597 * Carefully check if the request is complete, giving time for the
598 * seqno to be visible or if the GPU hung.
599 */
600 return __i915_request_irq_complete(request);
601}
602
603static struct drm_i915_gem_request *to_signaler(struct rb_node *rb)
604{
605 return rb_entry(rb, struct drm_i915_gem_request, signaling.node);
606}
607
608static void signaler_set_rtpriority(void) 599static void signaler_set_rtpriority(void)
609{ 600{
610 struct sched_param param = { .sched_priority = 1 }; 601 struct sched_param param = { .sched_priority = 1 };
@@ -612,78 +603,26 @@ static void signaler_set_rtpriority(void)
612 sched_setscheduler_nocheck(current, SCHED_FIFO, &param); 603 sched_setscheduler_nocheck(current, SCHED_FIFO, &param);
613} 604}
614 605
615static void __intel_engine_remove_signal(struct intel_engine_cs *engine,
616 struct drm_i915_gem_request *request)
617{
618 struct intel_breadcrumbs *b = &engine->breadcrumbs;
619
620 lockdep_assert_held(&b->rb_lock);
621
622 /*
623 * Wake up all other completed waiters and select the
624 * next bottom-half for the next user interrupt.
625 */
626 __intel_engine_remove_wait(engine, &request->signaling.wait);
627
628 /*
629 * Find the next oldest signal. Note that as we have
630 * not been holding the lock, another client may
631 * have installed an even older signal than the one
632 * we just completed - so double check we are still
633 * the oldest before picking the next one.
634 */
635 if (request->signaling.wait.seqno) {
636 if (request == rcu_access_pointer(b->first_signal)) {
637 struct rb_node *rb = rb_next(&request->signaling.node);
638 rcu_assign_pointer(b->first_signal,
639 rb ? to_signaler(rb) : NULL);
640 }
641
642 rb_erase(&request->signaling.node, &b->signals);
643 request->signaling.wait.seqno = 0;
644 }
645}
646
647static struct drm_i915_gem_request *
648get_first_signal_rcu(struct intel_breadcrumbs *b)
649{
650 /*
651 * See the big warnings for i915_gem_active_get_rcu() and similarly
652 * for dma_fence_get_rcu_safe() that explain the intricacies involved
653 * here with defeating CPU/compiler speculation and enforcing
654 * the required memory barriers.
655 */
656 do {
657 struct drm_i915_gem_request *request;
658
659 request = rcu_dereference(b->first_signal);
660 if (request)
661 request = i915_gem_request_get_rcu(request);
662
663 barrier();
664
665 if (!request || request == rcu_access_pointer(b->first_signal))
666 return rcu_pointer_handoff(request);
667
668 i915_gem_request_put(request);
669 } while (1);
670}
671
672static int intel_breadcrumbs_signaler(void *arg) 606static int intel_breadcrumbs_signaler(void *arg)
673{ 607{
674 struct intel_engine_cs *engine = arg; 608 struct intel_engine_cs *engine = arg;
675 struct intel_breadcrumbs *b = &engine->breadcrumbs; 609 struct intel_breadcrumbs *b = &engine->breadcrumbs;
676 struct drm_i915_gem_request *request; 610 struct i915_request *rq, *n;
677 611
678 /* Install ourselves with high priority to reduce signalling latency */ 612 /* Install ourselves with high priority to reduce signalling latency */
679 signaler_set_rtpriority(); 613 signaler_set_rtpriority();
680 614
681 do { 615 do {
682 bool do_schedule = true; 616 bool do_schedule = true;
617 LIST_HEAD(list);
618 u32 seqno;
683 619
684 set_current_state(TASK_INTERRUPTIBLE); 620 set_current_state(TASK_INTERRUPTIBLE);
621 if (list_empty(&b->signals))
622 goto sleep;
685 623
686 /* We are either woken up by the interrupt bottom-half, 624 /*
625 * We are either woken up by the interrupt bottom-half,
687 * or by a client adding a new signaller. In both cases, 626 * or by a client adding a new signaller. In both cases,
688 * the GPU seqno may have advanced beyond our oldest signal. 627 * the GPU seqno may have advanced beyond our oldest signal.
689 * If it has, propagate the signal, remove the waiter and 628 * If it has, propagate the signal, remove the waiter and
@@ -691,25 +630,45 @@ static int intel_breadcrumbs_signaler(void *arg)
691 * need to wait for a new interrupt from the GPU or for 630 * need to wait for a new interrupt from the GPU or for
692 * a new client. 631 * a new client.
693 */ 632 */
694 rcu_read_lock(); 633 seqno = intel_engine_get_seqno(engine);
695 request = get_first_signal_rcu(b); 634
696 rcu_read_unlock(); 635 spin_lock_irq(&b->rb_lock);
697 if (signal_complete(request)) { 636 list_for_each_entry_safe(rq, n, &b->signals, signaling.link) {
698 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 637 u32 this = rq->signaling.wait.seqno;
699 &request->fence.flags)) { 638
700 local_bh_disable(); 639 GEM_BUG_ON(!rq->signaling.wait.seqno);
701 dma_fence_signal(&request->fence); 640
702 GEM_BUG_ON(!i915_gem_request_completed(request)); 641 if (!i915_seqno_passed(seqno, this))
703 local_bh_enable(); /* kick start the tasklets */ 642 break;
704 }
705 643
706 if (READ_ONCE(request->signaling.wait.seqno)) { 644 if (likely(this == i915_request_global_seqno(rq))) {
707 spin_lock_irq(&b->rb_lock); 645 __intel_engine_remove_wait(engine,
708 __intel_engine_remove_signal(engine, request); 646 &rq->signaling.wait);
709 spin_unlock_irq(&b->rb_lock); 647
648 rq->signaling.wait.seqno = 0;
649 __list_del_entry(&rq->signaling.link);
650
651 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
652 &rq->fence.flags)) {
653 list_add_tail(&rq->signaling.link,
654 &list);
655 i915_request_get(rq);
656 }
657 }
658 }
659 spin_unlock_irq(&b->rb_lock);
660
661 if (!list_empty(&list)) {
662 local_bh_disable();
663 list_for_each_entry_safe(rq, n, &list, signaling.link) {
664 dma_fence_signal(&rq->fence);
665 GEM_BUG_ON(!i915_request_completed(rq));
666 i915_request_put(rq);
710 } 667 }
668 local_bh_enable(); /* kick start the tasklets */
711 669
712 /* If the engine is saturated we may be continually 670 /*
671 * If the engine is saturated we may be continually
713 * processing completed requests. This angers the 672 * processing completed requests. This angers the
714 * NMI watchdog if we never let anything else 673 * NMI watchdog if we never let anything else
715 * have access to the CPU. Let's pretend to be nice 674 * have access to the CPU. Let's pretend to be nice
@@ -718,9 +677,19 @@ static int intel_breadcrumbs_signaler(void *arg)
718 */ 677 */
719 do_schedule = need_resched(); 678 do_schedule = need_resched();
720 } 679 }
721 i915_gem_request_put(request);
722 680
723 if (unlikely(do_schedule)) { 681 if (unlikely(do_schedule)) {
682 /* Before we sleep, check for a missed seqno */
683 if (current->state & TASK_NORMAL &&
684 !list_empty(&b->signals) &&
685 engine->irq_seqno_barrier &&
686 test_and_clear_bit(ENGINE_IRQ_BREADCRUMB,
687 &engine->irq_posted)) {
688 engine->irq_seqno_barrier(engine);
689 intel_engine_wakeup(engine);
690 }
691
692sleep:
724 if (kthread_should_park()) 693 if (kthread_should_park())
725 kthread_parkme(); 694 kthread_parkme();
726 695
@@ -735,14 +704,40 @@ static int intel_breadcrumbs_signaler(void *arg)
735 return 0; 704 return 0;
736} 705}
737 706
738void intel_engine_enable_signaling(struct drm_i915_gem_request *request, 707static void insert_signal(struct intel_breadcrumbs *b,
739 bool wakeup) 708 struct i915_request *request,
709 const u32 seqno)
710{
711 struct i915_request *iter;
712
713 lockdep_assert_held(&b->rb_lock);
714
715 /*
716 * A reasonable assumption is that we are called to add signals
717 * in sequence, as the requests are submitted for execution and
718 * assigned a global_seqno. This will be the case for the majority
719 * of internally generated signals (inter-engine signaling).
720 *
721 * Out of order waiters triggering random signaling enabling will
722 * be more problematic, but hopefully rare enough and the list
723 * small enough that the O(N) insertion sort is not an issue.
724 */
725
726 list_for_each_entry_reverse(iter, &b->signals, signaling.link)
727 if (i915_seqno_passed(seqno, iter->signaling.wait.seqno))
728 break;
729
730 list_add(&request->signaling.link, &iter->signaling.link);
731}
732
733void intel_engine_enable_signaling(struct i915_request *request, bool wakeup)
740{ 734{
741 struct intel_engine_cs *engine = request->engine; 735 struct intel_engine_cs *engine = request->engine;
742 struct intel_breadcrumbs *b = &engine->breadcrumbs; 736 struct intel_breadcrumbs *b = &engine->breadcrumbs;
743 u32 seqno; 737 u32 seqno;
744 738
745 /* Note that we may be called from an interrupt handler on another 739 /*
740 * Note that we may be called from an interrupt handler on another
746 * device (e.g. nouveau signaling a fence completion causing us 741 * device (e.g. nouveau signaling a fence completion causing us
747 * to submit a request, and so enable signaling). As such, 742 * to submit a request, and so enable signaling). As such,
748 * we need to make sure that all other users of b->rb_lock protect 743 * we need to make sure that all other users of b->rb_lock protect
@@ -753,18 +748,17 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
753 GEM_BUG_ON(!irqs_disabled()); 748 GEM_BUG_ON(!irqs_disabled());
754 lockdep_assert_held(&request->lock); 749 lockdep_assert_held(&request->lock);
755 750
756 seqno = i915_gem_request_global_seqno(request); 751 seqno = i915_request_global_seqno(request);
757 if (!seqno) 752 if (!seqno) /* will be enabled later upon execution */
758 return; 753 return;
759 754
760 spin_lock(&b->rb_lock);
761
762 GEM_BUG_ON(request->signaling.wait.seqno); 755 GEM_BUG_ON(request->signaling.wait.seqno);
763 request->signaling.wait.tsk = b->signaler; 756 request->signaling.wait.tsk = b->signaler;
764 request->signaling.wait.request = request; 757 request->signaling.wait.request = request;
765 request->signaling.wait.seqno = seqno; 758 request->signaling.wait.seqno = seqno;
766 759
767 /* First add ourselves into the list of waiters, but register our 760 /*
761 * Add ourselves into the list of waiters, but registering our
768 * bottom-half as the signaller thread. As per usual, only the oldest 762 * bottom-half as the signaller thread. As per usual, only the oldest
769 * waiter (not just signaller) is tasked as the bottom-half waking 763 * waiter (not just signaller) is tasked as the bottom-half waking
770 * up all completed waiters after the user interrupt. 764 * up all completed waiters after the user interrupt.
@@ -772,58 +766,31 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
772 * If we are the oldest waiter, enable the irq (after which we 766 * If we are the oldest waiter, enable the irq (after which we
773 * must double check that the seqno did not complete). 767 * must double check that the seqno did not complete).
774 */ 768 */
769 spin_lock(&b->rb_lock);
770 insert_signal(b, request, seqno);
775 wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait); 771 wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait);
776
777 if (!__i915_gem_request_completed(request, seqno)) {
778 struct rb_node *parent, **p;
779 bool first;
780
781 /* Now insert ourselves into the retirement ordered list of
782 * signals on this engine. We track the oldest seqno as that
783 * will be the first signal to complete.
784 */
785 parent = NULL;
786 first = true;
787 p = &b->signals.rb_node;
788 while (*p) {
789 parent = *p;
790 if (i915_seqno_passed(seqno,
791 to_signaler(parent)->signaling.wait.seqno)) {
792 p = &parent->rb_right;
793 first = false;
794 } else {
795 p = &parent->rb_left;
796 }
797 }
798 rb_link_node(&request->signaling.node, parent, p);
799 rb_insert_color(&request->signaling.node, &b->signals);
800 if (first)
801 rcu_assign_pointer(b->first_signal, request);
802 } else {
803 __intel_engine_remove_wait(engine, &request->signaling.wait);
804 request->signaling.wait.seqno = 0;
805 wakeup = false;
806 }
807
808 spin_unlock(&b->rb_lock); 772 spin_unlock(&b->rb_lock);
809 773
810 if (wakeup) 774 if (wakeup)
811 wake_up_process(b->signaler); 775 wake_up_process(b->signaler);
812} 776}
813 777
814void intel_engine_cancel_signaling(struct drm_i915_gem_request *request) 778void intel_engine_cancel_signaling(struct i915_request *request)
815{ 779{
780 struct intel_engine_cs *engine = request->engine;
781 struct intel_breadcrumbs *b = &engine->breadcrumbs;
782
816 GEM_BUG_ON(!irqs_disabled()); 783 GEM_BUG_ON(!irqs_disabled());
817 lockdep_assert_held(&request->lock); 784 lockdep_assert_held(&request->lock);
818 785
819 if (READ_ONCE(request->signaling.wait.seqno)) { 786 if (!READ_ONCE(request->signaling.wait.seqno))
820 struct intel_engine_cs *engine = request->engine; 787 return;
821 struct intel_breadcrumbs *b = &engine->breadcrumbs;
822 788
823 spin_lock(&b->rb_lock); 789 spin_lock(&b->rb_lock);
824 __intel_engine_remove_signal(engine, request); 790 __intel_engine_remove_wait(engine, &request->signaling.wait);
825 spin_unlock(&b->rb_lock); 791 if (fetch_and_zero(&request->signaling.wait.seqno))
826 } 792 __list_del_entry(&request->signaling.link);
793 spin_unlock(&b->rb_lock);
827} 794}
828 795
829int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 796int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
@@ -837,6 +804,8 @@ int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
837 timer_setup(&b->fake_irq, intel_breadcrumbs_fake_irq, 0); 804 timer_setup(&b->fake_irq, intel_breadcrumbs_fake_irq, 0);
838 timer_setup(&b->hangcheck, intel_breadcrumbs_hangcheck, 0); 805 timer_setup(&b->hangcheck, intel_breadcrumbs_hangcheck, 0);
839 806
807 INIT_LIST_HEAD(&b->signals);
808
840 /* Spawn a thread to provide a common bottom-half for all signals. 809 /* Spawn a thread to provide a common bottom-half for all signals.
841 * As this is an asynchronous interface we cannot steal the current 810 * As this is an asynchronous interface we cannot steal the current
842 * task for handling the bottom-half to the user interrupt, therefore 811 * task for handling the bottom-half to the user interrupt, therefore
@@ -896,8 +865,7 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
896 /* The engines should be idle and all requests accounted for! */ 865 /* The engines should be idle and all requests accounted for! */
897 WARN_ON(READ_ONCE(b->irq_wait)); 866 WARN_ON(READ_ONCE(b->irq_wait));
898 WARN_ON(!RB_EMPTY_ROOT(&b->waiters)); 867 WARN_ON(!RB_EMPTY_ROOT(&b->waiters));
899 WARN_ON(rcu_access_pointer(b->first_signal)); 868 WARN_ON(!list_empty(&b->signals));
900 WARN_ON(!RB_EMPTY_ROOT(&b->signals));
901 869
902 if (!IS_ERR_OR_NULL(b->signaler)) 870 if (!IS_ERR_OR_NULL(b->signaler))
903 kthread_stop(b->signaler); 871 kthread_stop(b->signaler);
@@ -905,28 +873,6 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
905 cancel_fake_irq(engine); 873 cancel_fake_irq(engine);
906} 874}
907 875
908bool intel_breadcrumbs_busy(struct intel_engine_cs *engine)
909{
910 struct intel_breadcrumbs *b = &engine->breadcrumbs;
911 bool busy = false;
912
913 spin_lock_irq(&b->rb_lock);
914
915 if (b->irq_wait) {
916 wake_up_process(b->irq_wait->tsk);
917 busy = true;
918 }
919
920 if (rcu_access_pointer(b->first_signal)) {
921 wake_up_process(b->signaler);
922 busy = true;
923 }
924
925 spin_unlock_irq(&b->rb_lock);
926
927 return busy;
928}
929
930#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 876#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
931#include "selftests/intel_breadcrumbs.c" 877#include "selftests/intel_breadcrumbs.c"
932#endif 878#endif
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index aa66e952a95d..89ab0f70aa22 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -66,13 +66,13 @@
66 * of the CTM coefficient and we write the value from bit 3. We also round the 66 * of the CTM coefficient and we write the value from bit 3. We also round the
67 * value. 67 * value.
68 */ 68 */
69#define I9XX_CSC_COEFF_FP(coeff, fbits) \ 69#define ILK_CSC_COEFF_FP(coeff, fbits) \
70 (clamp_val(((coeff) >> (32 - (fbits) - 3)) + 4, 0, 0xfff) & 0xff8) 70 (clamp_val(((coeff) >> (32 - (fbits) - 3)) + 4, 0, 0xfff) & 0xff8)
71 71
72#define I9XX_CSC_COEFF_LIMITED_RANGE \ 72#define ILK_CSC_COEFF_LIMITED_RANGE \
73 I9XX_CSC_COEFF_FP(CTM_COEFF_LIMITED_RANGE, 9) 73 ILK_CSC_COEFF_FP(CTM_COEFF_LIMITED_RANGE, 9)
74#define I9XX_CSC_COEFF_1_0 \ 74#define ILK_CSC_COEFF_1_0 \
75 ((7 << 12) | I9XX_CSC_COEFF_FP(CTM_COEFF_1_0, 8)) 75 ((7 << 12) | ILK_CSC_COEFF_FP(CTM_COEFF_1_0, 8))
76 76
77static bool crtc_state_is_legacy_gamma(struct drm_crtc_state *state) 77static bool crtc_state_is_legacy_gamma(struct drm_crtc_state *state)
78{ 78{
@@ -84,30 +84,31 @@ static bool crtc_state_is_legacy_gamma(struct drm_crtc_state *state)
84 84
85/* 85/*
86 * When using limited range, multiply the matrix given by userspace by 86 * When using limited range, multiply the matrix given by userspace by
87 * the matrix that we would use for the limited range. We do the 87 * the matrix that we would use for the limited range.
88 * multiplication in U2.30 format.
89 */ 88 */
90static void ctm_mult_by_limited(uint64_t *result, int64_t *input) 89static u64 *ctm_mult_by_limited(u64 *result, const u64 *input)
91{ 90{
92 int i; 91 int i;
93 92
94 for (i = 0; i < 9; i++) 93 for (i = 0; i < 9; i++) {
95 result[i] = 0; 94 u64 user_coeff = input[i];
95 u32 limited_coeff = CTM_COEFF_LIMITED_RANGE;
96 u32 abs_coeff = clamp_val(CTM_COEFF_ABS(user_coeff), 0,
97 CTM_COEFF_4_0 - 1) >> 2;
96 98
97 for (i = 0; i < 3; i++) { 99 /*
98 int64_t user_coeff = input[i * 3 + i]; 100 * By scaling every co-efficient with limited range (16-235)
99 uint64_t limited_coeff = CTM_COEFF_LIMITED_RANGE >> 2; 101 * vs full range (0-255) the final o/p will be scaled down to
100 uint64_t abs_coeff = clamp_val(CTM_COEFF_ABS(user_coeff), 102 * fit in the limited range supported by the panel.
101 0, 103 */
102 CTM_COEFF_4_0 - 1) >> 2; 104 result[i] = mul_u32_u32(limited_coeff, abs_coeff) >> 30;
103 105 result[i] |= user_coeff & CTM_COEFF_SIGN;
104 result[i * 3 + i] = (limited_coeff * abs_coeff) >> 27;
105 if (CTM_COEFF_NEGATIVE(user_coeff))
106 result[i * 3 + i] |= CTM_COEFF_SIGN;
107 } 106 }
107
108 return result;
108} 109}
109 110
110static void i9xx_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc) 111static void ilk_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc)
111{ 112{
112 int pipe = intel_crtc->pipe; 113 int pipe = intel_crtc->pipe;
113 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); 114 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
@@ -131,8 +132,7 @@ static void i9xx_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc)
131 I915_WRITE(PIPE_CSC_MODE(pipe), 0); 132 I915_WRITE(PIPE_CSC_MODE(pipe), 0);
132} 133}
133 134
134/* Set up the pipe CSC unit. */ 135static void ilk_load_csc_matrix(struct drm_crtc_state *crtc_state)
135static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
136{ 136{
137 struct drm_crtc *crtc = crtc_state->crtc; 137 struct drm_crtc *crtc = crtc_state->crtc;
138 struct drm_i915_private *dev_priv = to_i915(crtc->dev); 138 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
@@ -140,21 +140,28 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
140 int i, pipe = intel_crtc->pipe; 140 int i, pipe = intel_crtc->pipe;
141 uint16_t coeffs[9] = { 0, }; 141 uint16_t coeffs[9] = { 0, };
142 struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state); 142 struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state);
143 bool limited_color_range = false;
144
145 /*
146 * FIXME if there's a gamma LUT after the CSC, we should
147 * do the range compression using the gamma LUT instead.
148 */
149 if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv))
150 limited_color_range = intel_crtc_state->limited_color_range;
143 151
144 if (intel_crtc_state->ycbcr420) { 152 if (intel_crtc_state->ycbcr420) {
145 i9xx_load_ycbcr_conversion_matrix(intel_crtc); 153 ilk_load_ycbcr_conversion_matrix(intel_crtc);
146 return; 154 return;
147 } else if (crtc_state->ctm) { 155 } else if (crtc_state->ctm) {
148 struct drm_color_ctm *ctm = 156 struct drm_color_ctm *ctm =
149 (struct drm_color_ctm *)crtc_state->ctm->data; 157 (struct drm_color_ctm *)crtc_state->ctm->data;
150 uint64_t input[9] = { 0, }; 158 const u64 *input;
159 u64 temp[9];
151 160
152 if (intel_crtc_state->limited_color_range) { 161 if (limited_color_range)
153 ctm_mult_by_limited(input, ctm->matrix); 162 input = ctm_mult_by_limited(temp, ctm->matrix);
154 } else { 163 else
155 for (i = 0; i < ARRAY_SIZE(input); i++) 164 input = ctm->matrix;
156 input[i] = ctm->matrix[i];
157 }
158 165
159 /* 166 /*
160 * Convert fixed point S31.32 input to format supported by the 167 * Convert fixed point S31.32 input to format supported by the
@@ -175,21 +182,21 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
175 182
176 if (abs_coeff < CTM_COEFF_0_125) 183 if (abs_coeff < CTM_COEFF_0_125)
177 coeffs[i] |= (3 << 12) | 184 coeffs[i] |= (3 << 12) |
178 I9XX_CSC_COEFF_FP(abs_coeff, 12); 185 ILK_CSC_COEFF_FP(abs_coeff, 12);
179 else if (abs_coeff < CTM_COEFF_0_25) 186 else if (abs_coeff < CTM_COEFF_0_25)
180 coeffs[i] |= (2 << 12) | 187 coeffs[i] |= (2 << 12) |
181 I9XX_CSC_COEFF_FP(abs_coeff, 11); 188 ILK_CSC_COEFF_FP(abs_coeff, 11);
182 else if (abs_coeff < CTM_COEFF_0_5) 189 else if (abs_coeff < CTM_COEFF_0_5)
183 coeffs[i] |= (1 << 12) | 190 coeffs[i] |= (1 << 12) |
184 I9XX_CSC_COEFF_FP(abs_coeff, 10); 191 ILK_CSC_COEFF_FP(abs_coeff, 10);
185 else if (abs_coeff < CTM_COEFF_1_0) 192 else if (abs_coeff < CTM_COEFF_1_0)
186 coeffs[i] |= I9XX_CSC_COEFF_FP(abs_coeff, 9); 193 coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9);
187 else if (abs_coeff < CTM_COEFF_2_0) 194 else if (abs_coeff < CTM_COEFF_2_0)
188 coeffs[i] |= (7 << 12) | 195 coeffs[i] |= (7 << 12) |
189 I9XX_CSC_COEFF_FP(abs_coeff, 8); 196 ILK_CSC_COEFF_FP(abs_coeff, 8);
190 else 197 else
191 coeffs[i] |= (6 << 12) | 198 coeffs[i] |= (6 << 12) |
192 I9XX_CSC_COEFF_FP(abs_coeff, 7); 199 ILK_CSC_COEFF_FP(abs_coeff, 7);
193 } 200 }
194 } else { 201 } else {
195 /* 202 /*
@@ -201,11 +208,11 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
201 * into consideration. 208 * into consideration.
202 */ 209 */
203 for (i = 0; i < 3; i++) { 210 for (i = 0; i < 3; i++) {
204 if (intel_crtc_state->limited_color_range) 211 if (limited_color_range)
205 coeffs[i * 3 + i] = 212 coeffs[i * 3 + i] =
206 I9XX_CSC_COEFF_LIMITED_RANGE; 213 ILK_CSC_COEFF_LIMITED_RANGE;
207 else 214 else
208 coeffs[i * 3 + i] = I9XX_CSC_COEFF_1_0; 215 coeffs[i * 3 + i] = ILK_CSC_COEFF_1_0;
209 } 216 }
210 } 217 }
211 218
@@ -225,7 +232,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
225 if (INTEL_GEN(dev_priv) > 6) { 232 if (INTEL_GEN(dev_priv) > 6) {
226 uint16_t postoff = 0; 233 uint16_t postoff = 0;
227 234
228 if (intel_crtc_state->limited_color_range) 235 if (limited_color_range)
229 postoff = (16 * (1 << 12) / 255) & 0x1fff; 236 postoff = (16 * (1 << 12) / 255) & 0x1fff;
230 237
231 I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff); 238 I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
@@ -236,7 +243,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
236 } else { 243 } else {
237 uint32_t mode = CSC_MODE_YUV_TO_RGB; 244 uint32_t mode = CSC_MODE_YUV_TO_RGB;
238 245
239 if (intel_crtc_state->limited_color_range) 246 if (limited_color_range)
240 mode |= CSC_BLACK_SCREEN_OFFSET; 247 mode |= CSC_BLACK_SCREEN_OFFSET;
241 248
242 I915_WRITE(PIPE_CSC_MODE(pipe), mode); 249 I915_WRITE(PIPE_CSC_MODE(pipe), mode);
@@ -651,14 +658,14 @@ void intel_color_init(struct drm_crtc *crtc)
651 dev_priv->display.load_csc_matrix = cherryview_load_csc_matrix; 658 dev_priv->display.load_csc_matrix = cherryview_load_csc_matrix;
652 dev_priv->display.load_luts = cherryview_load_luts; 659 dev_priv->display.load_luts = cherryview_load_luts;
653 } else if (IS_HASWELL(dev_priv)) { 660 } else if (IS_HASWELL(dev_priv)) {
654 dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix; 661 dev_priv->display.load_csc_matrix = ilk_load_csc_matrix;
655 dev_priv->display.load_luts = haswell_load_luts; 662 dev_priv->display.load_luts = haswell_load_luts;
656 } else if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv) || 663 } else if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv) ||
657 IS_BROXTON(dev_priv)) { 664 IS_BROXTON(dev_priv)) {
658 dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix; 665 dev_priv->display.load_csc_matrix = ilk_load_csc_matrix;
659 dev_priv->display.load_luts = broadwell_load_luts; 666 dev_priv->display.load_luts = broadwell_load_luts;
660 } else if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) { 667 } else if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
661 dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix; 668 dev_priv->display.load_csc_matrix = ilk_load_csc_matrix;
662 dev_priv->display.load_luts = glk_load_luts; 669 dev_priv->display.load_luts = glk_load_luts;
663 } else { 670 } else {
664 dev_priv->display.load_luts = i9xx_load_luts; 671 dev_priv->display.load_luts = i9xx_load_luts;
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 391dd69ae0a4..c0a8805b277f 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -956,8 +956,10 @@ void intel_crt_init(struct drm_i915_private *dev_priv)
956 crt->base.power_domain = POWER_DOMAIN_PORT_CRT; 956 crt->base.power_domain = POWER_DOMAIN_PORT_CRT;
957 957
958 if (I915_HAS_HOTPLUG(dev_priv) && 958 if (I915_HAS_HOTPLUG(dev_priv) &&
959 !dmi_check_system(intel_spurious_crt_detect)) 959 !dmi_check_system(intel_spurious_crt_detect)) {
960 crt->base.hpd_pin = HPD_CRT; 960 crt->base.hpd_pin = HPD_CRT;
961 crt->base.hotplug = intel_encoder_hotplug;
962 }
961 963
962 if (HAS_DDI(dev_priv)) { 964 if (HAS_DDI(dev_priv)) {
963 crt->base.port = PORT_E; 965 crt->base.port = PORT_E;
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 8ca376aca8bd..ac8fc2a44ac6 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -25,6 +25,7 @@
25 * 25 *
26 */ 26 */
27 27
28#include <drm/drm_scdc_helper.h>
28#include "i915_drv.h" 29#include "i915_drv.h"
29#include "intel_drv.h" 30#include "intel_drv.h"
30 31
@@ -2507,6 +2508,8 @@ static void intel_disable_ddi_dp(struct intel_encoder *encoder,
2507{ 2508{
2508 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 2509 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
2509 2510
2511 intel_dp->link_trained = false;
2512
2510 if (old_crtc_state->has_audio) 2513 if (old_crtc_state->has_audio)
2511 intel_audio_codec_disable(encoder, 2514 intel_audio_codec_disable(encoder,
2512 old_crtc_state, old_conn_state); 2515 old_crtc_state, old_conn_state);
@@ -2798,6 +2801,150 @@ intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port)
2798 return connector; 2801 return connector;
2799} 2802}
2800 2803
2804static int modeset_pipe(struct drm_crtc *crtc,
2805 struct drm_modeset_acquire_ctx *ctx)
2806{
2807 struct drm_atomic_state *state;
2808 struct drm_crtc_state *crtc_state;
2809 int ret;
2810
2811 state = drm_atomic_state_alloc(crtc->dev);
2812 if (!state)
2813 return -ENOMEM;
2814
2815 state->acquire_ctx = ctx;
2816
2817 crtc_state = drm_atomic_get_crtc_state(state, crtc);
2818 if (IS_ERR(crtc_state)) {
2819 ret = PTR_ERR(crtc_state);
2820 goto out;
2821 }
2822
2823 crtc_state->mode_changed = true;
2824
2825 ret = drm_atomic_add_affected_connectors(state, crtc);
2826 if (ret)
2827 goto out;
2828
2829 ret = drm_atomic_add_affected_planes(state, crtc);
2830 if (ret)
2831 goto out;
2832
2833 ret = drm_atomic_commit(state);
2834 if (ret)
2835 goto out;
2836
2837 return 0;
2838
2839 out:
2840 drm_atomic_state_put(state);
2841
2842 return ret;
2843}
2844
2845static int intel_hdmi_reset_link(struct intel_encoder *encoder,
2846 struct drm_modeset_acquire_ctx *ctx)
2847{
2848 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
2849 struct intel_hdmi *hdmi = enc_to_intel_hdmi(&encoder->base);
2850 struct intel_connector *connector = hdmi->attached_connector;
2851 struct i2c_adapter *adapter =
2852 intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus);
2853 struct drm_connector_state *conn_state;
2854 struct intel_crtc_state *crtc_state;
2855 struct intel_crtc *crtc;
2856 u8 config;
2857 int ret;
2858
2859 if (!connector || connector->base.status != connector_status_connected)
2860 return 0;
2861
2862 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
2863 ctx);
2864 if (ret)
2865 return ret;
2866
2867 conn_state = connector->base.state;
2868
2869 crtc = to_intel_crtc(conn_state->crtc);
2870 if (!crtc)
2871 return 0;
2872
2873 ret = drm_modeset_lock(&crtc->base.mutex, ctx);
2874 if (ret)
2875 return ret;
2876
2877 crtc_state = to_intel_crtc_state(crtc->base.state);
2878
2879 WARN_ON(!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI));
2880
2881 if (!crtc_state->base.active)
2882 return 0;
2883
2884 if (!crtc_state->hdmi_high_tmds_clock_ratio &&
2885 !crtc_state->hdmi_scrambling)
2886 return 0;
2887
2888 if (conn_state->commit &&
2889 !try_wait_for_completion(&conn_state->commit->hw_done))
2890 return 0;
2891
2892 ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config);
2893 if (ret < 0) {
2894 DRM_ERROR("Failed to read TMDS config: %d\n", ret);
2895 return 0;
2896 }
2897
2898 if (!!(config & SCDC_TMDS_BIT_CLOCK_RATIO_BY_40) ==
2899 crtc_state->hdmi_high_tmds_clock_ratio &&
2900 !!(config & SCDC_SCRAMBLING_ENABLE) ==
2901 crtc_state->hdmi_scrambling)
2902 return 0;
2903
2904 /*
2905 * HDMI 2.0 says that one should not send scrambled data
2906 * prior to configuring the sink scrambling, and that
2907 * TMDS clock/data transmission should be suspended when
2908 * changing the TMDS clock rate in the sink. So let's
2909 * just do a full modeset here, even though some sinks
2910 * would be perfectly happy if were to just reconfigure
2911 * the SCDC settings on the fly.
2912 */
2913 return modeset_pipe(&crtc->base, ctx);
2914}
2915
2916static bool intel_ddi_hotplug(struct intel_encoder *encoder,
2917 struct intel_connector *connector)
2918{
2919 struct drm_modeset_acquire_ctx ctx;
2920 bool changed;
2921 int ret;
2922
2923 changed = intel_encoder_hotplug(encoder, connector);
2924
2925 drm_modeset_acquire_init(&ctx, 0);
2926
2927 for (;;) {
2928 if (connector->base.connector_type == DRM_MODE_CONNECTOR_HDMIA)
2929 ret = intel_hdmi_reset_link(encoder, &ctx);
2930 else
2931 ret = intel_dp_retrain_link(encoder, &ctx);
2932
2933 if (ret == -EDEADLK) {
2934 drm_modeset_backoff(&ctx);
2935 continue;
2936 }
2937
2938 break;
2939 }
2940
2941 drm_modeset_drop_locks(&ctx);
2942 drm_modeset_acquire_fini(&ctx);
2943 WARN(ret, "Acquiring modeset locks failed with %i\n", ret);
2944
2945 return changed;
2946}
2947
2801static struct intel_connector * 2948static struct intel_connector *
2802intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port) 2949intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
2803{ 2950{
@@ -2842,39 +2989,45 @@ static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dport)
2842 return false; 2989 return false;
2843} 2990}
2844 2991
2992static int
2993intel_ddi_max_lanes(struct intel_digital_port *intel_dport)
2994{
2995 struct drm_i915_private *dev_priv = to_i915(intel_dport->base.base.dev);
2996 enum port port = intel_dport->base.port;
2997 int max_lanes = 4;
2998
2999 if (INTEL_GEN(dev_priv) >= 11)
3000 return max_lanes;
3001
3002 if (port == PORT_A || port == PORT_E) {
3003 if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
3004 max_lanes = port == PORT_A ? 4 : 0;
3005 else
3006 /* Both A and E share 2 lanes */
3007 max_lanes = 2;
3008 }
3009
3010 /*
3011 * Some BIOS might fail to set this bit on port A if eDP
3012 * wasn't lit up at boot. Force this bit set when needed
3013 * so we use the proper lane count for our calculations.
3014 */
3015 if (intel_ddi_a_force_4_lanes(intel_dport)) {
3016 DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n");
3017 intel_dport->saved_port_bits |= DDI_A_4_LANES;
3018 max_lanes = 4;
3019 }
3020
3021 return max_lanes;
3022}
3023
2845void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) 3024void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
2846{ 3025{
2847 struct intel_digital_port *intel_dig_port; 3026 struct intel_digital_port *intel_dig_port;
2848 struct intel_encoder *intel_encoder; 3027 struct intel_encoder *intel_encoder;
2849 struct drm_encoder *encoder; 3028 struct drm_encoder *encoder;
2850 bool init_hdmi, init_dp, init_lspcon = false; 3029 bool init_hdmi, init_dp, init_lspcon = false;
2851 int max_lanes;
2852 3030
2853 if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) {
2854 switch (port) {
2855 case PORT_A:
2856 max_lanes = 4;
2857 break;
2858 case PORT_E:
2859 max_lanes = 0;
2860 break;
2861 default:
2862 max_lanes = 4;
2863 break;
2864 }
2865 } else {
2866 switch (port) {
2867 case PORT_A:
2868 max_lanes = 2;
2869 break;
2870 case PORT_E:
2871 max_lanes = 2;
2872 break;
2873 default:
2874 max_lanes = 4;
2875 break;
2876 }
2877 }
2878 3031
2879 init_hdmi = (dev_priv->vbt.ddi_port_info[port].supports_dvi || 3032 init_hdmi = (dev_priv->vbt.ddi_port_info[port].supports_dvi ||
2880 dev_priv->vbt.ddi_port_info[port].supports_hdmi); 3033 dev_priv->vbt.ddi_port_info[port].supports_hdmi);
@@ -2908,6 +3061,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
2908 drm_encoder_init(&dev_priv->drm, encoder, &intel_ddi_funcs, 3061 drm_encoder_init(&dev_priv->drm, encoder, &intel_ddi_funcs,
2909 DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port)); 3062 DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port));
2910 3063
3064 intel_encoder->hotplug = intel_ddi_hotplug;
2911 intel_encoder->compute_output_type = intel_ddi_compute_output_type; 3065 intel_encoder->compute_output_type = intel_ddi_compute_output_type;
2912 intel_encoder->compute_config = intel_ddi_compute_config; 3066 intel_encoder->compute_config = intel_ddi_compute_config;
2913 intel_encoder->enable = intel_enable_ddi; 3067 intel_encoder->enable = intel_enable_ddi;
@@ -2920,10 +3074,17 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
2920 intel_encoder->get_config = intel_ddi_get_config; 3074 intel_encoder->get_config = intel_ddi_get_config;
2921 intel_encoder->suspend = intel_dp_encoder_suspend; 3075 intel_encoder->suspend = intel_dp_encoder_suspend;
2922 intel_encoder->get_power_domains = intel_ddi_get_power_domains; 3076 intel_encoder->get_power_domains = intel_ddi_get_power_domains;
3077 intel_encoder->type = INTEL_OUTPUT_DDI;
3078 intel_encoder->power_domain = intel_port_to_power_domain(port);
3079 intel_encoder->port = port;
3080 intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
3081 intel_encoder->cloneable = 0;
2923 3082
2924 intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) & 3083 intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) &
2925 (DDI_BUF_PORT_REVERSAL | 3084 (DDI_BUF_PORT_REVERSAL |
2926 DDI_A_4_LANES); 3085 DDI_A_4_LANES);
3086 intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
3087 intel_dig_port->max_lanes = intel_ddi_max_lanes(intel_dig_port);
2927 3088
2928 switch (port) { 3089 switch (port) {
2929 case PORT_A: 3090 case PORT_A:
@@ -2954,26 +3115,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
2954 MISSING_CASE(port); 3115 MISSING_CASE(port);
2955 } 3116 }
2956 3117
2957 /*
2958 * Some BIOS might fail to set this bit on port A if eDP
2959 * wasn't lit up at boot. Force this bit set when needed
2960 * so we use the proper lane count for our calculations.
2961 */
2962 if (intel_ddi_a_force_4_lanes(intel_dig_port)) {
2963 DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n");
2964 intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
2965 max_lanes = 4;
2966 }
2967
2968 intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
2969 intel_dig_port->max_lanes = max_lanes;
2970
2971 intel_encoder->type = INTEL_OUTPUT_DDI;
2972 intel_encoder->power_domain = intel_port_to_power_domain(port);
2973 intel_encoder->port = port;
2974 intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
2975 intel_encoder->cloneable = 0;
2976
2977 intel_infoframe_init(intel_dig_port); 3118 intel_infoframe_init(intel_dig_port);
2978 3119
2979 if (init_dp) { 3120 if (init_dp) {
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 298f8996cc54..3dd350f7b8e6 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -81,12 +81,16 @@ void intel_device_info_dump_flags(const struct intel_device_info *info,
81 81
82static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p) 82static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p)
83{ 83{
84 int s;
85
84 drm_printf(p, "slice mask: %04x\n", sseu->slice_mask); 86 drm_printf(p, "slice mask: %04x\n", sseu->slice_mask);
85 drm_printf(p, "slice total: %u\n", hweight8(sseu->slice_mask)); 87 drm_printf(p, "slice total: %u\n", hweight8(sseu->slice_mask));
86 drm_printf(p, "subslice total: %u\n", sseu_subslice_total(sseu)); 88 drm_printf(p, "subslice total: %u\n", sseu_subslice_total(sseu));
87 drm_printf(p, "subslice mask %04x\n", sseu->subslice_mask); 89 for (s = 0; s < ARRAY_SIZE(sseu->subslice_mask); s++) {
88 drm_printf(p, "subslice per slice: %u\n", 90 drm_printf(p, "slice%d %u subslices mask=%04x\n",
89 hweight8(sseu->subslice_mask)); 91 s, hweight8(sseu->subslice_mask[s]),
92 sseu->subslice_mask[s]);
93 }
90 drm_printf(p, "EU total: %u\n", sseu->eu_total); 94 drm_printf(p, "EU total: %u\n", sseu->eu_total);
91 drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice); 95 drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice);
92 drm_printf(p, "has slice power gating: %s\n", 96 drm_printf(p, "has slice power gating: %s\n",
@@ -120,22 +124,100 @@ void intel_device_info_dump(const struct intel_device_info *info,
120 intel_device_info_dump_flags(info, p); 124 intel_device_info_dump_flags(info, p);
121} 125}
122 126
127void intel_device_info_dump_topology(const struct sseu_dev_info *sseu,
128 struct drm_printer *p)
129{
130 int s, ss;
131
132 if (sseu->max_slices == 0) {
133 drm_printf(p, "Unavailable\n");
134 return;
135 }
136
137 for (s = 0; s < sseu->max_slices; s++) {
138 drm_printf(p, "slice%d: %u subslice(s) (0x%hhx):\n",
139 s, hweight8(sseu->subslice_mask[s]),
140 sseu->subslice_mask[s]);
141
142 for (ss = 0; ss < sseu->max_subslices; ss++) {
143 u16 enabled_eus = sseu_get_eus(sseu, s, ss);
144
145 drm_printf(p, "\tsubslice%d: %u EUs (0x%hx)\n",
146 ss, hweight16(enabled_eus), enabled_eus);
147 }
148 }
149}
150
151static u16 compute_eu_total(const struct sseu_dev_info *sseu)
152{
153 u16 i, total = 0;
154
155 for (i = 0; i < ARRAY_SIZE(sseu->eu_mask); i++)
156 total += hweight8(sseu->eu_mask[i]);
157
158 return total;
159}
160
123static void gen10_sseu_info_init(struct drm_i915_private *dev_priv) 161static void gen10_sseu_info_init(struct drm_i915_private *dev_priv)
124{ 162{
125 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; 163 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
126 const u32 fuse2 = I915_READ(GEN8_FUSE2); 164 const u32 fuse2 = I915_READ(GEN8_FUSE2);
165 int s, ss;
166 const int eu_mask = 0xff;
167 u32 subslice_mask, eu_en;
127 168
128 sseu->slice_mask = (fuse2 & GEN10_F2_S_ENA_MASK) >> 169 sseu->slice_mask = (fuse2 & GEN10_F2_S_ENA_MASK) >>
129 GEN10_F2_S_ENA_SHIFT; 170 GEN10_F2_S_ENA_SHIFT;
130 sseu->subslice_mask = (1 << 4) - 1; 171 sseu->max_slices = 6;
131 sseu->subslice_mask &= ~((fuse2 & GEN10_F2_SS_DIS_MASK) >> 172 sseu->max_subslices = 4;
132 GEN10_F2_SS_DIS_SHIFT); 173 sseu->max_eus_per_subslice = 8;
133 174
134 sseu->eu_total = hweight32(~I915_READ(GEN8_EU_DISABLE0)); 175 subslice_mask = (1 << 4) - 1;
135 sseu->eu_total += hweight32(~I915_READ(GEN8_EU_DISABLE1)); 176 subslice_mask &= ~((fuse2 & GEN10_F2_SS_DIS_MASK) >>
136 sseu->eu_total += hweight32(~I915_READ(GEN8_EU_DISABLE2)); 177 GEN10_F2_SS_DIS_SHIFT);
137 sseu->eu_total += hweight8(~(I915_READ(GEN10_EU_DISABLE3) & 178
138 GEN10_EU_DIS_SS_MASK)); 179 /*
180 * Slice0 can have up to 3 subslices, but there are only 2 in
181 * slice1/2.
182 */
183 sseu->subslice_mask[0] = subslice_mask;
184 for (s = 1; s < sseu->max_slices; s++)
185 sseu->subslice_mask[s] = subslice_mask & 0x3;
186
187 /* Slice0 */
188 eu_en = ~I915_READ(GEN8_EU_DISABLE0);
189 for (ss = 0; ss < sseu->max_subslices; ss++)
190 sseu_set_eus(sseu, 0, ss, (eu_en >> (8 * ss)) & eu_mask);
191 /* Slice1 */
192 sseu_set_eus(sseu, 1, 0, (eu_en >> 24) & eu_mask);
193 eu_en = ~I915_READ(GEN8_EU_DISABLE1);
194 sseu_set_eus(sseu, 1, 1, eu_en & eu_mask);
195 /* Slice2 */
196 sseu_set_eus(sseu, 2, 0, (eu_en >> 8) & eu_mask);
197 sseu_set_eus(sseu, 2, 1, (eu_en >> 16) & eu_mask);
198 /* Slice3 */
199 sseu_set_eus(sseu, 3, 0, (eu_en >> 24) & eu_mask);
200 eu_en = ~I915_READ(GEN8_EU_DISABLE2);
201 sseu_set_eus(sseu, 3, 1, eu_en & eu_mask);
202 /* Slice4 */
203 sseu_set_eus(sseu, 4, 0, (eu_en >> 8) & eu_mask);
204 sseu_set_eus(sseu, 4, 1, (eu_en >> 16) & eu_mask);
205 /* Slice5 */
206 sseu_set_eus(sseu, 5, 0, (eu_en >> 24) & eu_mask);
207 eu_en = ~I915_READ(GEN10_EU_DISABLE3);
208 sseu_set_eus(sseu, 5, 1, eu_en & eu_mask);
209
210 /* Do a second pass where we mark the subslices disabled if all their
211 * eus are off.
212 */
213 for (s = 0; s < sseu->max_slices; s++) {
214 for (ss = 0; ss < sseu->max_subslices; ss++) {
215 if (sseu_get_eus(sseu, s, ss) == 0)
216 sseu->subslice_mask[s] &= ~BIT(ss);
217 }
218 }
219
220 sseu->eu_total = compute_eu_total(sseu);
139 221
140 /* 222 /*
141 * CNL is expected to always have a uniform distribution 223 * CNL is expected to always have a uniform distribution
@@ -156,26 +238,39 @@ static void gen10_sseu_info_init(struct drm_i915_private *dev_priv)
156static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv) 238static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
157{ 239{
158 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; 240 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
159 u32 fuse, eu_dis; 241 u32 fuse;
160 242
161 fuse = I915_READ(CHV_FUSE_GT); 243 fuse = I915_READ(CHV_FUSE_GT);
162 244
163 sseu->slice_mask = BIT(0); 245 sseu->slice_mask = BIT(0);
246 sseu->max_slices = 1;
247 sseu->max_subslices = 2;
248 sseu->max_eus_per_subslice = 8;
164 249
165 if (!(fuse & CHV_FGT_DISABLE_SS0)) { 250 if (!(fuse & CHV_FGT_DISABLE_SS0)) {
166 sseu->subslice_mask |= BIT(0); 251 u8 disabled_mask =
167 eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK | 252 ((fuse & CHV_FGT_EU_DIS_SS0_R0_MASK) >>
168 CHV_FGT_EU_DIS_SS0_R1_MASK); 253 CHV_FGT_EU_DIS_SS0_R0_SHIFT) |
169 sseu->eu_total += 8 - hweight32(eu_dis); 254 (((fuse & CHV_FGT_EU_DIS_SS0_R1_MASK) >>
255 CHV_FGT_EU_DIS_SS0_R1_SHIFT) << 4);
256
257 sseu->subslice_mask[0] |= BIT(0);
258 sseu_set_eus(sseu, 0, 0, ~disabled_mask);
170 } 259 }
171 260
172 if (!(fuse & CHV_FGT_DISABLE_SS1)) { 261 if (!(fuse & CHV_FGT_DISABLE_SS1)) {
173 sseu->subslice_mask |= BIT(1); 262 u8 disabled_mask =
174 eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK | 263 ((fuse & CHV_FGT_EU_DIS_SS1_R0_MASK) >>
175 CHV_FGT_EU_DIS_SS1_R1_MASK); 264 CHV_FGT_EU_DIS_SS1_R0_SHIFT) |
176 sseu->eu_total += 8 - hweight32(eu_dis); 265 (((fuse & CHV_FGT_EU_DIS_SS1_R1_MASK) >>
266 CHV_FGT_EU_DIS_SS1_R1_SHIFT) << 4);
267
268 sseu->subslice_mask[0] |= BIT(1);
269 sseu_set_eus(sseu, 0, 1, ~disabled_mask);
177 } 270 }
178 271
272 sseu->eu_total = compute_eu_total(sseu);
273
179 /* 274 /*
180 * CHV expected to always have a uniform distribution of EU 275 * CHV expected to always have a uniform distribution of EU
181 * across subslices. 276 * across subslices.
@@ -197,41 +292,52 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
197{ 292{
198 struct intel_device_info *info = mkwrite_device_info(dev_priv); 293 struct intel_device_info *info = mkwrite_device_info(dev_priv);
199 struct sseu_dev_info *sseu = &info->sseu; 294 struct sseu_dev_info *sseu = &info->sseu;
200 int s_max = 3, ss_max = 4, eu_max = 8;
201 int s, ss; 295 int s, ss;
202 u32 fuse2, eu_disable; 296 u32 fuse2, eu_disable, subslice_mask;
203 u8 eu_mask = 0xff; 297 const u8 eu_mask = 0xff;
204 298
205 fuse2 = I915_READ(GEN8_FUSE2); 299 fuse2 = I915_READ(GEN8_FUSE2);
206 sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; 300 sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
207 301
302 /* BXT has a single slice and at most 3 subslices. */
303 sseu->max_slices = IS_GEN9_LP(dev_priv) ? 1 : 3;
304 sseu->max_subslices = IS_GEN9_LP(dev_priv) ? 3 : 4;
305 sseu->max_eus_per_subslice = 8;
306
208 /* 307 /*
209 * The subslice disable field is global, i.e. it applies 308 * The subslice disable field is global, i.e. it applies
210 * to each of the enabled slices. 309 * to each of the enabled slices.
211 */ 310 */
212 sseu->subslice_mask = (1 << ss_max) - 1; 311 subslice_mask = (1 << sseu->max_subslices) - 1;
213 sseu->subslice_mask &= ~((fuse2 & GEN9_F2_SS_DIS_MASK) >> 312 subslice_mask &= ~((fuse2 & GEN9_F2_SS_DIS_MASK) >>
214 GEN9_F2_SS_DIS_SHIFT); 313 GEN9_F2_SS_DIS_SHIFT);
215 314
216 /* 315 /*
217 * Iterate through enabled slices and subslices to 316 * Iterate through enabled slices and subslices to
218 * count the total enabled EU. 317 * count the total enabled EU.
219 */ 318 */
220 for (s = 0; s < s_max; s++) { 319 for (s = 0; s < sseu->max_slices; s++) {
221 if (!(sseu->slice_mask & BIT(s))) 320 if (!(sseu->slice_mask & BIT(s)))
222 /* skip disabled slice */ 321 /* skip disabled slice */
223 continue; 322 continue;
224 323
324 sseu->subslice_mask[s] = subslice_mask;
325
225 eu_disable = I915_READ(GEN9_EU_DISABLE(s)); 326 eu_disable = I915_READ(GEN9_EU_DISABLE(s));
226 for (ss = 0; ss < ss_max; ss++) { 327 for (ss = 0; ss < sseu->max_subslices; ss++) {
227 int eu_per_ss; 328 int eu_per_ss;
329 u8 eu_disabled_mask;
228 330
229 if (!(sseu->subslice_mask & BIT(ss))) 331 if (!(sseu->subslice_mask[s] & BIT(ss)))
230 /* skip disabled subslice */ 332 /* skip disabled subslice */
231 continue; 333 continue;
232 334
233 eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) & 335 eu_disabled_mask = (eu_disable >> (ss * 8)) & eu_mask;
234 eu_mask); 336
337 sseu_set_eus(sseu, s, ss, ~eu_disabled_mask);
338
339 eu_per_ss = sseu->max_eus_per_subslice -
340 hweight8(eu_disabled_mask);
235 341
236 /* 342 /*
237 * Record which subslice(s) has(have) 7 EUs. we 343 * Record which subslice(s) has(have) 7 EUs. we
@@ -240,11 +346,11 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
240 */ 346 */
241 if (eu_per_ss == 7) 347 if (eu_per_ss == 7)
242 sseu->subslice_7eu[s] |= BIT(ss); 348 sseu->subslice_7eu[s] |= BIT(ss);
243
244 sseu->eu_total += eu_per_ss;
245 } 349 }
246 } 350 }
247 351
352 sseu->eu_total = compute_eu_total(sseu);
353
248 /* 354 /*
249 * SKL is expected to always have a uniform distribution 355 * SKL is expected to always have a uniform distribution
250 * of EU across subslices with the exception that any one 356 * of EU across subslices with the exception that any one
@@ -270,8 +376,8 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
270 sseu->has_eu_pg = sseu->eu_per_subslice > 2; 376 sseu->has_eu_pg = sseu->eu_per_subslice > 2;
271 377
272 if (IS_GEN9_LP(dev_priv)) { 378 if (IS_GEN9_LP(dev_priv)) {
273#define IS_SS_DISABLED(ss) (!(sseu->subslice_mask & BIT(ss))) 379#define IS_SS_DISABLED(ss) (!(sseu->subslice_mask[0] & BIT(ss)))
274 info->has_pooled_eu = hweight8(sseu->subslice_mask) == 3; 380 info->has_pooled_eu = hweight8(sseu->subslice_mask[0]) == 3;
275 381
276 sseu->min_eu_in_pool = 0; 382 sseu->min_eu_in_pool = 0;
277 if (info->has_pooled_eu) { 383 if (info->has_pooled_eu) {
@@ -289,19 +395,22 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
289static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv) 395static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
290{ 396{
291 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu; 397 struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
292 const int s_max = 3, ss_max = 3, eu_max = 8;
293 int s, ss; 398 int s, ss;
294 u32 fuse2, eu_disable[3]; /* s_max */ 399 u32 fuse2, subslice_mask, eu_disable[3]; /* s_max */
295 400
296 fuse2 = I915_READ(GEN8_FUSE2); 401 fuse2 = I915_READ(GEN8_FUSE2);
297 sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; 402 sseu->slice_mask = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
403 sseu->max_slices = 3;
404 sseu->max_subslices = 3;
405 sseu->max_eus_per_subslice = 8;
406
298 /* 407 /*
299 * The subslice disable field is global, i.e. it applies 408 * The subslice disable field is global, i.e. it applies
300 * to each of the enabled slices. 409 * to each of the enabled slices.
301 */ 410 */
302 sseu->subslice_mask = GENMASK(ss_max - 1, 0); 411 subslice_mask = GENMASK(sseu->max_subslices - 1, 0);
303 sseu->subslice_mask &= ~((fuse2 & GEN8_F2_SS_DIS_MASK) >> 412 subslice_mask &= ~((fuse2 & GEN8_F2_SS_DIS_MASK) >>
304 GEN8_F2_SS_DIS_SHIFT); 413 GEN8_F2_SS_DIS_SHIFT);
305 414
306 eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK; 415 eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
307 eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) | 416 eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
@@ -315,30 +424,38 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
315 * Iterate through enabled slices and subslices to 424 * Iterate through enabled slices and subslices to
316 * count the total enabled EU. 425 * count the total enabled EU.
317 */ 426 */
318 for (s = 0; s < s_max; s++) { 427 for (s = 0; s < sseu->max_slices; s++) {
319 if (!(sseu->slice_mask & BIT(s))) 428 if (!(sseu->slice_mask & BIT(s)))
320 /* skip disabled slice */ 429 /* skip disabled slice */
321 continue; 430 continue;
322 431
323 for (ss = 0; ss < ss_max; ss++) { 432 sseu->subslice_mask[s] = subslice_mask;
433
434 for (ss = 0; ss < sseu->max_subslices; ss++) {
435 u8 eu_disabled_mask;
324 u32 n_disabled; 436 u32 n_disabled;
325 437
326 if (!(sseu->subslice_mask & BIT(ss))) 438 if (!(sseu->subslice_mask[ss] & BIT(ss)))
327 /* skip disabled subslice */ 439 /* skip disabled subslice */
328 continue; 440 continue;
329 441
330 n_disabled = hweight8(eu_disable[s] >> (ss * eu_max)); 442 eu_disabled_mask =
443 eu_disable[s] >> (ss * sseu->max_eus_per_subslice);
444
445 sseu_set_eus(sseu, s, ss, ~eu_disabled_mask);
446
447 n_disabled = hweight8(eu_disabled_mask);
331 448
332 /* 449 /*
333 * Record which subslices have 7 EUs. 450 * Record which subslices have 7 EUs.
334 */ 451 */
335 if (eu_max - n_disabled == 7) 452 if (sseu->max_eus_per_subslice - n_disabled == 7)
336 sseu->subslice_7eu[s] |= 1 << ss; 453 sseu->subslice_7eu[s] |= 1 << ss;
337
338 sseu->eu_total += eu_max - n_disabled;
339 } 454 }
340 } 455 }
341 456
457 sseu->eu_total = compute_eu_total(sseu);
458
342 /* 459 /*
343 * BDW is expected to always have a uniform distribution of EU across 460 * BDW is expected to always have a uniform distribution of EU across
344 * subslices with the exception that any one EU in any one subslice may 461 * subslices with the exception that any one EU in any one subslice may
@@ -357,6 +474,72 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
357 sseu->has_eu_pg = 0; 474 sseu->has_eu_pg = 0;
358} 475}
359 476
477static void haswell_sseu_info_init(struct drm_i915_private *dev_priv)
478{
479 struct intel_device_info *info = mkwrite_device_info(dev_priv);
480 struct sseu_dev_info *sseu = &info->sseu;
481 u32 fuse1;
482 int s, ss;
483
484 /*
485 * There isn't a register to tell us how many slices/subslices. We
486 * work off the PCI-ids here.
487 */
488 switch (info->gt) {
489 default:
490 MISSING_CASE(info->gt);
491 /* fall through */
492 case 1:
493 sseu->slice_mask = BIT(0);
494 sseu->subslice_mask[0] = BIT(0);
495 break;
496 case 2:
497 sseu->slice_mask = BIT(0);
498 sseu->subslice_mask[0] = BIT(0) | BIT(1);
499 break;
500 case 3:
501 sseu->slice_mask = BIT(0) | BIT(1);
502 sseu->subslice_mask[0] = BIT(0) | BIT(1);
503 sseu->subslice_mask[1] = BIT(0) | BIT(1);
504 break;
505 }
506
507 sseu->max_slices = hweight8(sseu->slice_mask);
508 sseu->max_subslices = hweight8(sseu->subslice_mask[0]);
509
510 fuse1 = I915_READ(HSW_PAVP_FUSE1);
511 switch ((fuse1 & HSW_F1_EU_DIS_MASK) >> HSW_F1_EU_DIS_SHIFT) {
512 default:
513 MISSING_CASE((fuse1 & HSW_F1_EU_DIS_MASK) >>
514 HSW_F1_EU_DIS_SHIFT);
515 /* fall through */
516 case HSW_F1_EU_DIS_10EUS:
517 sseu->eu_per_subslice = 10;
518 break;
519 case HSW_F1_EU_DIS_8EUS:
520 sseu->eu_per_subslice = 8;
521 break;
522 case HSW_F1_EU_DIS_6EUS:
523 sseu->eu_per_subslice = 6;
524 break;
525 }
526 sseu->max_eus_per_subslice = sseu->eu_per_subslice;
527
528 for (s = 0; s < sseu->max_slices; s++) {
529 for (ss = 0; ss < sseu->max_subslices; ss++) {
530 sseu_set_eus(sseu, s, ss,
531 (1UL << sseu->eu_per_subslice) - 1);
532 }
533 }
534
535 sseu->eu_total = compute_eu_total(sseu);
536
537 /* No powergating for you. */
538 sseu->has_slice_pg = 0;
539 sseu->has_subslice_pg = 0;
540 sseu->has_eu_pg = 0;
541}
542
360static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv) 543static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv)
361{ 544{
362 u32 ts_override = I915_READ(GEN9_TIMESTAMP_OVERRIDE); 545 u32 ts_override = I915_READ(GEN9_TIMESTAMP_OVERRIDE);
@@ -489,6 +672,9 @@ void intel_device_info_runtime_init(struct intel_device_info *info)
489 info->num_scalers[PIPE_C] = 1; 672 info->num_scalers[PIPE_C] = 1;
490 } 673 }
491 674
675 BUILD_BUG_ON(I915_NUM_ENGINES >
676 sizeof(intel_ring_mask_t) * BITS_PER_BYTE);
677
492 /* 678 /*
493 * Skylake and Broxton currently don't expose the topmost plane as its 679 * Skylake and Broxton currently don't expose the topmost plane as its
494 * use is exclusive with the legacy cursor and we only want to expose 680 * use is exclusive with the legacy cursor and we only want to expose
@@ -574,7 +760,9 @@ void intel_device_info_runtime_init(struct intel_device_info *info)
574 } 760 }
575 761
576 /* Initialize slice/subslice/EU info */ 762 /* Initialize slice/subslice/EU info */
577 if (IS_CHERRYVIEW(dev_priv)) 763 if (IS_HASWELL(dev_priv))
764 haswell_sseu_info_init(dev_priv);
765 else if (IS_CHERRYVIEW(dev_priv))
578 cherryview_sseu_info_init(dev_priv); 766 cherryview_sseu_info_init(dev_priv);
579 else if (IS_BROADWELL(dev_priv)) 767 else if (IS_BROADWELL(dev_priv))
580 broadwell_sseu_info_init(dev_priv); 768 broadwell_sseu_info_init(dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 71fdfb0451ef..0835752c8b22 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -96,6 +96,7 @@ enum intel_platform {
96 func(has_l3_dpf); \ 96 func(has_l3_dpf); \
97 func(has_llc); \ 97 func(has_llc); \
98 func(has_logical_ring_contexts); \ 98 func(has_logical_ring_contexts); \
99 func(has_logical_ring_elsq); \
99 func(has_logical_ring_preemption); \ 100 func(has_logical_ring_preemption); \
100 func(has_overlay); \ 101 func(has_overlay); \
101 func(has_pooled_eu); \ 102 func(has_pooled_eu); \
@@ -112,10 +113,13 @@ enum intel_platform {
112 func(supports_tv); \ 113 func(supports_tv); \
113 func(has_ipc); 114 func(has_ipc);
114 115
116#define GEN_MAX_SLICES (6) /* CNL upper bound */
117#define GEN_MAX_SUBSLICES (7)
118
115struct sseu_dev_info { 119struct sseu_dev_info {
116 u8 slice_mask; 120 u8 slice_mask;
117 u8 subslice_mask; 121 u8 subslice_mask[GEN_MAX_SUBSLICES];
118 u8 eu_total; 122 u16 eu_total;
119 u8 eu_per_subslice; 123 u8 eu_per_subslice;
120 u8 min_eu_in_pool; 124 u8 min_eu_in_pool;
121 /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */ 125 /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
@@ -123,8 +127,21 @@ struct sseu_dev_info {
123 u8 has_slice_pg:1; 127 u8 has_slice_pg:1;
124 u8 has_subslice_pg:1; 128 u8 has_subslice_pg:1;
125 u8 has_eu_pg:1; 129 u8 has_eu_pg:1;
130
131 /* Topology fields */
132 u8 max_slices;
133 u8 max_subslices;
134 u8 max_eus_per_subslice;
135
136 /* We don't have more than 8 eus per subslice at the moment and as we
137 * store eus enabled using bits, no need to multiply by eus per
138 * subslice.
139 */
140 u8 eu_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICES];
126}; 141};
127 142
143typedef u8 intel_ring_mask_t;
144
128struct intel_device_info { 145struct intel_device_info {
129 u16 device_id; 146 u16 device_id;
130 u16 gen_mask; 147 u16 gen_mask;
@@ -132,19 +149,19 @@ struct intel_device_info {
132 u8 gen; 149 u8 gen;
133 u8 gt; /* GT number, 0 if undefined */ 150 u8 gt; /* GT number, 0 if undefined */
134 u8 num_rings; 151 u8 num_rings;
135 u8 ring_mask; /* Rings supported by the HW */ 152 intel_ring_mask_t ring_mask; /* Rings supported by the HW */
136 153
137 enum intel_platform platform; 154 enum intel_platform platform;
138 u32 platform_mask; 155 u32 platform_mask;
139 156
157 unsigned int page_sizes; /* page sizes supported by the HW */
158
140 u32 display_mmio_offset; 159 u32 display_mmio_offset;
141 160
142 u8 num_pipes; 161 u8 num_pipes;
143 u8 num_sprites[I915_MAX_PIPES]; 162 u8 num_sprites[I915_MAX_PIPES];
144 u8 num_scalers[I915_MAX_PIPES]; 163 u8 num_scalers[I915_MAX_PIPES];
145 164
146 unsigned int page_sizes; /* page sizes supported by the HW */
147
148#define DEFINE_FLAG(name) u8 name:1 165#define DEFINE_FLAG(name) u8 name:1
149 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); 166 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
150#undef DEFINE_FLAG 167#undef DEFINE_FLAG
@@ -173,7 +190,49 @@ struct intel_driver_caps {
173 190
174static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu) 191static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
175{ 192{
176 return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask); 193 unsigned int i, total = 0;
194
195 for (i = 0; i < ARRAY_SIZE(sseu->subslice_mask); i++)
196 total += hweight8(sseu->subslice_mask[i]);
197
198 return total;
199}
200
201static inline int sseu_eu_idx(const struct sseu_dev_info *sseu,
202 int slice, int subslice)
203{
204 int subslice_stride = DIV_ROUND_UP(sseu->max_eus_per_subslice,
205 BITS_PER_BYTE);
206 int slice_stride = sseu->max_subslices * subslice_stride;
207
208 return slice * slice_stride + subslice * subslice_stride;
209}
210
211static inline u16 sseu_get_eus(const struct sseu_dev_info *sseu,
212 int slice, int subslice)
213{
214 int i, offset = sseu_eu_idx(sseu, slice, subslice);
215 u16 eu_mask = 0;
216
217 for (i = 0;
218 i < DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); i++) {
219 eu_mask |= ((u16) sseu->eu_mask[offset + i]) <<
220 (i * BITS_PER_BYTE);
221 }
222
223 return eu_mask;
224}
225
226static inline void sseu_set_eus(struct sseu_dev_info *sseu,
227 int slice, int subslice, u16 eu_mask)
228{
229 int i, offset = sseu_eu_idx(sseu, slice, subslice);
230
231 for (i = 0;
232 i < DIV_ROUND_UP(sseu->max_eus_per_subslice, BITS_PER_BYTE); i++) {
233 sseu->eu_mask[offset + i] =
234 (eu_mask >> (BITS_PER_BYTE * i)) & 0xff;
235 }
177} 236}
178 237
179const char *intel_platform_name(enum intel_platform platform); 238const char *intel_platform_name(enum intel_platform platform);
@@ -185,6 +244,8 @@ void intel_device_info_dump_flags(const struct intel_device_info *info,
185 struct drm_printer *p); 244 struct drm_printer *p);
186void intel_device_info_dump_runtime(const struct intel_device_info *info, 245void intel_device_info_dump_runtime(const struct intel_device_info *info,
187 struct drm_printer *p); 246 struct drm_printer *p);
247void intel_device_info_dump_topology(const struct sseu_dev_info *sseu,
248 struct drm_printer *p);
188 249
189void intel_driver_caps_print(const struct intel_driver_caps *caps, 250void intel_driver_caps_print(const struct intel_driver_caps *caps,
190 struct drm_printer *p); 251 struct drm_printer *p);
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 8be9c3371b7e..331084082545 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -2067,9 +2067,18 @@ static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
2067 } 2067 }
2068} 2068}
2069 2069
2070static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
2071{
2072 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
2073 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
2074
2075 return INTEL_GEN(dev_priv) < 4 || plane->has_fbc;
2076}
2077
2070struct i915_vma * 2078struct i915_vma *
2071intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, 2079intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2072 unsigned int rotation, 2080 unsigned int rotation,
2081 bool uses_fence,
2073 unsigned long *out_flags) 2082 unsigned long *out_flags)
2074{ 2083{
2075 struct drm_device *dev = fb->dev; 2084 struct drm_device *dev = fb->dev;
@@ -2122,7 +2131,9 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2122 if (IS_ERR(vma)) 2131 if (IS_ERR(vma))
2123 goto err; 2132 goto err;
2124 2133
2125 if (i915_vma_is_map_and_fenceable(vma)) { 2134 if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
2135 int ret;
2136
2126 /* Install a fence for tiled scan-out. Pre-i965 always needs a 2137 /* Install a fence for tiled scan-out. Pre-i965 always needs a
2127 * fence, whereas 965+ only requires a fence if using 2138 * fence, whereas 965+ only requires a fence if using
2128 * framebuffer compression. For simplicity, we always, when 2139 * framebuffer compression. For simplicity, we always, when
@@ -2139,7 +2150,14 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2139 * something and try to run the system in a "less than optimal" 2150 * something and try to run the system in a "less than optimal"
2140 * mode that matches the user configuration. 2151 * mode that matches the user configuration.
2141 */ 2152 */
2142 if (i915_vma_pin_fence(vma) == 0 && vma->fence) 2153 ret = i915_vma_pin_fence(vma);
2154 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
2155 i915_gem_object_unpin_from_display_plane(vma);
2156 vma = ERR_PTR(ret);
2157 goto err;
2158 }
2159
2160 if (ret == 0 && vma->fence)
2143 *out_flags |= PLANE_HAS_FENCE; 2161 *out_flags |= PLANE_HAS_FENCE;
2144 } 2162 }
2145 2163
@@ -2828,6 +2846,7 @@ valid_fb:
2828 intel_state->vma = 2846 intel_state->vma =
2829 intel_pin_and_fence_fb_obj(fb, 2847 intel_pin_and_fence_fb_obj(fb,
2830 primary->state->rotation, 2848 primary->state->rotation,
2849 intel_plane_uses_fence(intel_state),
2831 &intel_state->flags); 2850 &intel_state->flags);
2832 mutex_unlock(&dev->struct_mutex); 2851 mutex_unlock(&dev->struct_mutex);
2833 if (IS_ERR(intel_state->vma)) { 2852 if (IS_ERR(intel_state->vma)) {
@@ -12034,6 +12053,14 @@ static int intel_atomic_check(struct drm_device *dev,
12034 int ret, i; 12053 int ret, i;
12035 bool any_ms = false; 12054 bool any_ms = false;
12036 12055
12056 /* Catch I915_MODE_FLAG_INHERITED */
12057 for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
12058 crtc_state, i) {
12059 if (crtc_state->mode.private_flags !=
12060 old_crtc_state->mode.private_flags)
12061 crtc_state->mode_changed = true;
12062 }
12063
12037 ret = drm_atomic_helper_check_modeset(dev, state); 12064 ret = drm_atomic_helper_check_modeset(dev, state);
12038 if (ret) 12065 if (ret)
12039 return ret; 12066 return ret;
@@ -12042,10 +12069,6 @@ static int intel_atomic_check(struct drm_device *dev,
12042 struct intel_crtc_state *pipe_config = 12069 struct intel_crtc_state *pipe_config =
12043 to_intel_crtc_state(crtc_state); 12070 to_intel_crtc_state(crtc_state);
12044 12071
12045 /* Catch I915_MODE_FLAG_INHERITED */
12046 if (crtc_state->mode.private_flags != old_crtc_state->mode.private_flags)
12047 crtc_state->mode_changed = true;
12048
12049 if (!needs_modeset(crtc_state)) 12072 if (!needs_modeset(crtc_state))
12050 continue; 12073 continue;
12051 12074
@@ -12054,13 +12077,6 @@ static int intel_atomic_check(struct drm_device *dev,
12054 continue; 12077 continue;
12055 } 12078 }
12056 12079
12057 /* FIXME: For only active_changed we shouldn't need to do any
12058 * state recomputation at all. */
12059
12060 ret = drm_atomic_add_affected_connectors(state, crtc);
12061 if (ret)
12062 return ret;
12063
12064 ret = intel_modeset_pipe_config(crtc, pipe_config); 12080 ret = intel_modeset_pipe_config(crtc, pipe_config);
12065 if (ret) { 12081 if (ret) {
12066 intel_dump_pipe_config(to_intel_crtc(crtc), 12082 intel_dump_pipe_config(to_intel_crtc(crtc),
@@ -12079,10 +12095,6 @@ static int intel_atomic_check(struct drm_device *dev,
12079 if (needs_modeset(crtc_state)) 12095 if (needs_modeset(crtc_state))
12080 any_ms = true; 12096 any_ms = true;
12081 12097
12082 ret = drm_atomic_add_affected_planes(state, crtc);
12083 if (ret)
12084 return ret;
12085
12086 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, 12098 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
12087 needs_modeset(crtc_state) ? 12099 needs_modeset(crtc_state) ?
12088 "[modeset]" : "[fastset]"); 12100 "[modeset]" : "[fastset]");
@@ -12600,23 +12612,23 @@ struct wait_rps_boost {
12600 struct wait_queue_entry wait; 12612 struct wait_queue_entry wait;
12601 12613
12602 struct drm_crtc *crtc; 12614 struct drm_crtc *crtc;
12603 struct drm_i915_gem_request *request; 12615 struct i915_request *request;
12604}; 12616};
12605 12617
12606static int do_rps_boost(struct wait_queue_entry *_wait, 12618static int do_rps_boost(struct wait_queue_entry *_wait,
12607 unsigned mode, int sync, void *key) 12619 unsigned mode, int sync, void *key)
12608{ 12620{
12609 struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait); 12621 struct wait_rps_boost *wait = container_of(_wait, typeof(*wait), wait);
12610 struct drm_i915_gem_request *rq = wait->request; 12622 struct i915_request *rq = wait->request;
12611 12623
12612 /* 12624 /*
12613 * If we missed the vblank, but the request is already running it 12625 * If we missed the vblank, but the request is already running it
12614 * is reasonable to assume that it will complete before the next 12626 * is reasonable to assume that it will complete before the next
12615 * vblank without our intervention, so leave RPS alone. 12627 * vblank without our intervention, so leave RPS alone.
12616 */ 12628 */
12617 if (!i915_gem_request_started(rq)) 12629 if (!i915_request_started(rq))
12618 gen6_rps_boost(rq, NULL); 12630 gen6_rps_boost(rq, NULL);
12619 i915_gem_request_put(rq); 12631 i915_request_put(rq);
12620 12632
12621 drm_crtc_vblank_put(wait->crtc); 12633 drm_crtc_vblank_put(wait->crtc);
12622 12634
@@ -12654,6 +12666,42 @@ static void add_rps_boost_after_vblank(struct drm_crtc *crtc,
12654 add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait); 12666 add_wait_queue(drm_crtc_vblank_waitqueue(crtc), &wait->wait);
12655} 12667}
12656 12668
12669static int intel_plane_pin_fb(struct intel_plane_state *plane_state)
12670{
12671 struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
12672 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
12673 struct drm_framebuffer *fb = plane_state->base.fb;
12674 struct i915_vma *vma;
12675
12676 if (plane->id == PLANE_CURSOR &&
12677 INTEL_INFO(dev_priv)->cursor_needs_physical) {
12678 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
12679 const int align = intel_cursor_alignment(dev_priv);
12680
12681 return i915_gem_object_attach_phys(obj, align);
12682 }
12683
12684 vma = intel_pin_and_fence_fb_obj(fb,
12685 plane_state->base.rotation,
12686 intel_plane_uses_fence(plane_state),
12687 &plane_state->flags);
12688 if (IS_ERR(vma))
12689 return PTR_ERR(vma);
12690
12691 plane_state->vma = vma;
12692
12693 return 0;
12694}
12695
12696static void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
12697{
12698 struct i915_vma *vma;
12699
12700 vma = fetch_and_zero(&old_plane_state->vma);
12701 if (vma)
12702 intel_unpin_fb_vma(vma, old_plane_state->flags);
12703}
12704
12657/** 12705/**
12658 * intel_prepare_plane_fb - Prepare fb for usage on plane 12706 * intel_prepare_plane_fb - Prepare fb for usage on plane
12659 * @plane: drm plane to prepare for 12707 * @plane: drm plane to prepare for
@@ -12728,22 +12776,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
12728 return ret; 12776 return ret;
12729 } 12777 }
12730 12778
12731 if (plane->type == DRM_PLANE_TYPE_CURSOR && 12779 ret = intel_plane_pin_fb(to_intel_plane_state(new_state));
12732 INTEL_INFO(dev_priv)->cursor_needs_physical) {
12733 const int align = intel_cursor_alignment(dev_priv);
12734
12735 ret = i915_gem_object_attach_phys(obj, align);
12736 } else {
12737 struct i915_vma *vma;
12738
12739 vma = intel_pin_and_fence_fb_obj(fb,
12740 new_state->rotation,
12741 &to_intel_plane_state(new_state)->flags);
12742 if (!IS_ERR(vma))
12743 to_intel_plane_state(new_state)->vma = vma;
12744 else
12745 ret = PTR_ERR(vma);
12746 }
12747 12780
12748 i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY); 12781 i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
12749 12782
@@ -12787,15 +12820,12 @@ void
12787intel_cleanup_plane_fb(struct drm_plane *plane, 12820intel_cleanup_plane_fb(struct drm_plane *plane,
12788 struct drm_plane_state *old_state) 12821 struct drm_plane_state *old_state)
12789{ 12822{
12790 struct i915_vma *vma; 12823 struct drm_i915_private *dev_priv = to_i915(plane->dev);
12791 12824
12792 /* Should only be called after a successful intel_prepare_plane_fb()! */ 12825 /* Should only be called after a successful intel_prepare_plane_fb()! */
12793 vma = fetch_and_zero(&to_intel_plane_state(old_state)->vma); 12826 mutex_lock(&dev_priv->drm.struct_mutex);
12794 if (vma) { 12827 intel_plane_unpin_fb(to_intel_plane_state(old_state));
12795 mutex_lock(&plane->dev->struct_mutex); 12828 mutex_unlock(&dev_priv->drm.struct_mutex);
12796 intel_unpin_fb_vma(vma, to_intel_plane_state(old_state)->flags);
12797 mutex_unlock(&plane->dev->struct_mutex);
12798 }
12799} 12829}
12800 12830
12801int 12831int
@@ -13080,7 +13110,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
13080 struct intel_plane *intel_plane = to_intel_plane(plane); 13110 struct intel_plane *intel_plane = to_intel_plane(plane);
13081 struct drm_framebuffer *old_fb; 13111 struct drm_framebuffer *old_fb;
13082 struct drm_crtc_state *crtc_state = crtc->state; 13112 struct drm_crtc_state *crtc_state = crtc->state;
13083 struct i915_vma *old_vma, *vma;
13084 13113
13085 /* 13114 /*
13086 * When crtc is inactive or there is a modeset pending, 13115 * When crtc is inactive or there is a modeset pending,
@@ -13139,27 +13168,9 @@ intel_legacy_cursor_update(struct drm_plane *plane,
13139 if (ret) 13168 if (ret)
13140 goto out_free; 13169 goto out_free;
13141 13170
13142 if (INTEL_INFO(dev_priv)->cursor_needs_physical) { 13171 ret = intel_plane_pin_fb(to_intel_plane_state(new_plane_state));
13143 int align = intel_cursor_alignment(dev_priv); 13172 if (ret)
13144 13173 goto out_unlock;
13145 ret = i915_gem_object_attach_phys(intel_fb_obj(fb), align);
13146 if (ret) {
13147 DRM_DEBUG_KMS("failed to attach phys object\n");
13148 goto out_unlock;
13149 }
13150 } else {
13151 vma = intel_pin_and_fence_fb_obj(fb,
13152 new_plane_state->rotation,
13153 &to_intel_plane_state(new_plane_state)->flags);
13154 if (IS_ERR(vma)) {
13155 DRM_DEBUG_KMS("failed to pin object\n");
13156
13157 ret = PTR_ERR(vma);
13158 goto out_unlock;
13159 }
13160
13161 to_intel_plane_state(new_plane_state)->vma = vma;
13162 }
13163 13174
13164 old_fb = old_plane_state->fb; 13175 old_fb = old_plane_state->fb;
13165 13176
@@ -13179,10 +13190,7 @@ intel_legacy_cursor_update(struct drm_plane *plane,
13179 intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc)); 13190 intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
13180 } 13191 }
13181 13192
13182 old_vma = fetch_and_zero(&to_intel_plane_state(old_plane_state)->vma); 13193 intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
13183 if (old_vma)
13184 intel_unpin_fb_vma(old_vma,
13185 to_intel_plane_state(old_plane_state)->flags);
13186 13194
13187out_unlock: 13195out_unlock:
13188 mutex_unlock(&dev_priv->drm.struct_mutex); 13196 mutex_unlock(&dev_priv->drm.struct_mutex);
@@ -13210,6 +13218,32 @@ static const struct drm_plane_funcs intel_cursor_plane_funcs = {
13210 .format_mod_supported = intel_cursor_plane_format_mod_supported, 13218 .format_mod_supported = intel_cursor_plane_format_mod_supported,
13211}; 13219};
13212 13220
13221static bool i9xx_plane_has_fbc(struct drm_i915_private *dev_priv,
13222 enum i9xx_plane_id i9xx_plane)
13223{
13224 if (!HAS_FBC(dev_priv))
13225 return false;
13226
13227 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
13228 return i9xx_plane == PLANE_A; /* tied to pipe A */
13229 else if (IS_IVYBRIDGE(dev_priv))
13230 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B ||
13231 i9xx_plane == PLANE_C;
13232 else if (INTEL_GEN(dev_priv) >= 4)
13233 return i9xx_plane == PLANE_A || i9xx_plane == PLANE_B;
13234 else
13235 return i9xx_plane == PLANE_A;
13236}
13237
13238static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv,
13239 enum pipe pipe, enum plane_id plane_id)
13240{
13241 if (!HAS_FBC(dev_priv))
13242 return false;
13243
13244 return pipe == PIPE_A && plane_id == PLANE_PRIMARY;
13245}
13246
13213static struct intel_plane * 13247static struct intel_plane *
13214intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) 13248intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13215{ 13249{
@@ -13252,6 +13286,21 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
13252 primary->i9xx_plane = (enum i9xx_plane_id) pipe; 13286 primary->i9xx_plane = (enum i9xx_plane_id) pipe;
13253 primary->id = PLANE_PRIMARY; 13287 primary->id = PLANE_PRIMARY;
13254 primary->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, primary->id); 13288 primary->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, primary->id);
13289
13290 if (INTEL_GEN(dev_priv) >= 9)
13291 primary->has_fbc = skl_plane_has_fbc(dev_priv,
13292 primary->pipe,
13293 primary->id);
13294 else
13295 primary->has_fbc = i9xx_plane_has_fbc(dev_priv,
13296 primary->i9xx_plane);
13297
13298 if (primary->has_fbc) {
13299 struct intel_fbc *fbc = &dev_priv->fbc;
13300
13301 fbc->possible_framebuffer_bits |= primary->frontbuffer_bit;
13302 }
13303
13255 primary->check_plane = intel_check_primary_plane; 13304 primary->check_plane = intel_check_primary_plane;
13256 13305
13257 if (INTEL_GEN(dev_priv) >= 9) { 13306 if (INTEL_GEN(dev_priv) >= 9) {
diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h
index c4042e342f50..4e7418b345bc 100644
--- a/drivers/gpu/drm/i915/intel_display.h
+++ b/drivers/gpu/drm/i915/intel_display.h
@@ -139,6 +139,17 @@ enum dpio_phy {
139 139
140#define I915_NUM_PHYS_VLV 2 140#define I915_NUM_PHYS_VLV 2
141 141
142enum aux_ch {
143 AUX_CH_A,
144 AUX_CH_B,
145 AUX_CH_C,
146 AUX_CH_D,
147 _AUX_CH_E, /* does not exist */
148 AUX_CH_F,
149};
150
151#define aux_ch_name(a) ((a) + 'A')
152
142enum intel_display_power_domain { 153enum intel_display_power_domain {
143 POWER_DOMAIN_PIPE_A, 154 POWER_DOMAIN_PIPE_A,
144 POWER_DOMAIN_PIPE_B, 155 POWER_DOMAIN_PIPE_B,
@@ -175,6 +186,7 @@ enum intel_display_power_domain {
175 POWER_DOMAIN_AUX_C, 186 POWER_DOMAIN_AUX_C,
176 POWER_DOMAIN_AUX_D, 187 POWER_DOMAIN_AUX_D,
177 POWER_DOMAIN_AUX_F, 188 POWER_DOMAIN_AUX_F,
189 POWER_DOMAIN_AUX_IO_A,
178 POWER_DOMAIN_GMBUS, 190 POWER_DOMAIN_GMBUS,
179 POWER_DOMAIN_MODESET, 191 POWER_DOMAIN_MODESET,
180 POWER_DOMAIN_GT_IRQ, 192 POWER_DOMAIN_GT_IRQ,
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index f20b25f98e5a..9a4a51e79fa1 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -96,15 +96,6 @@ static const struct dp_link_dpll chv_dpll[] = {
96 { .p1 = 2, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } } 96 { .p1 = 2, .p2 = 1, .n = 1, .m1 = 2, .m2 = 0x6c00000 } }
97}; 97};
98 98
99static const int bxt_rates[] = { 162000, 216000, 243000, 270000,
100 324000, 432000, 540000 };
101static const int skl_rates[] = { 162000, 216000, 270000,
102 324000, 432000, 540000 };
103static const int cnl_rates[] = { 162000, 216000, 270000,
104 324000, 432000, 540000,
105 648000, 810000 };
106static const int default_rates[] = { 162000, 270000, 540000 };
107
108/** 99/**
109 * intel_dp_is_edp - is the given port attached to an eDP panel (either CPU or PCH) 100 * intel_dp_is_edp - is the given port attached to an eDP panel (either CPU or PCH)
110 * @intel_dp: DP struct 101 * @intel_dp: DP struct
@@ -144,14 +135,17 @@ static void intel_dp_unset_edid(struct intel_dp *intel_dp);
144/* update sink rates from dpcd */ 135/* update sink rates from dpcd */
145static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) 136static void intel_dp_set_sink_rates(struct intel_dp *intel_dp)
146{ 137{
138 static const int dp_rates[] = {
139 162000, 270000, 540000, 810000
140 };
147 int i, max_rate; 141 int i, max_rate;
148 142
149 max_rate = drm_dp_bw_code_to_link_rate(intel_dp->dpcd[DP_MAX_LINK_RATE]); 143 max_rate = drm_dp_bw_code_to_link_rate(intel_dp->dpcd[DP_MAX_LINK_RATE]);
150 144
151 for (i = 0; i < ARRAY_SIZE(default_rates); i++) { 145 for (i = 0; i < ARRAY_SIZE(dp_rates); i++) {
152 if (default_rates[i] > max_rate) 146 if (dp_rates[i] > max_rate)
153 break; 147 break;
154 intel_dp->sink_rates[i] = default_rates[i]; 148 intel_dp->sink_rates[i] = dp_rates[i];
155 } 149 }
156 150
157 intel_dp->num_sink_rates = i; 151 intel_dp->num_sink_rates = i;
@@ -268,6 +262,22 @@ static int cnl_max_source_rate(struct intel_dp *intel_dp)
268static void 262static void
269intel_dp_set_source_rates(struct intel_dp *intel_dp) 263intel_dp_set_source_rates(struct intel_dp *intel_dp)
270{ 264{
265 /* The values must be in increasing order */
266 static const int cnl_rates[] = {
267 162000, 216000, 270000, 324000, 432000, 540000, 648000, 810000
268 };
269 static const int bxt_rates[] = {
270 162000, 216000, 243000, 270000, 324000, 432000, 540000
271 };
272 static const int skl_rates[] = {
273 162000, 216000, 270000, 324000, 432000, 540000
274 };
275 static const int hsw_rates[] = {
276 162000, 270000, 540000
277 };
278 static const int g4x_rates[] = {
279 162000, 270000
280 };
271 struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); 281 struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
272 struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); 282 struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
273 const struct ddi_vbt_port_info *info = 283 const struct ddi_vbt_port_info *info =
@@ -278,23 +288,23 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp)
278 /* This should only be done once */ 288 /* This should only be done once */
279 WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates); 289 WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates);
280 290
281 if (IS_GEN9_LP(dev_priv)) { 291 if (IS_CANNONLAKE(dev_priv)) {
282 source_rates = bxt_rates;
283 size = ARRAY_SIZE(bxt_rates);
284 } else if (IS_CANNONLAKE(dev_priv)) {
285 source_rates = cnl_rates; 292 source_rates = cnl_rates;
286 size = ARRAY_SIZE(cnl_rates); 293 size = ARRAY_SIZE(cnl_rates);
287 max_rate = cnl_max_source_rate(intel_dp); 294 max_rate = cnl_max_source_rate(intel_dp);
295 } else if (IS_GEN9_LP(dev_priv)) {
296 source_rates = bxt_rates;
297 size = ARRAY_SIZE(bxt_rates);
288 } else if (IS_GEN9_BC(dev_priv)) { 298 } else if (IS_GEN9_BC(dev_priv)) {
289 source_rates = skl_rates; 299 source_rates = skl_rates;
290 size = ARRAY_SIZE(skl_rates); 300 size = ARRAY_SIZE(skl_rates);
291 } else if ((IS_HASWELL(dev_priv) && !IS_HSW_ULX(dev_priv)) || 301 } else if ((IS_HASWELL(dev_priv) && !IS_HSW_ULX(dev_priv)) ||
292 IS_BROADWELL(dev_priv)) { 302 IS_BROADWELL(dev_priv)) {
293 source_rates = default_rates; 303 source_rates = hsw_rates;
294 size = ARRAY_SIZE(default_rates); 304 size = ARRAY_SIZE(hsw_rates);
295 } else { 305 } else {
296 source_rates = default_rates; 306 source_rates = g4x_rates;
297 size = ARRAY_SIZE(default_rates) - 1; 307 size = ARRAY_SIZE(g4x_rates);
298 } 308 }
299 309
300 if (max_rate && vbt_max_rate) 310 if (max_rate && vbt_max_rate)
@@ -356,7 +366,7 @@ static void intel_dp_set_common_rates(struct intel_dp *intel_dp)
356 366
357 /* Paranoia, there should always be something in common. */ 367 /* Paranoia, there should always be something in common. */
358 if (WARN_ON(intel_dp->num_common_rates == 0)) { 368 if (WARN_ON(intel_dp->num_common_rates == 0)) {
359 intel_dp->common_rates[0] = default_rates[0]; 369 intel_dp->common_rates[0] = 162000;
360 intel_dp->num_common_rates = 1; 370 intel_dp->num_common_rates = 1;
361 } 371 }
362} 372}
@@ -656,19 +666,15 @@ static int
656bxt_power_sequencer_idx(struct intel_dp *intel_dp) 666bxt_power_sequencer_idx(struct intel_dp *intel_dp)
657{ 667{
658 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); 668 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
669 int backlight_controller = dev_priv->vbt.backlight.controller;
659 670
660 lockdep_assert_held(&dev_priv->pps_mutex); 671 lockdep_assert_held(&dev_priv->pps_mutex);
661 672
662 /* We should never land here with regular DP ports */ 673 /* We should never land here with regular DP ports */
663 WARN_ON(!intel_dp_is_edp(intel_dp)); 674 WARN_ON(!intel_dp_is_edp(intel_dp));
664 675
665 /*
666 * TODO: BXT has 2 PPS instances. The correct port->PPS instance
667 * mapping needs to be retrieved from VBT, for now just hard-code to
668 * use instance #0 always.
669 */
670 if (!intel_dp->pps_reset) 676 if (!intel_dp->pps_reset)
671 return 0; 677 return backlight_controller;
672 678
673 intel_dp->pps_reset = false; 679 intel_dp->pps_reset = false;
674 680
@@ -678,7 +684,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp)
678 */ 684 */
679 intel_dp_init_panel_power_sequencer_registers(intel_dp, false); 685 intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
680 686
681 return 0; 687 return backlight_controller;
682} 688}
683 689
684typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv, 690typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
@@ -936,7 +942,7 @@ static uint32_t
936intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq) 942intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
937{ 943{
938 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); 944 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
939 i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; 945 i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
940 uint32_t status; 946 uint32_t status;
941 bool done; 947 bool done;
942 948
@@ -956,8 +962,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
956 962
957static uint32_t g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index) 963static uint32_t g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
958{ 964{
959 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 965 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
960 struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
961 966
962 if (index) 967 if (index)
963 return 0; 968 return 0;
@@ -971,8 +976,7 @@ static uint32_t g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
971 976
972static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) 977static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
973{ 978{
974 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 979 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
975 struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
976 980
977 if (index) 981 if (index)
978 return 0; 982 return 0;
@@ -982,7 +986,7 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
982 * like to run at 2MHz. So, take the cdclk or PCH rawclk value and 986 * like to run at 2MHz. So, take the cdclk or PCH rawclk value and
983 * divide by 2000 and use that 987 * divide by 2000 and use that
984 */ 988 */
985 if (intel_dig_port->base.port == PORT_A) 989 if (intel_dp->aux_ch == AUX_CH_A)
986 return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000); 990 return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000);
987 else 991 else
988 return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000); 992 return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000);
@@ -990,10 +994,9 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
990 994
991static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index) 995static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
992{ 996{
993 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 997 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
994 struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
995 998
996 if (intel_dig_port->base.port != PORT_A && HAS_PCH_LPT_H(dev_priv)) { 999 if (intel_dp->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(dev_priv)) {
997 /* Workaround for non-ULT HSW */ 1000 /* Workaround for non-ULT HSW */
998 switch (index) { 1001 switch (index) {
999 case 0: return 63; 1002 case 0: return 63;
@@ -1062,34 +1065,16 @@ static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp,
1062 DP_AUX_CH_CTL_SYNC_PULSE_SKL(32); 1065 DP_AUX_CH_CTL_SYNC_PULSE_SKL(32);
1063} 1066}
1064 1067
1065static uint32_t intel_dp_get_aux_send_ctl(struct intel_dp *intel_dp,
1066 bool has_aux_irq,
1067 int send_bytes,
1068 uint32_t aux_clock_divider,
1069 bool aksv_write)
1070{
1071 uint32_t val = 0;
1072
1073 if (aksv_write) {
1074 send_bytes += 5;
1075 val |= DP_AUX_CH_CTL_AUX_AKSV_SELECT;
1076 }
1077
1078 return val | intel_dp->get_aux_send_ctl(intel_dp,
1079 has_aux_irq,
1080 send_bytes,
1081 aux_clock_divider);
1082}
1083
1084static int 1068static int
1085intel_dp_aux_ch(struct intel_dp *intel_dp, 1069intel_dp_aux_xfer(struct intel_dp *intel_dp,
1086 const uint8_t *send, int send_bytes, 1070 const uint8_t *send, int send_bytes,
1087 uint8_t *recv, int recv_size, bool aksv_write) 1071 uint8_t *recv, int recv_size,
1072 u32 aux_send_ctl_flags)
1088{ 1073{
1089 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 1074 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
1090 struct drm_i915_private *dev_priv = 1075 struct drm_i915_private *dev_priv =
1091 to_i915(intel_dig_port->base.base.dev); 1076 to_i915(intel_dig_port->base.base.dev);
1092 i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; 1077 i915_reg_t ch_ctl, ch_data[5];
1093 uint32_t aux_clock_divider; 1078 uint32_t aux_clock_divider;
1094 int i, ret, recv_bytes; 1079 int i, ret, recv_bytes;
1095 uint32_t status; 1080 uint32_t status;
@@ -1097,6 +1082,10 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
1097 bool has_aux_irq = HAS_AUX_IRQ(dev_priv); 1082 bool has_aux_irq = HAS_AUX_IRQ(dev_priv);
1098 bool vdd; 1083 bool vdd;
1099 1084
1085 ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
1086 for (i = 0; i < ARRAY_SIZE(ch_data); i++)
1087 ch_data[i] = intel_dp->aux_ch_data_reg(intel_dp, i);
1088
1100 pps_lock(intel_dp); 1089 pps_lock(intel_dp);
1101 1090
1102 /* 1091 /*
@@ -1144,17 +1133,18 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
1144 } 1133 }
1145 1134
1146 while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) { 1135 while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) {
1147 u32 send_ctl = intel_dp_get_aux_send_ctl(intel_dp, 1136 u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp,
1148 has_aux_irq, 1137 has_aux_irq,
1149 send_bytes, 1138 send_bytes,
1150 aux_clock_divider, 1139 aux_clock_divider);
1151 aksv_write); 1140
1141 send_ctl |= aux_send_ctl_flags;
1152 1142
1153 /* Must try at least 3 times according to DP spec */ 1143 /* Must try at least 3 times according to DP spec */
1154 for (try = 0; try < 5; try++) { 1144 for (try = 0; try < 5; try++) {
1155 /* Load the send data into the aux channel data registers */ 1145 /* Load the send data into the aux channel data registers */
1156 for (i = 0; i < send_bytes; i += 4) 1146 for (i = 0; i < send_bytes; i += 4)
1157 I915_WRITE(intel_dp->aux_ch_data_reg[i >> 2], 1147 I915_WRITE(ch_data[i >> 2],
1158 intel_dp_pack_aux(send + i, 1148 intel_dp_pack_aux(send + i,
1159 send_bytes - i)); 1149 send_bytes - i));
1160 1150
@@ -1170,14 +1160,14 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
1170 DP_AUX_CH_CTL_TIME_OUT_ERROR | 1160 DP_AUX_CH_CTL_TIME_OUT_ERROR |
1171 DP_AUX_CH_CTL_RECEIVE_ERROR); 1161 DP_AUX_CH_CTL_RECEIVE_ERROR);
1172 1162
1173 if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR)
1174 continue;
1175
1176 /* DP CTS 1.2 Core Rev 1.1, 4.2.1.1 & 4.2.1.2 1163 /* DP CTS 1.2 Core Rev 1.1, 4.2.1.1 & 4.2.1.2
1177 * 400us delay required for errors and timeouts 1164 * 400us delay required for errors and timeouts
1178 * Timeout errors from the HW already meet this 1165 * Timeout errors from the HW already meet this
1179 * requirement so skip to next iteration 1166 * requirement so skip to next iteration
1180 */ 1167 */
1168 if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR)
1169 continue;
1170
1181 if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) { 1171 if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) {
1182 usleep_range(400, 500); 1172 usleep_range(400, 500);
1183 continue; 1173 continue;
@@ -1223,14 +1213,6 @@ done:
1223 if (recv_bytes == 0 || recv_bytes > 20) { 1213 if (recv_bytes == 0 || recv_bytes > 20) {
1224 DRM_DEBUG_KMS("Forbidden recv_bytes = %d on aux transaction\n", 1214 DRM_DEBUG_KMS("Forbidden recv_bytes = %d on aux transaction\n",
1225 recv_bytes); 1215 recv_bytes);
1226 /*
1227 * FIXME: This patch was created on top of a series that
1228 * organize the retries at drm level. There EBUSY should
1229 * also take care for 1ms wait before retrying.
1230 * That aux retries re-org is still needed and after that is
1231 * merged we remove this sleep from here.
1232 */
1233 usleep_range(1000, 1500);
1234 ret = -EBUSY; 1216 ret = -EBUSY;
1235 goto out; 1217 goto out;
1236 } 1218 }
@@ -1239,7 +1221,7 @@ done:
1239 recv_bytes = recv_size; 1221 recv_bytes = recv_size;
1240 1222
1241 for (i = 0; i < recv_bytes; i += 4) 1223 for (i = 0; i < recv_bytes; i += 4)
1242 intel_dp_unpack_aux(I915_READ(intel_dp->aux_ch_data_reg[i >> 2]), 1224 intel_dp_unpack_aux(I915_READ(ch_data[i >> 2]),
1243 recv + i, recv_bytes - i); 1225 recv + i, recv_bytes - i);
1244 1226
1245 ret = recv_bytes; 1227 ret = recv_bytes;
@@ -1256,6 +1238,17 @@ out:
1256 1238
1257#define BARE_ADDRESS_SIZE 3 1239#define BARE_ADDRESS_SIZE 3
1258#define HEADER_SIZE (BARE_ADDRESS_SIZE + 1) 1240#define HEADER_SIZE (BARE_ADDRESS_SIZE + 1)
1241
1242static void
1243intel_dp_aux_header(u8 txbuf[HEADER_SIZE],
1244 const struct drm_dp_aux_msg *msg)
1245{
1246 txbuf[0] = (msg->request << 4) | ((msg->address >> 16) & 0xf);
1247 txbuf[1] = (msg->address >> 8) & 0xff;
1248 txbuf[2] = msg->address & 0xff;
1249 txbuf[3] = msg->size - 1;
1250}
1251
1259static ssize_t 1252static ssize_t
1260intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) 1253intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
1261{ 1254{
@@ -1264,11 +1257,7 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
1264 size_t txsize, rxsize; 1257 size_t txsize, rxsize;
1265 int ret; 1258 int ret;
1266 1259
1267 txbuf[0] = (msg->request << 4) | 1260 intel_dp_aux_header(txbuf, msg);
1268 ((msg->address >> 16) & 0xf);
1269 txbuf[1] = (msg->address >> 8) & 0xff;
1270 txbuf[2] = msg->address & 0xff;
1271 txbuf[3] = msg->size - 1;
1272 1261
1273 switch (msg->request & ~DP_AUX_I2C_MOT) { 1262 switch (msg->request & ~DP_AUX_I2C_MOT) {
1274 case DP_AUX_NATIVE_WRITE: 1263 case DP_AUX_NATIVE_WRITE:
@@ -1285,8 +1274,8 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
1285 if (msg->buffer) 1274 if (msg->buffer)
1286 memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size); 1275 memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size);
1287 1276
1288 ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize, 1277 ret = intel_dp_aux_xfer(intel_dp, txbuf, txsize,
1289 false); 1278 rxbuf, rxsize, 0);
1290 if (ret > 0) { 1279 if (ret > 0) {
1291 msg->reply = rxbuf[0] >> 4; 1280 msg->reply = rxbuf[0] >> 4;
1292 1281
@@ -1308,8 +1297,8 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
1308 if (WARN_ON(rxsize > 20)) 1297 if (WARN_ON(rxsize > 20))
1309 return -E2BIG; 1298 return -E2BIG;
1310 1299
1311 ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize, 1300 ret = intel_dp_aux_xfer(intel_dp, txbuf, txsize,
1312 false); 1301 rxbuf, rxsize, 0);
1313 if (ret > 0) { 1302 if (ret > 0) {
1314 msg->reply = rxbuf[0] >> 4; 1303 msg->reply = rxbuf[0] >> 4;
1315 /* 1304 /*
@@ -1331,171 +1320,173 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
1331 return ret; 1320 return ret;
1332} 1321}
1333 1322
1334static enum port intel_aux_port(struct drm_i915_private *dev_priv, 1323static enum aux_ch intel_aux_ch(struct intel_dp *intel_dp)
1335 enum port port)
1336{ 1324{
1325 struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
1326 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1327 enum port port = encoder->port;
1337 const struct ddi_vbt_port_info *info = 1328 const struct ddi_vbt_port_info *info =
1338 &dev_priv->vbt.ddi_port_info[port]; 1329 &dev_priv->vbt.ddi_port_info[port];
1339 enum port aux_port; 1330 enum aux_ch aux_ch;
1340 1331
1341 if (!info->alternate_aux_channel) { 1332 if (!info->alternate_aux_channel) {
1333 aux_ch = (enum aux_ch) port;
1334
1342 DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n", 1335 DRM_DEBUG_KMS("using AUX %c for port %c (platform default)\n",
1343 port_name(port), port_name(port)); 1336 aux_ch_name(aux_ch), port_name(port));
1344 return port; 1337 return aux_ch;
1345 } 1338 }
1346 1339
1347 switch (info->alternate_aux_channel) { 1340 switch (info->alternate_aux_channel) {
1348 case DP_AUX_A: 1341 case DP_AUX_A:
1349 aux_port = PORT_A; 1342 aux_ch = AUX_CH_A;
1350 break; 1343 break;
1351 case DP_AUX_B: 1344 case DP_AUX_B:
1352 aux_port = PORT_B; 1345 aux_ch = AUX_CH_B;
1353 break; 1346 break;
1354 case DP_AUX_C: 1347 case DP_AUX_C:
1355 aux_port = PORT_C; 1348 aux_ch = AUX_CH_C;
1356 break; 1349 break;
1357 case DP_AUX_D: 1350 case DP_AUX_D:
1358 aux_port = PORT_D; 1351 aux_ch = AUX_CH_D;
1359 break; 1352 break;
1360 case DP_AUX_F: 1353 case DP_AUX_F:
1361 aux_port = PORT_F; 1354 aux_ch = AUX_CH_F;
1362 break; 1355 break;
1363 default: 1356 default:
1364 MISSING_CASE(info->alternate_aux_channel); 1357 MISSING_CASE(info->alternate_aux_channel);
1365 aux_port = PORT_A; 1358 aux_ch = AUX_CH_A;
1366 break; 1359 break;
1367 } 1360 }
1368 1361
1369 DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n", 1362 DRM_DEBUG_KMS("using AUX %c for port %c (VBT)\n",
1370 port_name(aux_port), port_name(port)); 1363 aux_ch_name(aux_ch), port_name(port));
1371 1364
1372 return aux_port; 1365 return aux_ch;
1373} 1366}
1374 1367
1375static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv, 1368static enum intel_display_power_domain
1376 enum port port) 1369intel_aux_power_domain(struct intel_dp *intel_dp)
1377{ 1370{
1378 switch (port) { 1371 switch (intel_dp->aux_ch) {
1379 case PORT_B: 1372 case AUX_CH_A:
1380 case PORT_C: 1373 return POWER_DOMAIN_AUX_A;
1381 case PORT_D: 1374 case AUX_CH_B:
1382 return DP_AUX_CH_CTL(port); 1375 return POWER_DOMAIN_AUX_B;
1376 case AUX_CH_C:
1377 return POWER_DOMAIN_AUX_C;
1378 case AUX_CH_D:
1379 return POWER_DOMAIN_AUX_D;
1380 case AUX_CH_F:
1381 return POWER_DOMAIN_AUX_F;
1383 default: 1382 default:
1384 MISSING_CASE(port); 1383 MISSING_CASE(intel_dp->aux_ch);
1385 return DP_AUX_CH_CTL(PORT_B); 1384 return POWER_DOMAIN_AUX_A;
1386 } 1385 }
1387} 1386}
1388 1387
1389static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv, 1388static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
1390 enum port port, int index)
1391{ 1389{
1392 switch (port) { 1390 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1393 case PORT_B: 1391 enum aux_ch aux_ch = intel_dp->aux_ch;
1394 case PORT_C: 1392
1395 case PORT_D: 1393 switch (aux_ch) {
1396 return DP_AUX_CH_DATA(port, index); 1394 case AUX_CH_B:
1395 case AUX_CH_C:
1396 case AUX_CH_D:
1397 return DP_AUX_CH_CTL(aux_ch);
1397 default: 1398 default:
1398 MISSING_CASE(port); 1399 MISSING_CASE(aux_ch);
1399 return DP_AUX_CH_DATA(PORT_B, index); 1400 return DP_AUX_CH_CTL(AUX_CH_B);
1400 } 1401 }
1401} 1402}
1402 1403
1403static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv, 1404static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
1404 enum port port)
1405{ 1405{
1406 switch (port) { 1406 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1407 case PORT_A: 1407 enum aux_ch aux_ch = intel_dp->aux_ch;
1408 return DP_AUX_CH_CTL(port); 1408
1409 case PORT_B: 1409 switch (aux_ch) {
1410 case PORT_C: 1410 case AUX_CH_B:
1411 case PORT_D: 1411 case AUX_CH_C:
1412 return PCH_DP_AUX_CH_CTL(port); 1412 case AUX_CH_D:
1413 return DP_AUX_CH_DATA(aux_ch, index);
1413 default: 1414 default:
1414 MISSING_CASE(port); 1415 MISSING_CASE(aux_ch);
1415 return DP_AUX_CH_CTL(PORT_A); 1416 return DP_AUX_CH_DATA(AUX_CH_B, index);
1416 } 1417 }
1417} 1418}
1418 1419
1419static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv, 1420static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
1420 enum port port, int index)
1421{ 1421{
1422 switch (port) { 1422 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1423 case PORT_A: 1423 enum aux_ch aux_ch = intel_dp->aux_ch;
1424 return DP_AUX_CH_DATA(port, index); 1424
1425 case PORT_B: 1425 switch (aux_ch) {
1426 case PORT_C: 1426 case AUX_CH_A:
1427 case PORT_D: 1427 return DP_AUX_CH_CTL(aux_ch);
1428 return PCH_DP_AUX_CH_DATA(port, index); 1428 case AUX_CH_B:
1429 case AUX_CH_C:
1430 case AUX_CH_D:
1431 return PCH_DP_AUX_CH_CTL(aux_ch);
1429 default: 1432 default:
1430 MISSING_CASE(port); 1433 MISSING_CASE(aux_ch);
1431 return DP_AUX_CH_DATA(PORT_A, index); 1434 return DP_AUX_CH_CTL(AUX_CH_A);
1432 } 1435 }
1433} 1436}
1434 1437
1435static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv, 1438static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
1436 enum port port)
1437{ 1439{
1438 switch (port) { 1440 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1439 case PORT_A: 1441 enum aux_ch aux_ch = intel_dp->aux_ch;
1440 case PORT_B: 1442
1441 case PORT_C: 1443 switch (aux_ch) {
1442 case PORT_D: 1444 case AUX_CH_A:
1443 case PORT_F: 1445 return DP_AUX_CH_DATA(aux_ch, index);
1444 return DP_AUX_CH_CTL(port); 1446 case AUX_CH_B:
1447 case AUX_CH_C:
1448 case AUX_CH_D:
1449 return PCH_DP_AUX_CH_DATA(aux_ch, index);
1445 default: 1450 default:
1446 MISSING_CASE(port); 1451 MISSING_CASE(aux_ch);
1447 return DP_AUX_CH_CTL(PORT_A); 1452 return DP_AUX_CH_DATA(AUX_CH_A, index);
1448 } 1453 }
1449} 1454}
1450 1455
1451static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv, 1456static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
1452 enum port port, int index)
1453{ 1457{
1454 switch (port) { 1458 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1455 case PORT_A: 1459 enum aux_ch aux_ch = intel_dp->aux_ch;
1456 case PORT_B: 1460
1457 case PORT_C: 1461 switch (aux_ch) {
1458 case PORT_D: 1462 case AUX_CH_A:
1459 case PORT_F: 1463 case AUX_CH_B:
1460 return DP_AUX_CH_DATA(port, index); 1464 case AUX_CH_C:
1465 case AUX_CH_D:
1466 case AUX_CH_F:
1467 return DP_AUX_CH_CTL(aux_ch);
1461 default: 1468 default:
1462 MISSING_CASE(port); 1469 MISSING_CASE(aux_ch);
1463 return DP_AUX_CH_DATA(PORT_A, index); 1470 return DP_AUX_CH_CTL(AUX_CH_A);
1464 } 1471 }
1465} 1472}
1466 1473
1467static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv, 1474static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
1468 enum port port)
1469{
1470 if (INTEL_GEN(dev_priv) >= 9)
1471 return skl_aux_ctl_reg(dev_priv, port);
1472 else if (HAS_PCH_SPLIT(dev_priv))
1473 return ilk_aux_ctl_reg(dev_priv, port);
1474 else
1475 return g4x_aux_ctl_reg(dev_priv, port);
1476}
1477
1478static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv,
1479 enum port port, int index)
1480{
1481 if (INTEL_GEN(dev_priv) >= 9)
1482 return skl_aux_data_reg(dev_priv, port, index);
1483 else if (HAS_PCH_SPLIT(dev_priv))
1484 return ilk_aux_data_reg(dev_priv, port, index);
1485 else
1486 return g4x_aux_data_reg(dev_priv, port, index);
1487}
1488
1489static void intel_aux_reg_init(struct intel_dp *intel_dp)
1490{ 1475{
1491 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); 1476 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1492 enum port port = intel_aux_port(dev_priv, 1477 enum aux_ch aux_ch = intel_dp->aux_ch;
1493 dp_to_dig_port(intel_dp)->base.port); 1478
1494 int i; 1479 switch (aux_ch) {
1495 1480 case AUX_CH_A:
1496 intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port); 1481 case AUX_CH_B:
1497 for (i = 0; i < ARRAY_SIZE(intel_dp->aux_ch_data_reg); i++) 1482 case AUX_CH_C:
1498 intel_dp->aux_ch_data_reg[i] = intel_aux_data_reg(dev_priv, port, i); 1483 case AUX_CH_D:
1484 case AUX_CH_F:
1485 return DP_AUX_CH_DATA(aux_ch, index);
1486 default:
1487 MISSING_CASE(aux_ch);
1488 return DP_AUX_CH_DATA(AUX_CH_A, index);
1489 }
1499} 1490}
1500 1491
1501static void 1492static void
@@ -1507,14 +1498,42 @@ intel_dp_aux_fini(struct intel_dp *intel_dp)
1507static void 1498static void
1508intel_dp_aux_init(struct intel_dp *intel_dp) 1499intel_dp_aux_init(struct intel_dp *intel_dp)
1509{ 1500{
1510 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 1501 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
1511 enum port port = intel_dig_port->base.port; 1502 struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
1503
1504 intel_dp->aux_ch = intel_aux_ch(intel_dp);
1505 intel_dp->aux_power_domain = intel_aux_power_domain(intel_dp);
1506
1507 if (INTEL_GEN(dev_priv) >= 9) {
1508 intel_dp->aux_ch_ctl_reg = skl_aux_ctl_reg;
1509 intel_dp->aux_ch_data_reg = skl_aux_data_reg;
1510 } else if (HAS_PCH_SPLIT(dev_priv)) {
1511 intel_dp->aux_ch_ctl_reg = ilk_aux_ctl_reg;
1512 intel_dp->aux_ch_data_reg = ilk_aux_data_reg;
1513 } else {
1514 intel_dp->aux_ch_ctl_reg = g4x_aux_ctl_reg;
1515 intel_dp->aux_ch_data_reg = g4x_aux_data_reg;
1516 }
1517
1518 if (INTEL_GEN(dev_priv) >= 9)
1519 intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
1520 else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
1521 intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
1522 else if (HAS_PCH_SPLIT(dev_priv))
1523 intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
1524 else
1525 intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
1526
1527 if (INTEL_GEN(dev_priv) >= 9)
1528 intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl;
1529 else
1530 intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
1512 1531
1513 intel_aux_reg_init(intel_dp);
1514 drm_dp_aux_init(&intel_dp->aux); 1532 drm_dp_aux_init(&intel_dp->aux);
1515 1533
1516 /* Failure to allocate our preferred name is not critical */ 1534 /* Failure to allocate our preferred name is not critical */
1517 intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", port_name(port)); 1535 intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c",
1536 port_name(encoder->port));
1518 intel_dp->aux.transfer = intel_dp_aux_transfer; 1537 intel_dp->aux.transfer = intel_dp_aux_transfer;
1519} 1538}
1520 1539
@@ -1894,6 +1913,7 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
1894 int link_rate, uint8_t lane_count, 1913 int link_rate, uint8_t lane_count,
1895 bool link_mst) 1914 bool link_mst)
1896{ 1915{
1916 intel_dp->link_trained = false;
1897 intel_dp->link_rate = link_rate; 1917 intel_dp->link_rate = link_rate;
1898 intel_dp->lane_count = lane_count; 1918 intel_dp->lane_count = lane_count;
1899 intel_dp->link_mst = link_mst; 1919 intel_dp->link_mst = link_mst;
@@ -2742,6 +2762,8 @@ static void intel_disable_dp(struct intel_encoder *encoder,
2742{ 2762{
2743 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 2763 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
2744 2764
2765 intel_dp->link_trained = false;
2766
2745 if (old_crtc_state->has_audio) 2767 if (old_crtc_state->has_audio)
2746 intel_audio_codec_disable(encoder, 2768 intel_audio_codec_disable(encoder,
2747 old_crtc_state, old_conn_state); 2769 old_crtc_state, old_conn_state);
@@ -3172,35 +3194,6 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_
3172 DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE; 3194 DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE;
3173} 3195}
3174 3196
3175static bool intel_dp_get_y_cord_status(struct intel_dp *intel_dp)
3176{
3177 uint8_t psr_caps = 0;
3178
3179 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_CAPS, &psr_caps) != 1)
3180 return false;
3181 return psr_caps & DP_PSR2_SU_Y_COORDINATE_REQUIRED;
3182}
3183
3184static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp)
3185{
3186 uint8_t dprx = 0;
3187
3188 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST,
3189 &dprx) != 1)
3190 return false;
3191 return dprx & DP_VSC_SDP_EXT_FOR_COLORIMETRY_SUPPORTED;
3192}
3193
3194static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
3195{
3196 uint8_t alpm_caps = 0;
3197
3198 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
3199 &alpm_caps) != 1)
3200 return false;
3201 return alpm_caps & DP_ALPM_CAP;
3202}
3203
3204/* These are source-specific values. */ 3197/* These are source-specific values. */
3205uint8_t 3198uint8_t
3206intel_dp_voltage_max(struct intel_dp *intel_dp) 3199intel_dp_voltage_max(struct intel_dp *intel_dp)
@@ -3751,40 +3744,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
3751 dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] & 3744 dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
3752 DP_NO_AUX_HANDSHAKE_LINK_TRAINING; 3745 DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
3753 3746
3754 /* Check if the panel supports PSR */ 3747 intel_psr_init_dpcd(intel_dp);
3755 drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT,
3756 intel_dp->psr_dpcd,
3757 sizeof(intel_dp->psr_dpcd));
3758 if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
3759 dev_priv->psr.sink_support = true;
3760 DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
3761 }
3762
3763 if (INTEL_GEN(dev_priv) >= 9 &&
3764 (intel_dp->psr_dpcd[0] & DP_PSR2_IS_SUPPORTED)) {
3765 uint8_t frame_sync_cap;
3766
3767 dev_priv->psr.sink_support = true;
3768 if (drm_dp_dpcd_readb(&intel_dp->aux,
3769 DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
3770 &frame_sync_cap) != 1)
3771 frame_sync_cap = 0;
3772 dev_priv->psr.aux_frame_sync = frame_sync_cap ? true : false;
3773 /* PSR2 needs frame sync as well */
3774 dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
3775 DRM_DEBUG_KMS("PSR2 %s on sink",
3776 dev_priv->psr.psr2_support ? "supported" : "not supported");
3777
3778 if (dev_priv->psr.psr2_support) {
3779 dev_priv->psr.y_cord_support =
3780 intel_dp_get_y_cord_status(intel_dp);
3781 dev_priv->psr.colorimetry_support =
3782 intel_dp_get_colorimetry_status(intel_dp);
3783 dev_priv->psr.alpm =
3784 intel_dp_get_alpm_status(intel_dp);
3785 }
3786
3787 }
3788 3748
3789 /* 3749 /*
3790 * Read the eDP display control registers. 3750 * Read the eDP display control registers.
@@ -4315,12 +4275,85 @@ go_again:
4315 return -EINVAL; 4275 return -EINVAL;
4316} 4276}
4317 4277
4318static void 4278static bool
4319intel_dp_retrain_link(struct intel_dp *intel_dp) 4279intel_dp_needs_link_retrain(struct intel_dp *intel_dp)
4280{
4281 u8 link_status[DP_LINK_STATUS_SIZE];
4282
4283 if (!intel_dp->link_trained)
4284 return false;
4285
4286 if (!intel_dp_get_link_status(intel_dp, link_status))
4287 return false;
4288
4289 /*
4290 * Validate the cached values of intel_dp->link_rate and
4291 * intel_dp->lane_count before attempting to retrain.
4292 */
4293 if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate,
4294 intel_dp->lane_count))
4295 return false;
4296
4297 /* Retrain if Channel EQ or CR not ok */
4298 return !drm_dp_channel_eq_ok(link_status, intel_dp->lane_count);
4299}
4300
4301/*
4302 * If display is now connected check links status,
4303 * there has been known issues of link loss triggering
4304 * long pulse.
4305 *
4306 * Some sinks (eg. ASUS PB287Q) seem to perform some
4307 * weird HPD ping pong during modesets. So we can apparently
4308 * end up with HPD going low during a modeset, and then
4309 * going back up soon after. And once that happens we must
4310 * retrain the link to get a picture. That's in case no
4311 * userspace component reacted to intermittent HPD dip.
4312 */
4313int intel_dp_retrain_link(struct intel_encoder *encoder,
4314 struct drm_modeset_acquire_ctx *ctx)
4320{ 4315{
4321 struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
4322 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); 4316 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
4323 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); 4317 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
4318 struct intel_connector *connector = intel_dp->attached_connector;
4319 struct drm_connector_state *conn_state;
4320 struct intel_crtc_state *crtc_state;
4321 struct intel_crtc *crtc;
4322 int ret;
4323
4324 /* FIXME handle the MST connectors as well */
4325
4326 if (!connector || connector->base.status != connector_status_connected)
4327 return 0;
4328
4329 ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
4330 ctx);
4331 if (ret)
4332 return ret;
4333
4334 conn_state = connector->base.state;
4335
4336 crtc = to_intel_crtc(conn_state->crtc);
4337 if (!crtc)
4338 return 0;
4339
4340 ret = drm_modeset_lock(&crtc->base.mutex, ctx);
4341 if (ret)
4342 return ret;
4343
4344 crtc_state = to_intel_crtc_state(crtc->base.state);
4345
4346 WARN_ON(!intel_crtc_has_dp_encoder(crtc_state));
4347
4348 if (!crtc_state->base.active)
4349 return 0;
4350
4351 if (conn_state->commit &&
4352 !try_wait_for_completion(&conn_state->commit->hw_done))
4353 return 0;
4354
4355 if (!intel_dp_needs_link_retrain(intel_dp))
4356 return 0;
4324 4357
4325 /* Suppress underruns caused by re-training */ 4358 /* Suppress underruns caused by re-training */
4326 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); 4359 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
@@ -4338,51 +4371,49 @@ intel_dp_retrain_link(struct intel_dp *intel_dp)
4338 if (crtc->config->has_pch_encoder) 4371 if (crtc->config->has_pch_encoder)
4339 intel_set_pch_fifo_underrun_reporting(dev_priv, 4372 intel_set_pch_fifo_underrun_reporting(dev_priv,
4340 intel_crtc_pch_transcoder(crtc), true); 4373 intel_crtc_pch_transcoder(crtc), true);
4374
4375 return 0;
4341} 4376}
4342 4377
4343static void 4378/*
4344intel_dp_check_link_status(struct intel_dp *intel_dp) 4379 * If display is now connected check links status,
4380 * there has been known issues of link loss triggering
4381 * long pulse.
4382 *
4383 * Some sinks (eg. ASUS PB287Q) seem to perform some
4384 * weird HPD ping pong during modesets. So we can apparently
4385 * end up with HPD going low during a modeset, and then
4386 * going back up soon after. And once that happens we must
4387 * retrain the link to get a picture. That's in case no
4388 * userspace component reacted to intermittent HPD dip.
4389 */
4390static bool intel_dp_hotplug(struct intel_encoder *encoder,
4391 struct intel_connector *connector)
4345{ 4392{
4346 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); 4393 struct drm_modeset_acquire_ctx ctx;
4347 struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; 4394 bool changed;
4348 struct drm_connector_state *conn_state = 4395 int ret;
4349 intel_dp->attached_connector->base.state;
4350 u8 link_status[DP_LINK_STATUS_SIZE];
4351
4352 WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
4353
4354 if (!intel_dp_get_link_status(intel_dp, link_status)) {
4355 DRM_ERROR("Failed to get link status\n");
4356 return;
4357 }
4358 4396
4359 if (!conn_state->crtc) 4397 changed = intel_encoder_hotplug(encoder, connector);
4360 return;
4361 4398
4362 WARN_ON(!drm_modeset_is_locked(&conn_state->crtc->mutex)); 4399 drm_modeset_acquire_init(&ctx, 0);
4363 4400
4364 if (!conn_state->crtc->state->active) 4401 for (;;) {
4365 return; 4402 ret = intel_dp_retrain_link(encoder, &ctx);
4366 4403
4367 if (conn_state->commit && 4404 if (ret == -EDEADLK) {
4368 !try_wait_for_completion(&conn_state->commit->hw_done)) 4405 drm_modeset_backoff(&ctx);
4369 return; 4406 continue;
4407 }
4370 4408
4371 /* 4409 break;
4372 * Validate the cached values of intel_dp->link_rate and 4410 }
4373 * intel_dp->lane_count before attempting to retrain.
4374 */
4375 if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate,
4376 intel_dp->lane_count))
4377 return;
4378 4411
4379 /* Retrain if Channel EQ or CR not ok */ 4412 drm_modeset_drop_locks(&ctx);
4380 if (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { 4413 drm_modeset_acquire_fini(&ctx);
4381 DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n", 4414 WARN(ret, "Acquiring modeset locks failed with %i\n", ret);
4382 intel_encoder->base.name);
4383 4415
4384 intel_dp_retrain_link(intel_dp); 4416 return changed;
4385 }
4386} 4417}
4387 4418
4388/* 4419/*
@@ -4440,7 +4471,9 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
4440 DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n"); 4471 DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n");
4441 } 4472 }
4442 4473
4443 intel_dp_check_link_status(intel_dp); 4474 /* defer to the hotplug work for link retraining if needed */
4475 if (intel_dp_needs_link_retrain(intel_dp))
4476 return false;
4444 4477
4445 if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) { 4478 if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
4446 DRM_DEBUG_KMS("Link Training Compliance Test requested\n"); 4479 DRM_DEBUG_KMS("Link Training Compliance Test requested\n");
@@ -4825,20 +4858,6 @@ intel_dp_long_pulse(struct intel_connector *connector)
4825 */ 4858 */
4826 status = connector_status_disconnected; 4859 status = connector_status_disconnected;
4827 goto out; 4860 goto out;
4828 } else {
4829 /*
4830 * If display is now connected check links status,
4831 * there has been known issues of link loss triggerring
4832 * long pulse.
4833 *
4834 * Some sinks (eg. ASUS PB287Q) seem to perform some
4835 * weird HPD ping pong during modesets. So we can apparently
4836 * end up with HPD going low during a modeset, and then
4837 * going back up soon after. And once that happens we must
4838 * retrain the link to get a picture. That's in case no
4839 * userspace component reacted to intermittent HPD dip.
4840 */
4841 intel_dp_check_link_status(intel_dp);
4842 } 4861 }
4843 4862
4844 /* 4863 /*
@@ -5054,7 +5073,12 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port,
5054 u8 *an) 5073 u8 *an)
5055{ 5074{
5056 struct intel_dp *intel_dp = enc_to_intel_dp(&intel_dig_port->base.base); 5075 struct intel_dp *intel_dp = enc_to_intel_dp(&intel_dig_port->base.base);
5057 uint8_t txbuf[4], rxbuf[2], reply = 0; 5076 static const struct drm_dp_aux_msg msg = {
5077 .request = DP_AUX_NATIVE_WRITE,
5078 .address = DP_AUX_HDCP_AKSV,
5079 .size = DRM_HDCP_KSV_LEN,
5080 };
5081 uint8_t txbuf[HEADER_SIZE + DRM_HDCP_KSV_LEN] = {}, rxbuf[2], reply = 0;
5058 ssize_t dpcd_ret; 5082 ssize_t dpcd_ret;
5059 int ret; 5083 int ret;
5060 5084
@@ -5072,14 +5096,11 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port,
5072 * we were writing the data, and then tickle the hardware to output the 5096 * we were writing the data, and then tickle the hardware to output the
5073 * data once the header is sent out. 5097 * data once the header is sent out.
5074 */ 5098 */
5075 txbuf[0] = (DP_AUX_NATIVE_WRITE << 4) | 5099 intel_dp_aux_header(txbuf, &msg);
5076 ((DP_AUX_HDCP_AKSV >> 16) & 0xf); 5100
5077 txbuf[1] = (DP_AUX_HDCP_AKSV >> 8) & 0xff; 5101 ret = intel_dp_aux_xfer(intel_dp, txbuf, HEADER_SIZE + msg.size,
5078 txbuf[2] = DP_AUX_HDCP_AKSV & 0xff; 5102 rxbuf, sizeof(rxbuf),
5079 txbuf[3] = DRM_HDCP_KSV_LEN - 1; 5103 DP_AUX_CH_CTL_AUX_AKSV_SELECT);
5080
5081 ret = intel_dp_aux_ch(intel_dp, txbuf, sizeof(txbuf), rxbuf,
5082 sizeof(rxbuf), true);
5083 if (ret < 0) { 5104 if (ret < 0) {
5084 DRM_ERROR("Write Aksv over DP/AUX failed (%d)\n", ret); 5105 DRM_ERROR("Write Aksv over DP/AUX failed (%d)\n", ret);
5085 return ret; 5106 return ret;
@@ -5413,37 +5434,10 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
5413 } 5434 }
5414 5435
5415 if (!intel_dp->is_mst) { 5436 if (!intel_dp->is_mst) {
5416 struct drm_modeset_acquire_ctx ctx; 5437 bool handled;
5417 struct drm_connector *connector = &intel_dp->attached_connector->base;
5418 struct drm_crtc *crtc;
5419 int iret;
5420 bool handled = false;
5421
5422 drm_modeset_acquire_init(&ctx, 0);
5423retry:
5424 iret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, &ctx);
5425 if (iret)
5426 goto err;
5427
5428 crtc = connector->state->crtc;
5429 if (crtc) {
5430 iret = drm_modeset_lock(&crtc->mutex, &ctx);
5431 if (iret)
5432 goto err;
5433 }
5434 5438
5435 handled = intel_dp_short_pulse(intel_dp); 5439 handled = intel_dp_short_pulse(intel_dp);
5436 5440
5437err:
5438 if (iret == -EDEADLK) {
5439 drm_modeset_backoff(&ctx);
5440 goto retry;
5441 }
5442
5443 drm_modeset_drop_locks(&ctx);
5444 drm_modeset_acquire_fini(&ctx);
5445 WARN(iret, "Acquiring modeset locks failed with %i\n", iret);
5446
5447 /* Short pulse can signify loss of hdcp authentication */ 5441 /* Short pulse can signify loss of hdcp authentication */
5448 intel_hdcp_check_link(intel_dp->attached_connector); 5442 intel_hdcp_check_link(intel_dp->attached_connector);
5449 5443
@@ -6266,42 +6260,6 @@ out_vdd_off:
6266 return false; 6260 return false;
6267} 6261}
6268 6262
6269/* Set up the hotplug pin and aux power domain. */
6270static void
6271intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
6272{
6273 struct intel_encoder *encoder = &intel_dig_port->base;
6274 struct intel_dp *intel_dp = &intel_dig_port->dp;
6275 struct intel_encoder *intel_encoder = &intel_dig_port->base;
6276 struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
6277
6278 encoder->hpd_pin = intel_hpd_pin_default(dev_priv, encoder->port);
6279
6280 switch (encoder->port) {
6281 case PORT_A:
6282 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_A;
6283 break;
6284 case PORT_B:
6285 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_B;
6286 break;
6287 case PORT_C:
6288 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_C;
6289 break;
6290 case PORT_D:
6291 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
6292 break;
6293 case PORT_E:
6294 /* FIXME: Check VBT for actual wiring of PORT E */
6295 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
6296 break;
6297 case PORT_F:
6298 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_F;
6299 break;
6300 default:
6301 MISSING_CASE(encoder->port);
6302 }
6303}
6304
6305static void intel_dp_modeset_retry_work_fn(struct work_struct *work) 6263static void intel_dp_modeset_retry_work_fn(struct work_struct *work)
6306{ 6264{
6307 struct intel_connector *intel_connector; 6265 struct intel_connector *intel_connector;
@@ -6353,20 +6311,6 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
6353 intel_dp->active_pipe = INVALID_PIPE; 6311 intel_dp->active_pipe = INVALID_PIPE;
6354 6312
6355 /* intel_dp vfuncs */ 6313 /* intel_dp vfuncs */
6356 if (INTEL_GEN(dev_priv) >= 9)
6357 intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
6358 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6359 intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
6360 else if (HAS_PCH_SPLIT(dev_priv))
6361 intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
6362 else
6363 intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
6364
6365 if (INTEL_GEN(dev_priv) >= 9)
6366 intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl;
6367 else
6368 intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
6369
6370 if (HAS_DDI(dev_priv)) 6314 if (HAS_DDI(dev_priv))
6371 intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain; 6315 intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain;
6372 6316
@@ -6407,7 +6351,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
6407 connector->interlace_allowed = true; 6351 connector->interlace_allowed = true;
6408 connector->doublescan_allowed = 0; 6352 connector->doublescan_allowed = 0;
6409 6353
6410 intel_dp_init_connector_port_info(intel_dig_port); 6354 intel_encoder->hpd_pin = intel_hpd_pin_default(dev_priv, port);
6411 6355
6412 intel_dp_aux_init(intel_dp); 6356 intel_dp_aux_init(intel_dp);
6413 6357
@@ -6484,6 +6428,7 @@ bool intel_dp_init(struct drm_i915_private *dev_priv,
6484 "DP %c", port_name(port))) 6428 "DP %c", port_name(port)))
6485 goto err_encoder_init; 6429 goto err_encoder_init;
6486 6430
6431 intel_encoder->hotplug = intel_dp_hotplug;
6487 intel_encoder->compute_config = intel_dp_compute_config; 6432 intel_encoder->compute_config = intel_dp_compute_config;
6488 intel_encoder->get_hw_state = intel_dp_get_hw_state; 6433 intel_encoder->get_hw_state = intel_dp_get_hw_state;
6489 intel_encoder->get_config = intel_dp_get_config; 6434 intel_encoder->get_config = intel_dp_get_config;
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index cf8fef8b6f58..f59b59bb0a21 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -248,6 +248,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
248 int tries; 248 int tries;
249 u32 training_pattern; 249 u32 training_pattern;
250 uint8_t link_status[DP_LINK_STATUS_SIZE]; 250 uint8_t link_status[DP_LINK_STATUS_SIZE];
251 bool channel_eq = false;
251 252
252 training_pattern = intel_dp_training_pattern(intel_dp); 253 training_pattern = intel_dp_training_pattern(intel_dp);
253 254
@@ -259,7 +260,6 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
259 return false; 260 return false;
260 } 261 }
261 262
262 intel_dp->channel_eq_status = false;
263 for (tries = 0; tries < 5; tries++) { 263 for (tries = 0; tries < 5; tries++) {
264 264
265 drm_dp_link_train_channel_eq_delay(intel_dp->dpcd); 265 drm_dp_link_train_channel_eq_delay(intel_dp->dpcd);
@@ -279,7 +279,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
279 279
280 if (drm_dp_channel_eq_ok(link_status, 280 if (drm_dp_channel_eq_ok(link_status,
281 intel_dp->lane_count)) { 281 intel_dp->lane_count)) {
282 intel_dp->channel_eq_status = true; 282 channel_eq = true;
283 DRM_DEBUG_KMS("Channel EQ done. DP Training " 283 DRM_DEBUG_KMS("Channel EQ done. DP Training "
284 "successful\n"); 284 "successful\n");
285 break; 285 break;
@@ -301,12 +301,14 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp)
301 301
302 intel_dp_set_idle_link_train(intel_dp); 302 intel_dp_set_idle_link_train(intel_dp);
303 303
304 return intel_dp->channel_eq_status; 304 return channel_eq;
305 305
306} 306}
307 307
308void intel_dp_stop_link_train(struct intel_dp *intel_dp) 308void intel_dp_stop_link_train(struct intel_dp *intel_dp)
309{ 309{
310 intel_dp->link_trained = true;
311
310 intel_dp_set_link_train(intel_dp, 312 intel_dp_set_link_train(intel_dp,
311 DP_TRAINING_PATTERN_DISABLE); 313 DP_TRAINING_PATTERN_DISABLE);
312} 314}
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 24e9caebe463..d4368589b355 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -215,7 +215,8 @@ struct intel_encoder {
215 enum intel_output_type type; 215 enum intel_output_type type;
216 enum port port; 216 enum port port;
217 unsigned int cloneable; 217 unsigned int cloneable;
218 void (*hot_plug)(struct intel_encoder *); 218 bool (*hotplug)(struct intel_encoder *encoder,
219 struct intel_connector *connector);
219 enum intel_output_type (*compute_output_type)(struct intel_encoder *, 220 enum intel_output_type (*compute_output_type)(struct intel_encoder *,
220 struct intel_crtc_state *, 221 struct intel_crtc_state *,
221 struct drm_connector_state *); 222 struct drm_connector_state *);
@@ -935,6 +936,7 @@ struct intel_plane {
935 enum plane_id id; 936 enum plane_id id;
936 enum pipe pipe; 937 enum pipe pipe;
937 bool can_scale; 938 bool can_scale;
939 bool has_fbc;
938 int max_downscale; 940 int max_downscale;
939 uint32_t frontbuffer_bit; 941 uint32_t frontbuffer_bit;
940 942
@@ -1041,17 +1043,16 @@ struct intel_dp_compliance {
1041 1043
1042struct intel_dp { 1044struct intel_dp {
1043 i915_reg_t output_reg; 1045 i915_reg_t output_reg;
1044 i915_reg_t aux_ch_ctl_reg;
1045 i915_reg_t aux_ch_data_reg[5];
1046 uint32_t DP; 1046 uint32_t DP;
1047 int link_rate; 1047 int link_rate;
1048 uint8_t lane_count; 1048 uint8_t lane_count;
1049 uint8_t sink_count; 1049 uint8_t sink_count;
1050 bool link_mst; 1050 bool link_mst;
1051 bool link_trained;
1051 bool has_audio; 1052 bool has_audio;
1052 bool detect_done; 1053 bool detect_done;
1053 bool channel_eq_status;
1054 bool reset_link_params; 1054 bool reset_link_params;
1055 enum aux_ch aux_ch;
1055 uint8_t dpcd[DP_RECEIVER_CAP_SIZE]; 1056 uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
1056 uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; 1057 uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
1057 uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; 1058 uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
@@ -1126,6 +1127,9 @@ struct intel_dp {
1126 int send_bytes, 1127 int send_bytes,
1127 uint32_t aux_clock_divider); 1128 uint32_t aux_clock_divider);
1128 1129
1130 i915_reg_t (*aux_ch_ctl_reg)(struct intel_dp *dp);
1131 i915_reg_t (*aux_ch_data_reg)(struct intel_dp *dp, int index);
1132
1129 /* This is called before a link training is starterd */ 1133 /* This is called before a link training is starterd */
1130 void (*prepare_link_retrain)(struct intel_dp *intel_dp); 1134 void (*prepare_link_retrain)(struct intel_dp *intel_dp);
1131 1135
@@ -1508,6 +1512,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
1508struct i915_vma * 1512struct i915_vma *
1509intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, 1513intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
1510 unsigned int rotation, 1514 unsigned int rotation,
1515 bool uses_fence,
1511 unsigned long *out_flags); 1516 unsigned long *out_flags);
1512void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags); 1517void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags);
1513struct drm_framebuffer * 1518struct drm_framebuffer *
@@ -1622,6 +1627,8 @@ int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
1622 int link_rate, uint8_t lane_count); 1627 int link_rate, uint8_t lane_count);
1623void intel_dp_start_link_train(struct intel_dp *intel_dp); 1628void intel_dp_start_link_train(struct intel_dp *intel_dp);
1624void intel_dp_stop_link_train(struct intel_dp *intel_dp); 1629void intel_dp_stop_link_train(struct intel_dp *intel_dp);
1630int intel_dp_retrain_link(struct intel_encoder *encoder,
1631 struct drm_modeset_acquire_ctx *ctx);
1625void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode); 1632void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
1626void intel_dp_encoder_reset(struct drm_encoder *encoder); 1633void intel_dp_encoder_reset(struct drm_encoder *encoder);
1627void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder); 1634void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
@@ -1701,7 +1708,8 @@ int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector);
1701void intel_dvo_init(struct drm_i915_private *dev_priv); 1708void intel_dvo_init(struct drm_i915_private *dev_priv);
1702/* intel_hotplug.c */ 1709/* intel_hotplug.c */
1703void intel_hpd_poll_init(struct drm_i915_private *dev_priv); 1710void intel_hpd_poll_init(struct drm_i915_private *dev_priv);
1704 1711bool intel_encoder_hotplug(struct intel_encoder *encoder,
1712 struct intel_connector *connector);
1705 1713
1706/* legacy fbdev emulation in intel_fbdev.c */ 1714/* legacy fbdev emulation in intel_fbdev.c */
1707#ifdef CONFIG_DRM_FBDEV_EMULATION 1715#ifdef CONFIG_DRM_FBDEV_EMULATION
@@ -1863,6 +1871,7 @@ bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port);
1863 1871
1864/* intel_psr.c */ 1872/* intel_psr.c */
1865#define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support) 1873#define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support)
1874void intel_psr_init_dpcd(struct intel_dp *intel_dp);
1866void intel_psr_enable(struct intel_dp *intel_dp, 1875void intel_psr_enable(struct intel_dp *intel_dp,
1867 const struct intel_crtc_state *crtc_state); 1876 const struct intel_crtc_state *crtc_state);
1868void intel_psr_disable(struct intel_dp *intel_dp, 1877void intel_psr_disable(struct intel_dp *intel_dp,
@@ -1989,8 +1998,7 @@ void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
1989void gen6_rps_busy(struct drm_i915_private *dev_priv); 1998void gen6_rps_busy(struct drm_i915_private *dev_priv);
1990void gen6_rps_reset_ei(struct drm_i915_private *dev_priv); 1999void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
1991void gen6_rps_idle(struct drm_i915_private *dev_priv); 2000void gen6_rps_idle(struct drm_i915_private *dev_priv);
1992void gen6_rps_boost(struct drm_i915_gem_request *rq, 2001void gen6_rps_boost(struct i915_request *rq, struct intel_rps_client *rps);
1993 struct intel_rps_client *rps);
1994void g4x_wm_get_hw_state(struct drm_device *dev); 2002void g4x_wm_get_hw_state(struct drm_device *dev);
1995void vlv_wm_get_hw_state(struct drm_device *dev); 2003void vlv_wm_get_hw_state(struct drm_device *dev);
1996void ilk_wm_get_hw_state(struct drm_device *dev); 2004void ilk_wm_get_hw_state(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index f3c5100d629e..4ba139c27fba 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -123,6 +123,22 @@ static const struct engine_info intel_engines[] = {
123 .mmio_base = GEN8_BSD2_RING_BASE, 123 .mmio_base = GEN8_BSD2_RING_BASE,
124 .irq_shift = GEN8_VCS2_IRQ_SHIFT, 124 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
125 }, 125 },
126 [VCS3] = {
127 .hw_id = VCS3_HW,
128 .uabi_id = I915_EXEC_BSD,
129 .class = VIDEO_DECODE_CLASS,
130 .instance = 2,
131 .mmio_base = GEN11_BSD3_RING_BASE,
132 .irq_shift = 0, /* not used */
133 },
134 [VCS4] = {
135 .hw_id = VCS4_HW,
136 .uabi_id = I915_EXEC_BSD,
137 .class = VIDEO_DECODE_CLASS,
138 .instance = 3,
139 .mmio_base = GEN11_BSD4_RING_BASE,
140 .irq_shift = 0, /* not used */
141 },
126 [VECS] = { 142 [VECS] = {
127 .hw_id = VECS_HW, 143 .hw_id = VECS_HW,
128 .uabi_id = I915_EXEC_VEBOX, 144 .uabi_id = I915_EXEC_VEBOX,
@@ -131,6 +147,14 @@ static const struct engine_info intel_engines[] = {
131 .mmio_base = VEBOX_RING_BASE, 147 .mmio_base = VEBOX_RING_BASE,
132 .irq_shift = GEN8_VECS_IRQ_SHIFT, 148 .irq_shift = GEN8_VECS_IRQ_SHIFT,
133 }, 149 },
150 [VECS2] = {
151 .hw_id = VECS2_HW,
152 .uabi_id = I915_EXEC_VEBOX,
153 .class = VIDEO_ENHANCEMENT_CLASS,
154 .instance = 1,
155 .mmio_base = GEN11_VEBOX2_RING_BASE,
156 .irq_shift = 0, /* not used */
157 },
134}; 158};
135 159
136/** 160/**
@@ -210,6 +234,9 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
210 GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes)); 234 GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
211 class_info = &intel_engine_classes[info->class]; 235 class_info = &intel_engine_classes[info->class];
212 236
237 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
238 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
239
213 if (GEM_WARN_ON(info->class > MAX_ENGINE_CLASS)) 240 if (GEM_WARN_ON(info->class > MAX_ENGINE_CLASS))
214 return -EINVAL; 241 return -EINVAL;
215 242
@@ -230,7 +257,25 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
230 class_info->name, info->instance) >= 257 class_info->name, info->instance) >=
231 sizeof(engine->name)); 258 sizeof(engine->name));
232 engine->hw_id = engine->guc_id = info->hw_id; 259 engine->hw_id = engine->guc_id = info->hw_id;
233 engine->mmio_base = info->mmio_base; 260 if (INTEL_GEN(dev_priv) >= 11) {
261 switch (engine->id) {
262 case VCS:
263 engine->mmio_base = GEN11_BSD_RING_BASE;
264 break;
265 case VCS2:
266 engine->mmio_base = GEN11_BSD2_RING_BASE;
267 break;
268 case VECS:
269 engine->mmio_base = GEN11_VEBOX_RING_BASE;
270 break;
271 default:
272 /* take the original value for all other engines */
273 engine->mmio_base = info->mmio_base;
274 break;
275 }
276 } else {
277 engine->mmio_base = info->mmio_base;
278 }
234 engine->irq_shift = info->irq_shift; 279 engine->irq_shift = info->irq_shift;
235 engine->class = info->class; 280 engine->class = info->class;
236 engine->instance = info->instance; 281 engine->instance = info->instance;
@@ -423,6 +468,7 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
423 BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists)); 468 BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
424 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); 469 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
425 470
471 execlists->queue_priority = INT_MIN;
426 execlists->queue = RB_ROOT; 472 execlists->queue = RB_ROOT;
427 execlists->first = NULL; 473 execlists->first = NULL;
428} 474}
@@ -1426,20 +1472,20 @@ int init_workarounds_ring(struct intel_engine_cs *engine)
1426 return 0; 1472 return 0;
1427} 1473}
1428 1474
1429int intel_ring_workarounds_emit(struct drm_i915_gem_request *req) 1475int intel_ring_workarounds_emit(struct i915_request *rq)
1430{ 1476{
1431 struct i915_workarounds *w = &req->i915->workarounds; 1477 struct i915_workarounds *w = &rq->i915->workarounds;
1432 u32 *cs; 1478 u32 *cs;
1433 int ret, i; 1479 int ret, i;
1434 1480
1435 if (w->count == 0) 1481 if (w->count == 0)
1436 return 0; 1482 return 0;
1437 1483
1438 ret = req->engine->emit_flush(req, EMIT_BARRIER); 1484 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
1439 if (ret) 1485 if (ret)
1440 return ret; 1486 return ret;
1441 1487
1442 cs = intel_ring_begin(req, (w->count * 2 + 2)); 1488 cs = intel_ring_begin(rq, w->count * 2 + 2);
1443 if (IS_ERR(cs)) 1489 if (IS_ERR(cs))
1444 return PTR_ERR(cs); 1490 return PTR_ERR(cs);
1445 1491
@@ -1450,9 +1496,9 @@ int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
1450 } 1496 }
1451 *cs++ = MI_NOOP; 1497 *cs++ = MI_NOOP;
1452 1498
1453 intel_ring_advance(req, cs); 1499 intel_ring_advance(rq, cs);
1454 1500
1455 ret = req->engine->emit_flush(req, EMIT_BARRIER); 1501 ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
1456 if (ret) 1502 if (ret)
1457 return ret; 1503 return ret;
1458 1504
@@ -1552,7 +1598,7 @@ bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
1552{ 1598{
1553 const struct i915_gem_context * const kernel_context = 1599 const struct i915_gem_context * const kernel_context =
1554 engine->i915->kernel_context; 1600 engine->i915->kernel_context;
1555 struct drm_i915_gem_request *rq; 1601 struct i915_request *rq;
1556 1602
1557 lockdep_assert_held(&engine->i915->drm.struct_mutex); 1603 lockdep_assert_held(&engine->i915->drm.struct_mutex);
1558 1604
@@ -1664,13 +1710,13 @@ unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1664} 1710}
1665 1711
1666static void print_request(struct drm_printer *m, 1712static void print_request(struct drm_printer *m,
1667 struct drm_i915_gem_request *rq, 1713 struct i915_request *rq,
1668 const char *prefix) 1714 const char *prefix)
1669{ 1715{
1670 drm_printf(m, "%s%x%s [%x:%x] prio=%d @ %dms: %s\n", prefix, 1716 drm_printf(m, "%s%x%s [%llx:%x] prio=%d @ %dms: %s\n", prefix,
1671 rq->global_seqno, 1717 rq->global_seqno,
1672 i915_gem_request_completed(rq) ? "!" : "", 1718 i915_request_completed(rq) ? "!" : "",
1673 rq->ctx->hw_id, rq->fence.seqno, 1719 rq->fence.context, rq->fence.seqno,
1674 rq->priotree.priority, 1720 rq->priotree.priority,
1675 jiffies_to_msecs(jiffies - rq->emitted_jiffies), 1721 jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1676 rq->timeline->common->name); 1722 rq->timeline->common->name);
@@ -1803,7 +1849,7 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
1803 1849
1804 rcu_read_lock(); 1850 rcu_read_lock();
1805 for (idx = 0; idx < execlists_num_ports(execlists); idx++) { 1851 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1806 struct drm_i915_gem_request *rq; 1852 struct i915_request *rq;
1807 unsigned int count; 1853 unsigned int count;
1808 1854
1809 rq = port_unpack(&execlists->port[idx], &count); 1855 rq = port_unpack(&execlists->port[idx], &count);
@@ -1837,7 +1883,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
1837 struct intel_breadcrumbs * const b = &engine->breadcrumbs; 1883 struct intel_breadcrumbs * const b = &engine->breadcrumbs;
1838 const struct intel_engine_execlists * const execlists = &engine->execlists; 1884 const struct intel_engine_execlists * const execlists = &engine->execlists;
1839 struct i915_gpu_error * const error = &engine->i915->gpu_error; 1885 struct i915_gpu_error * const error = &engine->i915->gpu_error;
1840 struct drm_i915_gem_request *rq; 1886 struct i915_request *rq;
1841 struct rb_node *rb; 1887 struct rb_node *rb;
1842 1888
1843 if (header) { 1889 if (header) {
@@ -1866,12 +1912,12 @@ void intel_engine_dump(struct intel_engine_cs *engine,
1866 drm_printf(m, "\tRequests:\n"); 1912 drm_printf(m, "\tRequests:\n");
1867 1913
1868 rq = list_first_entry(&engine->timeline->requests, 1914 rq = list_first_entry(&engine->timeline->requests,
1869 struct drm_i915_gem_request, link); 1915 struct i915_request, link);
1870 if (&rq->link != &engine->timeline->requests) 1916 if (&rq->link != &engine->timeline->requests)
1871 print_request(m, rq, "\t\tfirst "); 1917 print_request(m, rq, "\t\tfirst ");
1872 1918
1873 rq = list_last_entry(&engine->timeline->requests, 1919 rq = list_last_entry(&engine->timeline->requests,
1874 struct drm_i915_gem_request, link); 1920 struct i915_request, link);
1875 if (&rq->link != &engine->timeline->requests) 1921 if (&rq->link != &engine->timeline->requests)
1876 print_request(m, rq, "\t\tlast "); 1922 print_request(m, rq, "\t\tlast ");
1877 1923
@@ -1903,6 +1949,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
1903 spin_lock_irq(&engine->timeline->lock); 1949 spin_lock_irq(&engine->timeline->lock);
1904 list_for_each_entry(rq, &engine->timeline->requests, link) 1950 list_for_each_entry(rq, &engine->timeline->requests, link)
1905 print_request(m, rq, "\t\tE "); 1951 print_request(m, rq, "\t\tE ");
1952 drm_printf(m, "\t\tQueue priority: %d\n", execlists->queue_priority);
1906 for (rb = execlists->first; rb; rb = rb_next(rb)) { 1953 for (rb = execlists->first; rb; rb = rb_next(rb)) {
1907 struct i915_priolist *p = 1954 struct i915_priolist *p =
1908 rb_entry(rb, typeof(*p), node); 1955 rb_entry(rb, typeof(*p), node);
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index f66f6fb5743d..707d49c12638 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -46,16 +46,6 @@ static inline bool fbc_supported(struct drm_i915_private *dev_priv)
46 return HAS_FBC(dev_priv); 46 return HAS_FBC(dev_priv);
47} 47}
48 48
49static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv)
50{
51 return IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8;
52}
53
54static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv)
55{
56 return INTEL_GEN(dev_priv) < 4;
57}
58
59static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv) 49static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv)
60{ 50{
61 return INTEL_GEN(dev_priv) <= 3; 51 return INTEL_GEN(dev_priv) <= 3;
@@ -819,6 +809,12 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
819 * Note that is possible for a tiled surface to be unmappable (and 809 * Note that is possible for a tiled surface to be unmappable (and
820 * so have no fence associated with it) due to aperture constaints 810 * so have no fence associated with it) due to aperture constaints
821 * at the time of pinning. 811 * at the time of pinning.
812 *
813 * FIXME with 90/270 degree rotation we should use the fence on
814 * the normal GTT view (the rotated view doesn't even have a
815 * fence). Would need changes to the FBC fence Y offset as well.
816 * For now this will effecively disable FBC with 90/270 degree
817 * rotation.
822 */ 818 */
823 if (!(cache->flags & PLANE_HAS_FENCE)) { 819 if (!(cache->flags & PLANE_HAS_FENCE)) {
824 fbc->no_fbc_reason = "framebuffer not tiled or fenced"; 820 fbc->no_fbc_reason = "framebuffer not tiled or fenced";
@@ -863,6 +859,17 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
863 return false; 859 return false;
864 } 860 }
865 861
862 /*
863 * Work around a problem on GEN9+ HW, where enabling FBC on a plane
864 * having a Y offset that isn't divisible by 4 causes FIFO underrun
865 * and screen flicker.
866 */
867 if (IS_GEN(dev_priv, 9, 10) &&
868 (fbc->state_cache.plane.adjusted_y & 3)) {
869 fbc->no_fbc_reason = "plane Y offset is misaligned";
870 return false;
871 }
872
866 return true; 873 return true;
867} 874}
868 875
@@ -953,6 +960,30 @@ unlock:
953 mutex_unlock(&fbc->lock); 960 mutex_unlock(&fbc->lock);
954} 961}
955 962
963/**
964 * __intel_fbc_disable - disable FBC
965 * @dev_priv: i915 device instance
966 *
967 * This is the low level function that actually disables FBC. Callers should
968 * grab the FBC lock.
969 */
970static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
971{
972 struct intel_fbc *fbc = &dev_priv->fbc;
973 struct intel_crtc *crtc = fbc->crtc;
974
975 WARN_ON(!mutex_is_locked(&fbc->lock));
976 WARN_ON(!fbc->enabled);
977 WARN_ON(fbc->active);
978
979 DRM_DEBUG_KMS("Disabling FBC on pipe %c\n", pipe_name(crtc->pipe));
980
981 __intel_fbc_cleanup_cfb(dev_priv);
982
983 fbc->enabled = false;
984 fbc->crtc = NULL;
985}
986
956static void __intel_fbc_post_update(struct intel_crtc *crtc) 987static void __intel_fbc_post_update(struct intel_crtc *crtc)
957{ 988{
958 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); 989 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
@@ -964,6 +995,13 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
964 if (!fbc->enabled || fbc->crtc != crtc) 995 if (!fbc->enabled || fbc->crtc != crtc)
965 return; 996 return;
966 997
998 if (!i915_modparams.enable_fbc) {
999 intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");
1000 __intel_fbc_disable(dev_priv);
1001
1002 return;
1003 }
1004
967 if (!intel_fbc_can_activate(crtc)) { 1005 if (!intel_fbc_can_activate(crtc)) {
968 WARN_ON(fbc->active); 1006 WARN_ON(fbc->active);
969 return; 1007 return;
@@ -1094,13 +1132,10 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
1094 struct intel_crtc_state *crtc_state; 1132 struct intel_crtc_state *crtc_state;
1095 struct intel_crtc *crtc = to_intel_crtc(plane_state->base.crtc); 1133 struct intel_crtc *crtc = to_intel_crtc(plane_state->base.crtc);
1096 1134
1097 if (!plane_state->base.visible) 1135 if (!plane->has_fbc)
1098 continue;
1099
1100 if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A)
1101 continue; 1136 continue;
1102 1137
1103 if (fbc_on_plane_a_only(dev_priv) && plane->i9xx_plane != PLANE_A) 1138 if (!plane_state->base.visible)
1104 continue; 1139 continue;
1105 1140
1106 crtc_state = intel_atomic_get_new_crtc_state(state, crtc); 1141 crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
@@ -1171,31 +1206,6 @@ out:
1171} 1206}
1172 1207
1173/** 1208/**
1174 * __intel_fbc_disable - disable FBC
1175 * @dev_priv: i915 device instance
1176 *
1177 * This is the low level function that actually disables FBC. Callers should
1178 * grab the FBC lock.
1179 */
1180static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
1181{
1182 struct intel_fbc *fbc = &dev_priv->fbc;
1183 struct intel_crtc *crtc = fbc->crtc;
1184
1185 WARN_ON(!mutex_is_locked(&fbc->lock));
1186 WARN_ON(!fbc->enabled);
1187 WARN_ON(fbc->active);
1188 WARN_ON(crtc->active);
1189
1190 DRM_DEBUG_KMS("Disabling FBC on pipe %c\n", pipe_name(crtc->pipe));
1191
1192 __intel_fbc_cleanup_cfb(dev_priv);
1193
1194 fbc->enabled = false;
1195 fbc->crtc = NULL;
1196}
1197
1198/**
1199 * intel_fbc_disable - disable FBC if it's associated with crtc 1209 * intel_fbc_disable - disable FBC if it's associated with crtc
1200 * @crtc: the CRTC 1210 * @crtc: the CRTC
1201 * 1211 *
@@ -1209,6 +1219,8 @@ void intel_fbc_disable(struct intel_crtc *crtc)
1209 if (!fbc_supported(dev_priv)) 1219 if (!fbc_supported(dev_priv))
1210 return; 1220 return;
1211 1221
1222 WARN_ON(crtc->active);
1223
1212 mutex_lock(&fbc->lock); 1224 mutex_lock(&fbc->lock);
1213 if (fbc->crtc == crtc) 1225 if (fbc->crtc == crtc)
1214 __intel_fbc_disable(dev_priv); 1226 __intel_fbc_disable(dev_priv);
@@ -1231,8 +1243,10 @@ void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
1231 return; 1243 return;
1232 1244
1233 mutex_lock(&fbc->lock); 1245 mutex_lock(&fbc->lock);
1234 if (fbc->enabled) 1246 if (fbc->enabled) {
1247 WARN_ON(fbc->crtc->active);
1235 __intel_fbc_disable(dev_priv); 1248 __intel_fbc_disable(dev_priv);
1249 }
1236 mutex_unlock(&fbc->lock); 1250 mutex_unlock(&fbc->lock);
1237 1251
1238 cancel_work_sync(&fbc->work.work); 1252 cancel_work_sync(&fbc->work.work);
@@ -1357,7 +1371,6 @@ static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv)
1357void intel_fbc_init(struct drm_i915_private *dev_priv) 1371void intel_fbc_init(struct drm_i915_private *dev_priv)
1358{ 1372{
1359 struct intel_fbc *fbc = &dev_priv->fbc; 1373 struct intel_fbc *fbc = &dev_priv->fbc;
1360 enum pipe pipe;
1361 1374
1362 INIT_WORK(&fbc->work.work, intel_fbc_work_fn); 1375 INIT_WORK(&fbc->work.work, intel_fbc_work_fn);
1363 INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn); 1376 INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
@@ -1378,14 +1391,6 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
1378 return; 1391 return;
1379 } 1392 }
1380 1393
1381 for_each_pipe(dev_priv, pipe) {
1382 fbc->possible_framebuffer_bits |=
1383 INTEL_FRONTBUFFER(pipe, PLANE_PRIMARY);
1384
1385 if (fbc_on_pipe_a_only(dev_priv))
1386 break;
1387 }
1388
1389 /* This value was pulled out of someone's hat */ 1394 /* This value was pulled out of someone's hat */
1390 if (INTEL_GEN(dev_priv) <= 4 && !IS_GM45(dev_priv)) 1395 if (INTEL_GEN(dev_priv) <= 4 && !IS_GM45(dev_priv))
1391 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); 1396 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 055f409f8b75..6f12adc06365 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -215,7 +215,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
215 */ 215 */
216 vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, 216 vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base,
217 DRM_MODE_ROTATE_0, 217 DRM_MODE_ROTATE_0,
218 &flags); 218 false, &flags);
219 if (IS_ERR(vma)) { 219 if (IS_ERR(vma)) {
220 ret = PTR_ERR(vma); 220 ret = PTR_ERR(vma);
221 goto out_unlock; 221 goto out_unlock;
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c
index fcfc217e754e..3a8d3d06c26a 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.c
@@ -79,6 +79,7 @@ void __intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
79 spin_unlock(&dev_priv->fb_tracking.lock); 79 spin_unlock(&dev_priv->fb_tracking.lock);
80 } 80 }
81 81
82 might_sleep();
82 intel_psr_invalidate(dev_priv, frontbuffer_bits); 83 intel_psr_invalidate(dev_priv, frontbuffer_bits);
83 intel_edp_drrs_invalidate(dev_priv, frontbuffer_bits); 84 intel_edp_drrs_invalidate(dev_priv, frontbuffer_bits);
84 intel_fbc_invalidate(dev_priv, frontbuffer_bits, origin); 85 intel_fbc_invalidate(dev_priv, frontbuffer_bits, origin);
@@ -108,6 +109,7 @@ static void intel_frontbuffer_flush(struct drm_i915_private *dev_priv,
108 if (!frontbuffer_bits) 109 if (!frontbuffer_bits)
109 return; 110 return;
110 111
112 might_sleep();
111 intel_edp_drrs_flush(dev_priv, frontbuffer_bits); 113 intel_edp_drrs_flush(dev_priv, frontbuffer_bits);
112 intel_psr_flush(dev_priv, frontbuffer_bits, origin); 114 intel_psr_flush(dev_priv, frontbuffer_bits, origin);
113 intel_fbc_flush(dev_priv, frontbuffer_bits, origin); 115 intel_fbc_flush(dev_priv, frontbuffer_bits, origin);
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 21140ccd7a97..ff08ea0ebf49 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -370,7 +370,7 @@ int intel_guc_sample_forcewake(struct intel_guc *guc)
370 u32 action[2]; 370 u32 action[2];
371 371
372 action[0] = INTEL_GUC_ACTION_SAMPLE_FORCEWAKE; 372 action[0] = INTEL_GUC_ACTION_SAMPLE_FORCEWAKE;
373 /* WaRsDisableCoarsePowerGating:skl,bxt */ 373 /* WaRsDisableCoarsePowerGating:skl,cnl */
374 if (!HAS_RC6(dev_priv) || NEEDS_WaRsDisableCoarsePowerGating(dev_priv)) 374 if (!HAS_RC6(dev_priv) || NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
375 action[1] = 0; 375 action[1] = 0;
376 else 376 else
@@ -403,22 +403,15 @@ int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset)
403 403
404/** 404/**
405 * intel_guc_suspend() - notify GuC entering suspend state 405 * intel_guc_suspend() - notify GuC entering suspend state
406 * @dev_priv: i915 device private 406 * @guc: the guc
407 */ 407 */
408int intel_guc_suspend(struct drm_i915_private *dev_priv) 408int intel_guc_suspend(struct intel_guc *guc)
409{ 409{
410 struct intel_guc *guc = &dev_priv->guc; 410 u32 data[] = {
411 u32 data[3]; 411 INTEL_GUC_ACTION_ENTER_S_STATE,
412 412 GUC_POWER_D1, /* any value greater than GUC_POWER_D0 */
413 if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS) 413 guc_ggtt_offset(guc->shared_data)
414 return 0; 414 };
415
416 gen9_disable_guc_interrupts(dev_priv);
417
418 data[0] = INTEL_GUC_ACTION_ENTER_S_STATE;
419 /* any value greater than GUC_POWER_D0 */
420 data[1] = GUC_POWER_D1;
421 data[2] = guc_ggtt_offset(guc->shared_data);
422 415
423 return intel_guc_send(guc, data, ARRAY_SIZE(data)); 416 return intel_guc_send(guc, data, ARRAY_SIZE(data));
424} 417}
@@ -448,22 +441,15 @@ int intel_guc_reset_engine(struct intel_guc *guc,
448 441
449/** 442/**
450 * intel_guc_resume() - notify GuC resuming from suspend state 443 * intel_guc_resume() - notify GuC resuming from suspend state
451 * @dev_priv: i915 device private 444 * @guc: the guc
452 */ 445 */
453int intel_guc_resume(struct drm_i915_private *dev_priv) 446int intel_guc_resume(struct intel_guc *guc)
454{ 447{
455 struct intel_guc *guc = &dev_priv->guc; 448 u32 data[] = {
456 u32 data[3]; 449 INTEL_GUC_ACTION_EXIT_S_STATE,
457 450 GUC_POWER_D0,
458 if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS) 451 guc_ggtt_offset(guc->shared_data)
459 return 0; 452 };
460
461 if (i915_modparams.guc_log_level)
462 gen9_enable_guc_interrupts(dev_priv);
463
464 data[0] = INTEL_GUC_ACTION_EXIT_S_STATE;
465 data[1] = GUC_POWER_D0;
466 data[2] = guc_ggtt_offset(guc->shared_data);
467 453
468 return intel_guc_send(guc, data, ARRAY_SIZE(data)); 454 return intel_guc_send(guc, data, ARRAY_SIZE(data));
469} 455}
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 52856a97477d..b9424ac644ac 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -127,8 +127,8 @@ int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len);
127int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len); 127int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len);
128int intel_guc_sample_forcewake(struct intel_guc *guc); 128int intel_guc_sample_forcewake(struct intel_guc *guc);
129int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset); 129int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
130int intel_guc_suspend(struct drm_i915_private *dev_priv); 130int intel_guc_suspend(struct intel_guc *guc);
131int intel_guc_resume(struct drm_i915_private *dev_priv); 131int intel_guc_resume(struct intel_guc *guc);
132struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size); 132struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
133u32 intel_guc_wopcm_size(struct drm_i915_private *dev_priv); 133u32 intel_guc_wopcm_size(struct drm_i915_private *dev_priv);
134 134
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c
index 3b0932942857..d07f2b985f1c 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/intel_guc_fw.c
@@ -269,15 +269,15 @@ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
269} 269}
270 270
271/** 271/**
272 * intel_guc_fw_upload() - finish preparing the GuC for activity 272 * intel_guc_fw_upload() - load GuC uCode to device
273 * @guc: intel_guc structure 273 * @guc: intel_guc structure
274 * 274 *
275 * Called during driver loading and also after a GPU reset. 275 * Called from intel_uc_init_hw() during driver load, resume from sleep and
276 * after a GPU reset.
276 * 277 *
277 * The main action required here it to load the GuC uCode into the device.
278 * The firmware image should have already been fetched into memory by the 278 * The firmware image should have already been fetched into memory by the
279 * earlier call to intel_guc_init(), so here we need only check that 279 * earlier call to intel_uc_init_fw(), so here we need to only check that
280 * worked, and then transfer the image to the h/w. 280 * fetch succeeded, and then transfer the image to the h/w.
281 * 281 *
282 * Return: non-zero code on error 282 * Return: non-zero code on error
283 */ 283 */
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index 7b5074e2120c..c0c2e7d1c7d7 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -61,8 +61,10 @@ static int guc_log_flush(struct intel_guc *guc)
61static int guc_log_control(struct intel_guc *guc, bool enable, u32 verbosity) 61static int guc_log_control(struct intel_guc *guc, bool enable, u32 verbosity)
62{ 62{
63 union guc_log_control control_val = { 63 union guc_log_control control_val = {
64 .logging_enabled = enable, 64 {
65 .verbosity = verbosity, 65 .logging_enabled = enable,
66 .verbosity = verbosity,
67 },
66 }; 68 };
67 u32 action[] = { 69 u32 action[] = {
68 INTEL_GUC_ACTION_UK_LOG_ENABLE_LOGGING, 70 INTEL_GUC_ACTION_UK_LOG_ENABLE_LOGGING,
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index 946766b62459..8a8ad2fe158d 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -26,8 +26,14 @@
26#include <trace/events/dma_fence.h> 26#include <trace/events/dma_fence.h>
27 27
28#include "intel_guc_submission.h" 28#include "intel_guc_submission.h"
29#include "intel_lrc_reg.h"
29#include "i915_drv.h" 30#include "i915_drv.h"
30 31
32#define GUC_PREEMPT_FINISHED 0x1
33#define GUC_PREEMPT_BREADCRUMB_DWORDS 0x8
34#define GUC_PREEMPT_BREADCRUMB_BYTES \
35 (sizeof(u32) * GUC_PREEMPT_BREADCRUMB_DWORDS)
36
31/** 37/**
32 * DOC: GuC-based command submission 38 * DOC: GuC-based command submission
33 * 39 *
@@ -75,6 +81,11 @@
75 * 81 *
76 */ 82 */
77 83
84static inline struct i915_priolist *to_priolist(struct rb_node *rb)
85{
86 return rb_entry(rb, struct i915_priolist, node);
87}
88
78static inline bool is_high_priority(struct intel_guc_client *client) 89static inline bool is_high_priority(struct intel_guc_client *client)
79{ 90{
80 return (client->priority == GUC_CLIENT_PRIORITY_KMD_HIGH || 91 return (client->priority == GUC_CLIENT_PRIORITY_KMD_HIGH ||
@@ -496,8 +507,7 @@ static void guc_ring_doorbell(struct intel_guc_client *client)
496 GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED); 507 GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED);
497} 508}
498 509
499static void guc_add_request(struct intel_guc *guc, 510static void guc_add_request(struct intel_guc *guc, struct i915_request *rq)
500 struct drm_i915_gem_request *rq)
501{ 511{
502 struct intel_guc_client *client = guc->execbuf_client; 512 struct intel_guc_client *client = guc->execbuf_client;
503 struct intel_engine_cs *engine = rq->engine; 513 struct intel_engine_cs *engine = rq->engine;
@@ -531,8 +541,6 @@ static void flush_ggtt_writes(struct i915_vma *vma)
531 POSTING_READ_FW(GUC_STATUS); 541 POSTING_READ_FW(GUC_STATUS);
532} 542}
533 543
534#define GUC_PREEMPT_FINISHED 0x1
535#define GUC_PREEMPT_BREADCRUMB_DWORDS 0x8
536static void inject_preempt_context(struct work_struct *work) 544static void inject_preempt_context(struct work_struct *work)
537{ 545{
538 struct guc_preempt_work *preempt_work = 546 struct guc_preempt_work *preempt_work =
@@ -542,37 +550,17 @@ static void inject_preempt_context(struct work_struct *work)
542 preempt_work[engine->id]); 550 preempt_work[engine->id]);
543 struct intel_guc_client *client = guc->preempt_client; 551 struct intel_guc_client *client = guc->preempt_client;
544 struct guc_stage_desc *stage_desc = __get_stage_desc(client); 552 struct guc_stage_desc *stage_desc = __get_stage_desc(client);
545 struct intel_ring *ring = client->owner->engine[engine->id].ring;
546 u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(client->owner, 553 u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(client->owner,
547 engine)); 554 engine));
548 u32 *cs = ring->vaddr + ring->tail;
549 u32 data[7]; 555 u32 data[7];
550 556
551 if (engine->id == RCS) { 557 /*
552 cs = gen8_emit_ggtt_write_rcs(cs, GUC_PREEMPT_FINISHED, 558 * The ring contains commands to write GUC_PREEMPT_FINISHED into HWSP.
553 intel_hws_preempt_done_address(engine)); 559 * See guc_fill_preempt_context().
554 } else { 560 */
555 cs = gen8_emit_ggtt_write(cs, GUC_PREEMPT_FINISHED,
556 intel_hws_preempt_done_address(engine));
557 *cs++ = MI_NOOP;
558 *cs++ = MI_NOOP;
559 }
560 *cs++ = MI_USER_INTERRUPT;
561 *cs++ = MI_NOOP;
562
563 GEM_BUG_ON(!IS_ALIGNED(ring->size,
564 GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32)));
565 GEM_BUG_ON((void *)cs - (ring->vaddr + ring->tail) !=
566 GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32));
567
568 ring->tail += GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32);
569 ring->tail &= (ring->size - 1);
570
571 flush_ggtt_writes(ring->vma);
572
573 spin_lock_irq(&client->wq_lock); 561 spin_lock_irq(&client->wq_lock);
574 guc_wq_item_append(client, engine->guc_id, ctx_desc, 562 guc_wq_item_append(client, engine->guc_id, ctx_desc,
575 ring->tail / sizeof(u64), 0); 563 GUC_PREEMPT_BREADCRUMB_BYTES / sizeof(u64), 0);
576 spin_unlock_irq(&client->wq_lock); 564 spin_unlock_irq(&client->wq_lock);
577 565
578 /* 566 /*
@@ -648,7 +636,7 @@ static void guc_submit(struct intel_engine_cs *engine)
648 unsigned int n; 636 unsigned int n;
649 637
650 for (n = 0; n < execlists_num_ports(execlists); n++) { 638 for (n = 0; n < execlists_num_ports(execlists); n++) {
651 struct drm_i915_gem_request *rq; 639 struct i915_request *rq;
652 unsigned int count; 640 unsigned int count;
653 641
654 rq = port_unpack(&port[n], &count); 642 rq = port_unpack(&port[n], &count);
@@ -662,19 +650,18 @@ static void guc_submit(struct intel_engine_cs *engine)
662 } 650 }
663} 651}
664 652
665static void port_assign(struct execlist_port *port, 653static void port_assign(struct execlist_port *port, struct i915_request *rq)
666 struct drm_i915_gem_request *rq)
667{ 654{
668 GEM_BUG_ON(port_isset(port)); 655 GEM_BUG_ON(port_isset(port));
669 656
670 port_set(port, i915_gem_request_get(rq)); 657 port_set(port, i915_request_get(rq));
671} 658}
672 659
673static void guc_dequeue(struct intel_engine_cs *engine) 660static void guc_dequeue(struct intel_engine_cs *engine)
674{ 661{
675 struct intel_engine_execlists * const execlists = &engine->execlists; 662 struct intel_engine_execlists * const execlists = &engine->execlists;
676 struct execlist_port *port = execlists->port; 663 struct execlist_port *port = execlists->port;
677 struct drm_i915_gem_request *last = NULL; 664 struct i915_request *last = NULL;
678 const struct execlist_port * const last_port = 665 const struct execlist_port * const last_port =
679 &execlists->port[execlists->port_mask]; 666 &execlists->port[execlists->port_mask];
680 bool submit = false; 667 bool submit = false;
@@ -684,15 +671,12 @@ static void guc_dequeue(struct intel_engine_cs *engine)
684 rb = execlists->first; 671 rb = execlists->first;
685 GEM_BUG_ON(rb_first(&execlists->queue) != rb); 672 GEM_BUG_ON(rb_first(&execlists->queue) != rb);
686 673
687 if (!rb)
688 goto unlock;
689
690 if (port_isset(port)) { 674 if (port_isset(port)) {
691 if (engine->i915->preempt_context) { 675 if (engine->i915->preempt_context) {
692 struct guc_preempt_work *preempt_work = 676 struct guc_preempt_work *preempt_work =
693 &engine->i915->guc.preempt_work[engine->id]; 677 &engine->i915->guc.preempt_work[engine->id];
694 678
695 if (rb_entry(rb, struct i915_priolist, node)->priority > 679 if (execlists->queue_priority >
696 max(port_request(port)->priotree.priority, 0)) { 680 max(port_request(port)->priotree.priority, 0)) {
697 execlists_set_active(execlists, 681 execlists_set_active(execlists,
698 EXECLISTS_ACTIVE_PREEMPT); 682 EXECLISTS_ACTIVE_PREEMPT);
@@ -708,9 +692,9 @@ static void guc_dequeue(struct intel_engine_cs *engine)
708 } 692 }
709 GEM_BUG_ON(port_isset(port)); 693 GEM_BUG_ON(port_isset(port));
710 694
711 do { 695 while (rb) {
712 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 696 struct i915_priolist *p = to_priolist(rb);
713 struct drm_i915_gem_request *rq, *rn; 697 struct i915_request *rq, *rn;
714 698
715 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 699 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
716 if (last && rq->ctx != last->ctx) { 700 if (last && rq->ctx != last->ctx) {
@@ -727,9 +711,8 @@ static void guc_dequeue(struct intel_engine_cs *engine)
727 711
728 INIT_LIST_HEAD(&rq->priotree.link); 712 INIT_LIST_HEAD(&rq->priotree.link);
729 713
730 __i915_gem_request_submit(rq); 714 __i915_request_submit(rq);
731 trace_i915_gem_request_in(rq, 715 trace_i915_request_in(rq, port_index(port, execlists));
732 port_index(port, execlists));
733 last = rq; 716 last = rq;
734 submit = true; 717 submit = true;
735 } 718 }
@@ -739,8 +722,9 @@ static void guc_dequeue(struct intel_engine_cs *engine)
739 INIT_LIST_HEAD(&p->requests); 722 INIT_LIST_HEAD(&p->requests);
740 if (p->priority != I915_PRIORITY_NORMAL) 723 if (p->priority != I915_PRIORITY_NORMAL)
741 kmem_cache_free(engine->i915->priorities, p); 724 kmem_cache_free(engine->i915->priorities, p);
742 } while (rb); 725 }
743done: 726done:
727 execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
744 execlists->first = rb; 728 execlists->first = rb;
745 if (submit) { 729 if (submit) {
746 port_assign(port, last); 730 port_assign(port, last);
@@ -762,12 +746,12 @@ static void guc_submission_tasklet(unsigned long data)
762 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; 746 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
763 struct intel_engine_execlists * const execlists = &engine->execlists; 747 struct intel_engine_execlists * const execlists = &engine->execlists;
764 struct execlist_port *port = execlists->port; 748 struct execlist_port *port = execlists->port;
765 struct drm_i915_gem_request *rq; 749 struct i915_request *rq;
766 750
767 rq = port_request(&port[0]); 751 rq = port_request(&port[0]);
768 while (rq && i915_gem_request_completed(rq)) { 752 while (rq && i915_request_completed(rq)) {
769 trace_i915_gem_request_out(rq); 753 trace_i915_request_out(rq);
770 i915_gem_request_put(rq); 754 i915_request_put(rq);
771 755
772 execlists_port_complete(execlists, port); 756 execlists_port_complete(execlists, port);
773 757
@@ -972,6 +956,62 @@ static void guc_client_free(struct intel_guc_client *client)
972 kfree(client); 956 kfree(client);
973} 957}
974 958
959static inline bool ctx_save_restore_disabled(struct intel_context *ce)
960{
961 u32 sr = ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1];
962
963#define SR_DISABLED \
964 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | \
965 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)
966
967 return (sr & SR_DISABLED) == SR_DISABLED;
968
969#undef SR_DISABLED
970}
971
972static void guc_fill_preempt_context(struct intel_guc *guc)
973{
974 struct drm_i915_private *dev_priv = guc_to_i915(guc);
975 struct intel_guc_client *client = guc->preempt_client;
976 struct intel_engine_cs *engine;
977 enum intel_engine_id id;
978
979 for_each_engine(engine, dev_priv, id) {
980 struct intel_context *ce = &client->owner->engine[id];
981 u32 addr = intel_hws_preempt_done_address(engine);
982 u32 *cs;
983
984 GEM_BUG_ON(!ce->pin_count);
985
986 /*
987 * We rely on this context image *not* being saved after
988 * preemption. This ensures that the RING_HEAD / RING_TAIL
989 * remain pointing at initial values forever.
990 */
991 GEM_BUG_ON(!ctx_save_restore_disabled(ce));
992
993 cs = ce->ring->vaddr;
994 if (id == RCS) {
995 cs = gen8_emit_ggtt_write_rcs(cs,
996 GUC_PREEMPT_FINISHED,
997 addr);
998 } else {
999 cs = gen8_emit_ggtt_write(cs,
1000 GUC_PREEMPT_FINISHED,
1001 addr);
1002 *cs++ = MI_NOOP;
1003 *cs++ = MI_NOOP;
1004 }
1005 *cs++ = MI_USER_INTERRUPT;
1006 *cs++ = MI_NOOP;
1007
1008 GEM_BUG_ON((void *)cs - ce->ring->vaddr !=
1009 GUC_PREEMPT_BREADCRUMB_BYTES);
1010
1011 flush_ggtt_writes(ce->ring->vma);
1012 }
1013}
1014
975static int guc_clients_create(struct intel_guc *guc) 1015static int guc_clients_create(struct intel_guc *guc)
976{ 1016{
977 struct drm_i915_private *dev_priv = guc_to_i915(guc); 1017 struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -1002,6 +1042,8 @@ static int guc_clients_create(struct intel_guc *guc)
1002 return PTR_ERR(client); 1042 return PTR_ERR(client);
1003 } 1043 }
1004 guc->preempt_client = client; 1044 guc->preempt_client = client;
1045
1046 guc_fill_preempt_context(guc);
1005 } 1047 }
1006 1048
1007 return 0; 1049 return 0;
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index f5d7bfb43006..1baef4ac7ecb 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -2383,6 +2383,7 @@ void intel_hdmi_init(struct drm_i915_private *dev_priv,
2383 &intel_hdmi_enc_funcs, DRM_MODE_ENCODER_TMDS, 2383 &intel_hdmi_enc_funcs, DRM_MODE_ENCODER_TMDS,
2384 "HDMI %c", port_name(port)); 2384 "HDMI %c", port_name(port));
2385 2385
2386 intel_encoder->hotplug = intel_encoder_hotplug;
2386 intel_encoder->compute_config = intel_hdmi_compute_config; 2387 intel_encoder->compute_config = intel_hdmi_compute_config;
2387 if (HAS_PCH_SPLIT(dev_priv)) { 2388 if (HAS_PCH_SPLIT(dev_priv)) {
2388 intel_encoder->disable = pch_disable_hdmi; 2389 intel_encoder->disable = pch_disable_hdmi;
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index fe28c1ea84a5..0e3d3e89d66a 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -274,24 +274,26 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
274 intel_runtime_pm_put(dev_priv); 274 intel_runtime_pm_put(dev_priv);
275} 275}
276 276
277static bool intel_hpd_irq_event(struct drm_device *dev, 277bool intel_encoder_hotplug(struct intel_encoder *encoder,
278 struct drm_connector *connector) 278 struct intel_connector *connector)
279{ 279{
280 struct drm_device *dev = connector->base.dev;
280 enum drm_connector_status old_status; 281 enum drm_connector_status old_status;
281 282
282 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); 283 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
283 old_status = connector->status; 284 old_status = connector->base.status;
284 285
285 connector->status = drm_helper_probe_detect(connector, NULL, false); 286 connector->base.status =
287 drm_helper_probe_detect(&connector->base, NULL, false);
286 288
287 if (old_status == connector->status) 289 if (old_status == connector->base.status)
288 return false; 290 return false;
289 291
290 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n", 292 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
291 connector->base.id, 293 connector->base.base.id,
292 connector->name, 294 connector->base.name,
293 drm_get_connector_status_name(old_status), 295 drm_get_connector_status_name(old_status),
294 drm_get_connector_status_name(connector->status)); 296 drm_get_connector_status_name(connector->base.status));
295 297
296 return true; 298 return true;
297} 299}
@@ -381,10 +383,9 @@ static void i915_hotplug_work_func(struct work_struct *work)
381 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) { 383 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
382 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n", 384 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
383 connector->name, intel_encoder->hpd_pin); 385 connector->name, intel_encoder->hpd_pin);
384 if (intel_encoder->hot_plug) 386
385 intel_encoder->hot_plug(intel_encoder); 387 changed |= intel_encoder->hotplug(intel_encoder,
386 if (intel_hpd_irq_event(dev, connector)) 388 intel_connector);
387 changed = true;
388 } 389 }
389 } 390 }
390 drm_connector_list_iter_end(&conn_iter); 391 drm_connector_list_iter_end(&conn_iter);
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index ef9a05d8e5a9..65e2afb9b955 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -27,161 +27,9 @@
27#include "intel_huc.h" 27#include "intel_huc.h"
28#include "i915_drv.h" 28#include "i915_drv.h"
29 29
30/**
31 * DOC: HuC Firmware
32 *
33 * Motivation:
34 * GEN9 introduces a new dedicated firmware for usage in media HEVC (High
35 * Efficiency Video Coding) operations. Userspace can use the firmware
36 * capabilities by adding HuC specific commands to batch buffers.
37 *
38 * Implementation:
39 * The same firmware loader is used as the GuC. However, the actual
40 * loading to HW is deferred until GEM initialization is done.
41 *
42 * Note that HuC firmware loading must be done before GuC loading.
43 */
44
45#define BXT_HUC_FW_MAJOR 01
46#define BXT_HUC_FW_MINOR 07
47#define BXT_BLD_NUM 1398
48
49#define SKL_HUC_FW_MAJOR 01
50#define SKL_HUC_FW_MINOR 07
51#define SKL_BLD_NUM 1398
52
53#define KBL_HUC_FW_MAJOR 02
54#define KBL_HUC_FW_MINOR 00
55#define KBL_BLD_NUM 1810
56
57#define HUC_FW_PATH(platform, major, minor, bld_num) \
58 "i915/" __stringify(platform) "_huc_ver" __stringify(major) "_" \
59 __stringify(minor) "_" __stringify(bld_num) ".bin"
60
61#define I915_SKL_HUC_UCODE HUC_FW_PATH(skl, SKL_HUC_FW_MAJOR, \
62 SKL_HUC_FW_MINOR, SKL_BLD_NUM)
63MODULE_FIRMWARE(I915_SKL_HUC_UCODE);
64
65#define I915_BXT_HUC_UCODE HUC_FW_PATH(bxt, BXT_HUC_FW_MAJOR, \
66 BXT_HUC_FW_MINOR, BXT_BLD_NUM)
67MODULE_FIRMWARE(I915_BXT_HUC_UCODE);
68
69#define I915_KBL_HUC_UCODE HUC_FW_PATH(kbl, KBL_HUC_FW_MAJOR, \
70 KBL_HUC_FW_MINOR, KBL_BLD_NUM)
71MODULE_FIRMWARE(I915_KBL_HUC_UCODE);
72
73static void huc_fw_select(struct intel_uc_fw *huc_fw)
74{
75 struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
76 struct drm_i915_private *dev_priv = huc_to_i915(huc);
77
78 GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
79
80 if (!HAS_HUC(dev_priv))
81 return;
82
83 if (i915_modparams.huc_firmware_path) {
84 huc_fw->path = i915_modparams.huc_firmware_path;
85 huc_fw->major_ver_wanted = 0;
86 huc_fw->minor_ver_wanted = 0;
87 } else if (IS_SKYLAKE(dev_priv)) {
88 huc_fw->path = I915_SKL_HUC_UCODE;
89 huc_fw->major_ver_wanted = SKL_HUC_FW_MAJOR;
90 huc_fw->minor_ver_wanted = SKL_HUC_FW_MINOR;
91 } else if (IS_BROXTON(dev_priv)) {
92 huc_fw->path = I915_BXT_HUC_UCODE;
93 huc_fw->major_ver_wanted = BXT_HUC_FW_MAJOR;
94 huc_fw->minor_ver_wanted = BXT_HUC_FW_MINOR;
95 } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
96 huc_fw->path = I915_KBL_HUC_UCODE;
97 huc_fw->major_ver_wanted = KBL_HUC_FW_MAJOR;
98 huc_fw->minor_ver_wanted = KBL_HUC_FW_MINOR;
99 } else {
100 DRM_WARN("%s: No firmware known for this platform!\n",
101 intel_uc_fw_type_repr(huc_fw->type));
102 }
103}
104
105/**
106 * intel_huc_init_early() - initializes HuC struct
107 * @huc: intel_huc struct
108 *
109 * On platforms with HuC selects firmware for uploading
110 */
111void intel_huc_init_early(struct intel_huc *huc) 30void intel_huc_init_early(struct intel_huc *huc)
112{ 31{
113 struct intel_uc_fw *huc_fw = &huc->fw; 32 intel_huc_fw_init_early(huc);
114
115 intel_uc_fw_init(huc_fw, INTEL_UC_FW_TYPE_HUC);
116 huc_fw_select(huc_fw);
117}
118
119/**
120 * huc_ucode_xfer() - DMA's the firmware
121 * @huc_fw: the firmware descriptor
122 * @vma: the firmware image (bound into the GGTT)
123 *
124 * Transfer the firmware image to RAM for execution by the microcontroller.
125 *
126 * Return: 0 on success, non-zero on failure
127 */
128static int huc_ucode_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
129{
130 struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
131 struct drm_i915_private *dev_priv = huc_to_i915(huc);
132 unsigned long offset = 0;
133 u32 size;
134 int ret;
135
136 GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
137
138 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
139
140 /* Set the source address for the uCode */
141 offset = guc_ggtt_offset(vma) + huc_fw->header_offset;
142 I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
143 I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
144
145 /* Hardware doesn't look at destination address for HuC. Set it to 0,
146 * but still program the correct address space.
147 */
148 I915_WRITE(DMA_ADDR_1_LOW, 0);
149 I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
150
151 size = huc_fw->header_size + huc_fw->ucode_size;
152 I915_WRITE(DMA_COPY_SIZE, size);
153
154 /* Start the DMA */
155 I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
156
157 /* Wait for DMA to finish */
158 ret = intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, 100);
159
160 DRM_DEBUG_DRIVER("HuC DMA transfer wait over with ret %d\n", ret);
161
162 /* Disable the bits once DMA is over */
163 I915_WRITE(DMA_CTRL, _MASKED_BIT_DISABLE(HUC_UKERNEL));
164
165 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
166
167 return ret;
168}
169
170/**
171 * intel_huc_init_hw() - load HuC uCode to device
172 * @huc: intel_huc structure
173 *
174 * Called from intel_uc_init_hw() during driver loading and also after a GPU
175 * reset. Be note that HuC loading must be done before GuC loading.
176 *
177 * The firmware image should have already been fetched into memory by the
178 * earlier call to intel_uc_init_fw(), so here we need only check that
179 * is succeeded, and then transfer the image to the h/w.
180 *
181 */
182int intel_huc_init_hw(struct intel_huc *huc)
183{
184 return intel_uc_fw_upload(&huc->fw, huc_ucode_xfer);
185} 33}
186 34
187/** 35/**
@@ -200,6 +48,7 @@ int intel_huc_auth(struct intel_huc *huc)
200 struct drm_i915_private *i915 = huc_to_i915(huc); 48 struct drm_i915_private *i915 = huc_to_i915(huc);
201 struct intel_guc *guc = &i915->guc; 49 struct intel_guc *guc = &i915->guc;
202 struct i915_vma *vma; 50 struct i915_vma *vma;
51 u32 status;
203 int ret; 52 int ret;
204 53
205 if (huc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS) 54 if (huc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
@@ -210,28 +59,35 @@ int intel_huc_auth(struct intel_huc *huc)
210 if (IS_ERR(vma)) { 59 if (IS_ERR(vma)) {
211 ret = PTR_ERR(vma); 60 ret = PTR_ERR(vma);
212 DRM_ERROR("HuC: Failed to pin huc fw object %d\n", ret); 61 DRM_ERROR("HuC: Failed to pin huc fw object %d\n", ret);
213 return ret; 62 goto fail;
214 } 63 }
215 64
216 ret = intel_guc_auth_huc(guc, 65 ret = intel_guc_auth_huc(guc,
217 guc_ggtt_offset(vma) + huc->fw.rsa_offset); 66 guc_ggtt_offset(vma) + huc->fw.rsa_offset);
218 if (ret) { 67 if (ret) {
219 DRM_ERROR("HuC: GuC did not ack Auth request %d\n", ret); 68 DRM_ERROR("HuC: GuC did not ack Auth request %d\n", ret);
220 goto out; 69 goto fail_unpin;
221 } 70 }
222 71
223 /* Check authentication status, it should be done by now */ 72 /* Check authentication status, it should be done by now */
224 ret = intel_wait_for_register(i915, 73 ret = __intel_wait_for_register(i915,
225 HUC_STATUS2, 74 HUC_STATUS2,
226 HUC_FW_VERIFIED, 75 HUC_FW_VERIFIED,
227 HUC_FW_VERIFIED, 76 HUC_FW_VERIFIED,
228 50); 77 2, 50, &status);
229 if (ret) { 78 if (ret) {
230 DRM_ERROR("HuC: Authentication failed %d\n", ret); 79 DRM_ERROR("HuC: Firmware not verified %#x\n", status);
231 goto out; 80 goto fail_unpin;
232 } 81 }
233 82
234out:
235 i915_vma_unpin(vma); 83 i915_vma_unpin(vma);
84 return 0;
85
86fail_unpin:
87 i915_vma_unpin(vma);
88fail:
89 huc->fw.load_status = INTEL_UC_FIRMWARE_FAIL;
90
91 DRM_ERROR("HuC: Authentication failed %d\n", ret);
236 return ret; 92 return ret;
237} 93}
diff --git a/drivers/gpu/drm/i915/intel_huc.h b/drivers/gpu/drm/i915/intel_huc.h
index 40039db59e04..5d6e804f9771 100644
--- a/drivers/gpu/drm/i915/intel_huc.h
+++ b/drivers/gpu/drm/i915/intel_huc.h
@@ -26,6 +26,7 @@
26#define _INTEL_HUC_H_ 26#define _INTEL_HUC_H_
27 27
28#include "intel_uc_fw.h" 28#include "intel_uc_fw.h"
29#include "intel_huc_fw.h"
29 30
30struct intel_huc { 31struct intel_huc {
31 /* Generic uC firmware management */ 32 /* Generic uC firmware management */
@@ -35,7 +36,6 @@ struct intel_huc {
35}; 36};
36 37
37void intel_huc_init_early(struct intel_huc *huc); 38void intel_huc_init_early(struct intel_huc *huc);
38int intel_huc_init_hw(struct intel_huc *huc);
39int intel_huc_auth(struct intel_huc *huc); 39int intel_huc_auth(struct intel_huc *huc);
40 40
41#endif 41#endif
diff --git a/drivers/gpu/drm/i915/intel_huc_fw.c b/drivers/gpu/drm/i915/intel_huc_fw.c
new file mode 100644
index 000000000000..c66afa9b989a
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_huc_fw.c
@@ -0,0 +1,166 @@
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7#include "intel_huc_fw.h"
8#include "i915_drv.h"
9
10/**
11 * DOC: HuC Firmware
12 *
13 * Motivation:
14 * GEN9 introduces a new dedicated firmware for usage in media HEVC (High
15 * Efficiency Video Coding) operations. Userspace can use the firmware
16 * capabilities by adding HuC specific commands to batch buffers.
17 *
18 * Implementation:
19 * The same firmware loader is used as the GuC. However, the actual
20 * loading to HW is deferred until GEM initialization is done.
21 *
22 * Note that HuC firmware loading must be done before GuC loading.
23 */
24
25#define BXT_HUC_FW_MAJOR 01
26#define BXT_HUC_FW_MINOR 07
27#define BXT_BLD_NUM 1398
28
29#define SKL_HUC_FW_MAJOR 01
30#define SKL_HUC_FW_MINOR 07
31#define SKL_BLD_NUM 1398
32
33#define KBL_HUC_FW_MAJOR 02
34#define KBL_HUC_FW_MINOR 00
35#define KBL_BLD_NUM 1810
36
37#define HUC_FW_PATH(platform, major, minor, bld_num) \
38 "i915/" __stringify(platform) "_huc_ver" __stringify(major) "_" \
39 __stringify(minor) "_" __stringify(bld_num) ".bin"
40
41#define I915_SKL_HUC_UCODE HUC_FW_PATH(skl, SKL_HUC_FW_MAJOR, \
42 SKL_HUC_FW_MINOR, SKL_BLD_NUM)
43MODULE_FIRMWARE(I915_SKL_HUC_UCODE);
44
45#define I915_BXT_HUC_UCODE HUC_FW_PATH(bxt, BXT_HUC_FW_MAJOR, \
46 BXT_HUC_FW_MINOR, BXT_BLD_NUM)
47MODULE_FIRMWARE(I915_BXT_HUC_UCODE);
48
49#define I915_KBL_HUC_UCODE HUC_FW_PATH(kbl, KBL_HUC_FW_MAJOR, \
50 KBL_HUC_FW_MINOR, KBL_BLD_NUM)
51MODULE_FIRMWARE(I915_KBL_HUC_UCODE);
52
53static void huc_fw_select(struct intel_uc_fw *huc_fw)
54{
55 struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
56 struct drm_i915_private *dev_priv = huc_to_i915(huc);
57
58 GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
59
60 if (!HAS_HUC(dev_priv))
61 return;
62
63 if (i915_modparams.huc_firmware_path) {
64 huc_fw->path = i915_modparams.huc_firmware_path;
65 huc_fw->major_ver_wanted = 0;
66 huc_fw->minor_ver_wanted = 0;
67 } else if (IS_SKYLAKE(dev_priv)) {
68 huc_fw->path = I915_SKL_HUC_UCODE;
69 huc_fw->major_ver_wanted = SKL_HUC_FW_MAJOR;
70 huc_fw->minor_ver_wanted = SKL_HUC_FW_MINOR;
71 } else if (IS_BROXTON(dev_priv)) {
72 huc_fw->path = I915_BXT_HUC_UCODE;
73 huc_fw->major_ver_wanted = BXT_HUC_FW_MAJOR;
74 huc_fw->minor_ver_wanted = BXT_HUC_FW_MINOR;
75 } else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
76 huc_fw->path = I915_KBL_HUC_UCODE;
77 huc_fw->major_ver_wanted = KBL_HUC_FW_MAJOR;
78 huc_fw->minor_ver_wanted = KBL_HUC_FW_MINOR;
79 } else {
80 DRM_WARN("%s: No firmware known for this platform!\n",
81 intel_uc_fw_type_repr(huc_fw->type));
82 }
83}
84
85/**
86 * intel_huc_fw_init_early() - initializes HuC firmware struct
87 * @huc: intel_huc struct
88 *
89 * On platforms with HuC selects firmware for uploading
90 */
91void intel_huc_fw_init_early(struct intel_huc *huc)
92{
93 struct intel_uc_fw *huc_fw = &huc->fw;
94
95 intel_uc_fw_init(huc_fw, INTEL_UC_FW_TYPE_HUC);
96 huc_fw_select(huc_fw);
97}
98
99/**
100 * huc_fw_xfer() - DMA's the firmware
101 * @huc_fw: the firmware descriptor
102 * @vma: the firmware image (bound into the GGTT)
103 *
104 * Transfer the firmware image to RAM for execution by the microcontroller.
105 *
106 * Return: 0 on success, non-zero on failure
107 */
108static int huc_fw_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
109{
110 struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
111 struct drm_i915_private *dev_priv = huc_to_i915(huc);
112 unsigned long offset = 0;
113 u32 size;
114 int ret;
115
116 GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
117
118 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
119
120 /* Set the source address for the uCode */
121 offset = guc_ggtt_offset(vma) + huc_fw->header_offset;
122 I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
123 I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
124
125 /* Hardware doesn't look at destination address for HuC. Set it to 0,
126 * but still program the correct address space.
127 */
128 I915_WRITE(DMA_ADDR_1_LOW, 0);
129 I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
130
131 size = huc_fw->header_size + huc_fw->ucode_size;
132 I915_WRITE(DMA_COPY_SIZE, size);
133
134 /* Start the DMA */
135 I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
136
137 /* Wait for DMA to finish */
138 ret = intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, 100);
139
140 DRM_DEBUG_DRIVER("HuC DMA transfer wait over with ret %d\n", ret);
141
142 /* Disable the bits once DMA is over */
143 I915_WRITE(DMA_CTRL, _MASKED_BIT_DISABLE(HUC_UKERNEL));
144
145 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
146
147 return ret;
148}
149
150/**
151 * intel_huc_fw_upload() - load HuC uCode to device
152 * @huc: intel_huc structure
153 *
154 * Called from intel_uc_init_hw() during driver load, resume from sleep and
155 * after a GPU reset. Note that HuC must be loaded before GuC.
156 *
157 * The firmware image should have already been fetched into memory by the
158 * earlier call to intel_uc_init_fw(), so here we need to only check that
159 * fetch succeeded, and then transfer the image to the h/w.
160 *
161 * Return: non-zero code on error
162 */
163int intel_huc_fw_upload(struct intel_huc *huc)
164{
165 return intel_uc_fw_upload(&huc->fw, huc_fw_xfer);
166}
diff --git a/drivers/gpu/drm/i915/intel_huc_fw.h b/drivers/gpu/drm/i915/intel_huc_fw.h
new file mode 100644
index 000000000000..8a00a0ebddc5
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_huc_fw.h
@@ -0,0 +1,15 @@
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7#ifndef _INTEL_HUC_FW_H_
8#define _INTEL_HUC_FW_H_
9
10struct intel_huc;
11
12void intel_huc_fw_init_early(struct intel_huc *huc);
13int intel_huc_fw_upload(struct intel_huc *huc);
14
15#endif
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 9b6d781b22ec..3a69b367e565 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -169,6 +169,23 @@ static void execlists_init_reg_state(u32 *reg_state,
169 struct intel_engine_cs *engine, 169 struct intel_engine_cs *engine,
170 struct intel_ring *ring); 170 struct intel_ring *ring);
171 171
172static inline struct i915_priolist *to_priolist(struct rb_node *rb)
173{
174 return rb_entry(rb, struct i915_priolist, node);
175}
176
177static inline int rq_prio(const struct i915_request *rq)
178{
179 return rq->priotree.priority;
180}
181
182static inline bool need_preempt(const struct intel_engine_cs *engine,
183 const struct i915_request *last,
184 int prio)
185{
186 return engine->i915->preempt_context && prio > max(rq_prio(last), 0);
187}
188
172/** 189/**
173 * intel_lr_context_descriptor_update() - calculate & cache the descriptor 190 * intel_lr_context_descriptor_update() - calculate & cache the descriptor
174 * descriptor for a pinned context 191 * descriptor for a pinned context
@@ -187,6 +204,18 @@ static void execlists_init_reg_state(u32 *reg_state,
187 * bits 32-52: ctx ID, a globally unique tag 204 * bits 32-52: ctx ID, a globally unique tag
188 * bits 53-54: mbz, reserved for use by hardware 205 * bits 53-54: mbz, reserved for use by hardware
189 * bits 55-63: group ID, currently unused and set to 0 206 * bits 55-63: group ID, currently unused and set to 0
207 *
208 * Starting from Gen11, the upper dword of the descriptor has a new format:
209 *
210 * bits 32-36: reserved
211 * bits 37-47: SW context ID
212 * bits 48:53: engine instance
213 * bit 54: mbz, reserved for use by hardware
214 * bits 55-60: SW counter
215 * bits 61-63: engine class
216 *
217 * engine info, SW context ID and SW counter need to form a unique number
218 * (Context ID) per lrc.
190 */ 219 */
191static void 220static void
192intel_lr_context_descriptor_update(struct i915_gem_context *ctx, 221intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
@@ -195,12 +224,32 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
195 struct intel_context *ce = &ctx->engine[engine->id]; 224 struct intel_context *ce = &ctx->engine[engine->id];
196 u64 desc; 225 u64 desc;
197 226
198 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH)); 227 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
228 BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > (BIT(GEN11_SW_CTX_ID_WIDTH)));
199 229
200 desc = ctx->desc_template; /* bits 0-11 */ 230 desc = ctx->desc_template; /* bits 0-11 */
231 GEM_BUG_ON(desc & GENMASK_ULL(63, 12));
232
201 desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE; 233 desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
202 /* bits 12-31 */ 234 /* bits 12-31 */
203 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */ 235 GEM_BUG_ON(desc & GENMASK_ULL(63, 32));
236
237 if (INTEL_GEN(ctx->i915) >= 11) {
238 GEM_BUG_ON(ctx->hw_id >= BIT(GEN11_SW_CTX_ID_WIDTH));
239 desc |= (u64)ctx->hw_id << GEN11_SW_CTX_ID_SHIFT;
240 /* bits 37-47 */
241
242 desc |= (u64)engine->instance << GEN11_ENGINE_INSTANCE_SHIFT;
243 /* bits 48-53 */
244
245 /* TODO: decide what to do with SW counter (bits 55-60) */
246
247 desc |= (u64)engine->class << GEN11_ENGINE_CLASS_SHIFT;
248 /* bits 61-63 */
249 } else {
250 GEM_BUG_ON(ctx->hw_id >= BIT(GEN8_CTX_ID_WIDTH));
251 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */
252 }
204 253
205 ce->lrc_desc = desc; 254 ce->lrc_desc = desc;
206} 255}
@@ -224,7 +273,7 @@ find_priolist:
224 parent = &execlists->queue.rb_node; 273 parent = &execlists->queue.rb_node;
225 while (*parent) { 274 while (*parent) {
226 rb = *parent; 275 rb = *parent;
227 p = rb_entry(rb, typeof(*p), node); 276 p = to_priolist(rb);
228 if (prio > p->priority) { 277 if (prio > p->priority) {
229 parent = &rb->rb_left; 278 parent = &rb->rb_left;
230 } else if (prio < p->priority) { 279 } else if (prio < p->priority) {
@@ -264,10 +313,10 @@ find_priolist:
264 if (first) 313 if (first)
265 execlists->first = &p->node; 314 execlists->first = &p->node;
266 315
267 return ptr_pack_bits(p, first, 1); 316 return p;
268} 317}
269 318
270static void unwind_wa_tail(struct drm_i915_gem_request *rq) 319static void unwind_wa_tail(struct i915_request *rq)
271{ 320{
272 rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES); 321 rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES);
273 assert_ring_tail_valid(rq->ring, rq->tail); 322 assert_ring_tail_valid(rq->ring, rq->tail);
@@ -275,7 +324,7 @@ static void unwind_wa_tail(struct drm_i915_gem_request *rq)
275 324
276static void __unwind_incomplete_requests(struct intel_engine_cs *engine) 325static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
277{ 326{
278 struct drm_i915_gem_request *rq, *rn; 327 struct i915_request *rq, *rn;
279 struct i915_priolist *uninitialized_var(p); 328 struct i915_priolist *uninitialized_var(p);
280 int last_prio = I915_PRIORITY_INVALID; 329 int last_prio = I915_PRIORITY_INVALID;
281 330
@@ -284,20 +333,16 @@ static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
284 list_for_each_entry_safe_reverse(rq, rn, 333 list_for_each_entry_safe_reverse(rq, rn,
285 &engine->timeline->requests, 334 &engine->timeline->requests,
286 link) { 335 link) {
287 if (i915_gem_request_completed(rq)) 336 if (i915_request_completed(rq))
288 return; 337 return;
289 338
290 __i915_gem_request_unsubmit(rq); 339 __i915_request_unsubmit(rq);
291 unwind_wa_tail(rq); 340 unwind_wa_tail(rq);
292 341
293 GEM_BUG_ON(rq->priotree.priority == I915_PRIORITY_INVALID); 342 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
294 if (rq->priotree.priority != last_prio) { 343 if (rq_prio(rq) != last_prio) {
295 p = lookup_priolist(engine, 344 last_prio = rq_prio(rq);
296 &rq->priotree, 345 p = lookup_priolist(engine, &rq->priotree, last_prio);
297 rq->priotree.priority);
298 p = ptr_mask_bits(p, 1);
299
300 last_prio = rq->priotree.priority;
301 } 346 }
302 347
303 list_add(&rq->priotree.link, &p->requests); 348 list_add(&rq->priotree.link, &p->requests);
@@ -316,8 +361,7 @@ execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
316} 361}
317 362
318static inline void 363static inline void
319execlists_context_status_change(struct drm_i915_gem_request *rq, 364execlists_context_status_change(struct i915_request *rq, unsigned long status)
320 unsigned long status)
321{ 365{
322 /* 366 /*
323 * Only used when GVT-g is enabled now. When GVT-g is disabled, 367 * Only used when GVT-g is enabled now. When GVT-g is disabled,
@@ -331,14 +375,14 @@ execlists_context_status_change(struct drm_i915_gem_request *rq,
331} 375}
332 376
333static inline void 377static inline void
334execlists_context_schedule_in(struct drm_i915_gem_request *rq) 378execlists_context_schedule_in(struct i915_request *rq)
335{ 379{
336 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); 380 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
337 intel_engine_context_in(rq->engine); 381 intel_engine_context_in(rq->engine);
338} 382}
339 383
340static inline void 384static inline void
341execlists_context_schedule_out(struct drm_i915_gem_request *rq) 385execlists_context_schedule_out(struct i915_request *rq)
342{ 386{
343 intel_engine_context_out(rq->engine); 387 intel_engine_context_out(rq->engine);
344 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); 388 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
@@ -353,7 +397,7 @@ execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
353 ASSIGN_CTX_PDP(ppgtt, reg_state, 0); 397 ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
354} 398}
355 399
356static u64 execlists_update_context(struct drm_i915_gem_request *rq) 400static u64 execlists_update_context(struct i915_request *rq)
357{ 401{
358 struct intel_context *ce = &rq->ctx->engine[rq->engine->id]; 402 struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
359 struct i915_hw_ppgtt *ppgtt = 403 struct i915_hw_ppgtt *ppgtt =
@@ -373,19 +417,31 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq)
373 return ce->lrc_desc; 417 return ce->lrc_desc;
374} 418}
375 419
376static inline void elsp_write(u64 desc, u32 __iomem *elsp) 420static inline void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
377{ 421{
378 writel(upper_32_bits(desc), elsp); 422 if (execlists->ctrl_reg) {
379 writel(lower_32_bits(desc), elsp); 423 writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
424 writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
425 } else {
426 writel(upper_32_bits(desc), execlists->submit_reg);
427 writel(lower_32_bits(desc), execlists->submit_reg);
428 }
380} 429}
381 430
382static void execlists_submit_ports(struct intel_engine_cs *engine) 431static void execlists_submit_ports(struct intel_engine_cs *engine)
383{ 432{
384 struct execlist_port *port = engine->execlists.port; 433 struct intel_engine_execlists *execlists = &engine->execlists;
434 struct execlist_port *port = execlists->port;
385 unsigned int n; 435 unsigned int n;
386 436
387 for (n = execlists_num_ports(&engine->execlists); n--; ) { 437 /*
388 struct drm_i915_gem_request *rq; 438 * ELSQ note: the submit queue is not cleared after being submitted
439 * to the HW so we need to make sure we always clean it up. This is
440 * currently ensured by the fact that we always write the same number
441 * of elsq entries, keep this in mind before changing the loop below.
442 */
443 for (n = execlists_num_ports(execlists); n--; ) {
444 struct i915_request *rq;
389 unsigned int count; 445 unsigned int count;
390 u64 desc; 446 u64 desc;
391 447
@@ -398,18 +454,24 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
398 desc = execlists_update_context(rq); 454 desc = execlists_update_context(rq);
399 GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc)); 455 GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
400 456
401 GEM_TRACE("%s in[%d]: ctx=%d.%d, seqno=%x\n", 457 GEM_TRACE("%s in[%d]: ctx=%d.%d, seqno=%x, prio=%d\n",
402 engine->name, n, 458 engine->name, n,
403 port[n].context_id, count, 459 port[n].context_id, count,
404 rq->global_seqno); 460 rq->global_seqno,
461 rq_prio(rq));
405 } else { 462 } else {
406 GEM_BUG_ON(!n); 463 GEM_BUG_ON(!n);
407 desc = 0; 464 desc = 0;
408 } 465 }
409 466
410 elsp_write(desc, engine->execlists.elsp); 467 write_desc(execlists, desc, n);
411 } 468 }
412 execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK); 469
470 /* we need to manually load the submit queue */
471 if (execlists->ctrl_reg)
472 writel(EL_CTRL_LOAD, execlists->ctrl_reg);
473
474 execlists_clear_active(execlists, EXECLISTS_ACTIVE_HWACK);
413} 475}
414 476
415static bool ctx_single_port_submission(const struct i915_gem_context *ctx) 477static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
@@ -430,44 +492,47 @@ static bool can_merge_ctx(const struct i915_gem_context *prev,
430 return true; 492 return true;
431} 493}
432 494
433static void port_assign(struct execlist_port *port, 495static void port_assign(struct execlist_port *port, struct i915_request *rq)
434 struct drm_i915_gem_request *rq)
435{ 496{
436 GEM_BUG_ON(rq == port_request(port)); 497 GEM_BUG_ON(rq == port_request(port));
437 498
438 if (port_isset(port)) 499 if (port_isset(port))
439 i915_gem_request_put(port_request(port)); 500 i915_request_put(port_request(port));
440 501
441 port_set(port, port_pack(i915_gem_request_get(rq), port_count(port))); 502 port_set(port, port_pack(i915_request_get(rq), port_count(port)));
442} 503}
443 504
444static void inject_preempt_context(struct intel_engine_cs *engine) 505static void inject_preempt_context(struct intel_engine_cs *engine)
445{ 506{
507 struct intel_engine_execlists *execlists = &engine->execlists;
446 struct intel_context *ce = 508 struct intel_context *ce =
447 &engine->i915->preempt_context->engine[engine->id]; 509 &engine->i915->preempt_context->engine[engine->id];
448 unsigned int n; 510 unsigned int n;
449 511
450 GEM_BUG_ON(engine->execlists.preempt_complete_status != 512 GEM_BUG_ON(execlists->preempt_complete_status !=
451 upper_32_bits(ce->lrc_desc)); 513 upper_32_bits(ce->lrc_desc));
452 GEM_BUG_ON(!IS_ALIGNED(ce->ring->size, WA_TAIL_BYTES));
453
454 memset(ce->ring->vaddr + ce->ring->tail, 0, WA_TAIL_BYTES);
455 ce->ring->tail += WA_TAIL_BYTES;
456 ce->ring->tail &= (ce->ring->size - 1);
457 ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail;
458
459 GEM_BUG_ON((ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1] & 514 GEM_BUG_ON((ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1] &
460 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 515 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
461 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)) != 516 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)) !=
462 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 517 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
463 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)); 518 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT));
464 519
520 /*
521 * Switch to our empty preempt context so
522 * the state of the GPU is known (idle).
523 */
465 GEM_TRACE("%s\n", engine->name); 524 GEM_TRACE("%s\n", engine->name);
466 for (n = execlists_num_ports(&engine->execlists); --n; ) 525 for (n = execlists_num_ports(execlists); --n; )
467 elsp_write(0, engine->execlists.elsp); 526 write_desc(execlists, 0, n);
527
528 write_desc(execlists, ce->lrc_desc, n);
529
530 /* we need to manually load the submit queue */
531 if (execlists->ctrl_reg)
532 writel(EL_CTRL_LOAD, execlists->ctrl_reg);
468 533
469 elsp_write(ce->lrc_desc, engine->execlists.elsp);
470 execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK); 534 execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
535 execlists_set_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT);
471} 536}
472 537
473static void execlists_dequeue(struct intel_engine_cs *engine) 538static void execlists_dequeue(struct intel_engine_cs *engine)
@@ -476,7 +541,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
476 struct execlist_port *port = execlists->port; 541 struct execlist_port *port = execlists->port;
477 const struct execlist_port * const last_port = 542 const struct execlist_port * const last_port =
478 &execlists->port[execlists->port_mask]; 543 &execlists->port[execlists->port_mask];
479 struct drm_i915_gem_request *last = port_request(port); 544 struct i915_request *last = port_request(port);
480 struct rb_node *rb; 545 struct rb_node *rb;
481 bool submit = false; 546 bool submit = false;
482 547
@@ -504,8 +569,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
504 spin_lock_irq(&engine->timeline->lock); 569 spin_lock_irq(&engine->timeline->lock);
505 rb = execlists->first; 570 rb = execlists->first;
506 GEM_BUG_ON(rb_first(&execlists->queue) != rb); 571 GEM_BUG_ON(rb_first(&execlists->queue) != rb);
507 if (!rb)
508 goto unlock;
509 572
510 if (last) { 573 if (last) {
511 /* 574 /*
@@ -528,55 +591,49 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
528 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK)) 591 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
529 goto unlock; 592 goto unlock;
530 593
531 if (engine->i915->preempt_context && 594 if (need_preempt(engine, last, execlists->queue_priority)) {
532 rb_entry(rb, struct i915_priolist, node)->priority >
533 max(last->priotree.priority, 0)) {
534 /*
535 * Switch to our empty preempt context so
536 * the state of the GPU is known (idle).
537 */
538 inject_preempt_context(engine); 595 inject_preempt_context(engine);
539 execlists_set_active(execlists,
540 EXECLISTS_ACTIVE_PREEMPT);
541 goto unlock; 596 goto unlock;
542 } else {
543 /*
544 * In theory, we could coalesce more requests onto
545 * the second port (the first port is active, with
546 * no preemptions pending). However, that means we
547 * then have to deal with the possible lite-restore
548 * of the second port (as we submit the ELSP, there
549 * may be a context-switch) but also we may complete
550 * the resubmission before the context-switch. Ergo,
551 * coalescing onto the second port will cause a
552 * preemption event, but we cannot predict whether
553 * that will affect port[0] or port[1].
554 *
555 * If the second port is already active, we can wait
556 * until the next context-switch before contemplating
557 * new requests. The GPU will be busy and we should be
558 * able to resubmit the new ELSP before it idles,
559 * avoiding pipeline bubbles (momentary pauses where
560 * the driver is unable to keep up the supply of new
561 * work).
562 */
563 if (port_count(&port[1]))
564 goto unlock;
565
566 /* WaIdleLiteRestore:bdw,skl
567 * Apply the wa NOOPs to prevent
568 * ring:HEAD == req:TAIL as we resubmit the
569 * request. See gen8_emit_breadcrumb() for
570 * where we prepare the padding after the
571 * end of the request.
572 */
573 last->tail = last->wa_tail;
574 } 597 }
598
599 /*
600 * In theory, we could coalesce more requests onto
601 * the second port (the first port is active, with
602 * no preemptions pending). However, that means we
603 * then have to deal with the possible lite-restore
604 * of the second port (as we submit the ELSP, there
605 * may be a context-switch) but also we may complete
606 * the resubmission before the context-switch. Ergo,
607 * coalescing onto the second port will cause a
608 * preemption event, but we cannot predict whether
609 * that will affect port[0] or port[1].
610 *
611 * If the second port is already active, we can wait
612 * until the next context-switch before contemplating
613 * new requests. The GPU will be busy and we should be
614 * able to resubmit the new ELSP before it idles,
615 * avoiding pipeline bubbles (momentary pauses where
616 * the driver is unable to keep up the supply of new
617 * work). However, we have to double check that the
618 * priorities of the ports haven't been switch.
619 */
620 if (port_count(&port[1]))
621 goto unlock;
622
623 /*
624 * WaIdleLiteRestore:bdw,skl
625 * Apply the wa NOOPs to prevent
626 * ring:HEAD == rq:TAIL as we resubmit the
627 * request. See gen8_emit_breadcrumb() for
628 * where we prepare the padding after the
629 * end of the request.
630 */
631 last->tail = last->wa_tail;
575 } 632 }
576 633
577 do { 634 while (rb) {
578 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 635 struct i915_priolist *p = to_priolist(rb);
579 struct drm_i915_gem_request *rq, *rn; 636 struct i915_request *rq, *rn;
580 637
581 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 638 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
582 /* 639 /*
@@ -626,8 +683,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
626 } 683 }
627 684
628 INIT_LIST_HEAD(&rq->priotree.link); 685 INIT_LIST_HEAD(&rq->priotree.link);
629 __i915_gem_request_submit(rq); 686 __i915_request_submit(rq);
630 trace_i915_gem_request_in(rq, port_index(port, execlists)); 687 trace_i915_request_in(rq, port_index(port, execlists));
631 last = rq; 688 last = rq;
632 submit = true; 689 submit = true;
633 } 690 }
@@ -637,8 +694,9 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
637 INIT_LIST_HEAD(&p->requests); 694 INIT_LIST_HEAD(&p->requests);
638 if (p->priority != I915_PRIORITY_NORMAL) 695 if (p->priority != I915_PRIORITY_NORMAL)
639 kmem_cache_free(engine->i915->priorities, p); 696 kmem_cache_free(engine->i915->priorities, p);
640 } while (rb); 697 }
641done: 698done:
699 execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
642 execlists->first = rb; 700 execlists->first = rb;
643 if (submit) 701 if (submit)
644 port_assign(port, last); 702 port_assign(port, last);
@@ -665,12 +723,17 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
665 unsigned int num_ports = execlists_num_ports(execlists); 723 unsigned int num_ports = execlists_num_ports(execlists);
666 724
667 while (num_ports-- && port_isset(port)) { 725 while (num_ports-- && port_isset(port)) {
668 struct drm_i915_gem_request *rq = port_request(port); 726 struct i915_request *rq = port_request(port);
669 727
670 GEM_BUG_ON(!execlists->active); 728 GEM_BUG_ON(!execlists->active);
671 intel_engine_context_out(rq->engine); 729 intel_engine_context_out(rq->engine);
672 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_PREEMPTED); 730
673 i915_gem_request_put(rq); 731 execlists_context_status_change(rq,
732 i915_request_completed(rq) ?
733 INTEL_CONTEXT_SCHEDULE_OUT :
734 INTEL_CONTEXT_SCHEDULE_PREEMPTED);
735
736 i915_request_put(rq);
674 737
675 memset(port, 0, sizeof(*port)); 738 memset(port, 0, sizeof(*port));
676 port++; 739 port++;
@@ -680,32 +743,50 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
680static void execlists_cancel_requests(struct intel_engine_cs *engine) 743static void execlists_cancel_requests(struct intel_engine_cs *engine)
681{ 744{
682 struct intel_engine_execlists * const execlists = &engine->execlists; 745 struct intel_engine_execlists * const execlists = &engine->execlists;
683 struct drm_i915_gem_request *rq, *rn; 746 struct i915_request *rq, *rn;
684 struct rb_node *rb; 747 struct rb_node *rb;
685 unsigned long flags; 748 unsigned long flags;
686 749
687 spin_lock_irqsave(&engine->timeline->lock, flags); 750 GEM_TRACE("%s\n", engine->name);
751
752 /*
753 * Before we call engine->cancel_requests(), we should have exclusive
754 * access to the submission state. This is arranged for us by the
755 * caller disabling the interrupt generation, the tasklet and other
756 * threads that may then access the same state, giving us a free hand
757 * to reset state. However, we still need to let lockdep be aware that
758 * we know this state may be accessed in hardirq context, so we
759 * disable the irq around this manipulation and we want to keep
760 * the spinlock focused on its duties and not accidentally conflate
761 * coverage to the submission's irq state. (Similarly, although we
762 * shouldn't need to disable irq around the manipulation of the
763 * submission's irq state, we also wish to remind ourselves that
764 * it is irq state.)
765 */
766 local_irq_save(flags);
688 767
689 /* Cancel the requests on the HW and clear the ELSP tracker. */ 768 /* Cancel the requests on the HW and clear the ELSP tracker. */
690 execlists_cancel_port_requests(execlists); 769 execlists_cancel_port_requests(execlists);
691 770
771 spin_lock(&engine->timeline->lock);
772
692 /* Mark all executing requests as skipped. */ 773 /* Mark all executing requests as skipped. */
693 list_for_each_entry(rq, &engine->timeline->requests, link) { 774 list_for_each_entry(rq, &engine->timeline->requests, link) {
694 GEM_BUG_ON(!rq->global_seqno); 775 GEM_BUG_ON(!rq->global_seqno);
695 if (!i915_gem_request_completed(rq)) 776 if (!i915_request_completed(rq))
696 dma_fence_set_error(&rq->fence, -EIO); 777 dma_fence_set_error(&rq->fence, -EIO);
697 } 778 }
698 779
699 /* Flush the queued requests to the timeline list (for retiring). */ 780 /* Flush the queued requests to the timeline list (for retiring). */
700 rb = execlists->first; 781 rb = execlists->first;
701 while (rb) { 782 while (rb) {
702 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 783 struct i915_priolist *p = to_priolist(rb);
703 784
704 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 785 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
705 INIT_LIST_HEAD(&rq->priotree.link); 786 INIT_LIST_HEAD(&rq->priotree.link);
706 787
707 dma_fence_set_error(&rq->fence, -EIO); 788 dma_fence_set_error(&rq->fence, -EIO);
708 __i915_gem_request_submit(rq); 789 __i915_request_submit(rq);
709 } 790 }
710 791
711 rb = rb_next(rb); 792 rb = rb_next(rb);
@@ -717,11 +798,13 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
717 798
718 /* Remaining _unready_ requests will be nop'ed when submitted */ 799 /* Remaining _unready_ requests will be nop'ed when submitted */
719 800
720 801 execlists->queue_priority = INT_MIN;
721 execlists->queue = RB_ROOT; 802 execlists->queue = RB_ROOT;
722 execlists->first = NULL; 803 execlists->first = NULL;
723 GEM_BUG_ON(port_isset(execlists->port)); 804 GEM_BUG_ON(port_isset(execlists->port));
724 805
806 spin_unlock(&engine->timeline->lock);
807
725 /* 808 /*
726 * The port is checked prior to scheduling a tasklet, but 809 * The port is checked prior to scheduling a tasklet, but
727 * just in case we have suspended the tasklet to do the 810 * just in case we have suspended the tasklet to do the
@@ -730,7 +813,10 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
730 */ 813 */
731 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 814 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
732 815
733 spin_unlock_irqrestore(&engine->timeline->lock, flags); 816 /* Mark all CS interrupts as complete */
817 execlists->active = 0;
818
819 local_irq_restore(flags);
734} 820}
735 821
736/* 822/*
@@ -806,7 +892,7 @@ static void execlists_submission_tasklet(unsigned long data)
806 tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?"); 892 tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?");
807 893
808 while (head != tail) { 894 while (head != tail) {
809 struct drm_i915_gem_request *rq; 895 struct i915_request *rq;
810 unsigned int status; 896 unsigned int status;
811 unsigned int count; 897 unsigned int count;
812 898
@@ -872,23 +958,28 @@ static void execlists_submission_tasklet(unsigned long data)
872 GEM_BUG_ON(!execlists_is_active(execlists, 958 GEM_BUG_ON(!execlists_is_active(execlists,
873 EXECLISTS_ACTIVE_USER)); 959 EXECLISTS_ACTIVE_USER));
874 960
875 /* Check the context/desc id for this event matches */
876 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
877
878 rq = port_unpack(port, &count); 961 rq = port_unpack(port, &count);
879 GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x\n", 962 GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x, prio=%d\n",
880 engine->name, 963 engine->name,
881 port->context_id, count, 964 port->context_id, count,
882 rq ? rq->global_seqno : 0); 965 rq ? rq->global_seqno : 0,
966 rq ? rq_prio(rq) : 0);
967
968 /* Check the context/desc id for this event matches */
969 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
970
883 GEM_BUG_ON(count == 0); 971 GEM_BUG_ON(count == 0);
884 if (--count == 0) { 972 if (--count == 0) {
885 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED); 973 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
886 GEM_BUG_ON(port_isset(&port[1]) && 974 GEM_BUG_ON(port_isset(&port[1]) &&
887 !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH)); 975 !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
888 GEM_BUG_ON(!i915_gem_request_completed(rq)); 976 GEM_BUG_ON(!i915_request_completed(rq));
889 execlists_context_schedule_out(rq); 977 execlists_context_schedule_out(rq);
890 trace_i915_gem_request_out(rq); 978 trace_i915_request_out(rq);
891 i915_gem_request_put(rq); 979 i915_request_put(rq);
980
981 GEM_TRACE("%s completed ctx=%d\n",
982 engine->name, port->context_id);
892 983
893 execlists_port_complete(execlists, port); 984 execlists_port_complete(execlists, port);
894 } else { 985 } else {
@@ -917,18 +1008,22 @@ static void execlists_submission_tasklet(unsigned long data)
917 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains); 1008 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains);
918} 1009}
919 1010
920static void insert_request(struct intel_engine_cs *engine, 1011static void queue_request(struct intel_engine_cs *engine,
921 struct i915_priotree *pt, 1012 struct i915_priotree *pt,
922 int prio) 1013 int prio)
923{ 1014{
924 struct i915_priolist *p = lookup_priolist(engine, pt, prio); 1015 list_add_tail(&pt->link, &lookup_priolist(engine, pt, prio)->requests);
1016}
925 1017
926 list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests); 1018static void submit_queue(struct intel_engine_cs *engine, int prio)
927 if (ptr_unmask_bits(p, 1)) 1019{
1020 if (prio > engine->execlists.queue_priority) {
1021 engine->execlists.queue_priority = prio;
928 tasklet_hi_schedule(&engine->execlists.tasklet); 1022 tasklet_hi_schedule(&engine->execlists.tasklet);
1023 }
929} 1024}
930 1025
931static void execlists_submit_request(struct drm_i915_gem_request *request) 1026static void execlists_submit_request(struct i915_request *request)
932{ 1027{
933 struct intel_engine_cs *engine = request->engine; 1028 struct intel_engine_cs *engine = request->engine;
934 unsigned long flags; 1029 unsigned long flags;
@@ -936,7 +1031,8 @@ static void execlists_submit_request(struct drm_i915_gem_request *request)
936 /* Will be called from irq-context when using foreign fences. */ 1031 /* Will be called from irq-context when using foreign fences. */
937 spin_lock_irqsave(&engine->timeline->lock, flags); 1032 spin_lock_irqsave(&engine->timeline->lock, flags);
938 1033
939 insert_request(engine, &request->priotree, request->priotree.priority); 1034 queue_request(engine, &request->priotree, rq_prio(request));
1035 submit_queue(engine, rq_prio(request));
940 1036
941 GEM_BUG_ON(!engine->execlists.first); 1037 GEM_BUG_ON(!engine->execlists.first);
942 GEM_BUG_ON(list_empty(&request->priotree.link)); 1038 GEM_BUG_ON(list_empty(&request->priotree.link));
@@ -944,9 +1040,9 @@ static void execlists_submit_request(struct drm_i915_gem_request *request)
944 spin_unlock_irqrestore(&engine->timeline->lock, flags); 1040 spin_unlock_irqrestore(&engine->timeline->lock, flags);
945} 1041}
946 1042
947static struct drm_i915_gem_request *pt_to_request(struct i915_priotree *pt) 1043static struct i915_request *pt_to_request(struct i915_priotree *pt)
948{ 1044{
949 return container_of(pt, struct drm_i915_gem_request, priotree); 1045 return container_of(pt, struct i915_request, priotree);
950} 1046}
951 1047
952static struct intel_engine_cs * 1048static struct intel_engine_cs *
@@ -964,7 +1060,7 @@ pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
964 return engine; 1060 return engine;
965} 1061}
966 1062
967static void execlists_schedule(struct drm_i915_gem_request *request, int prio) 1063static void execlists_schedule(struct i915_request *request, int prio)
968{ 1064{
969 struct intel_engine_cs *engine; 1065 struct intel_engine_cs *engine;
970 struct i915_dependency *dep, *p; 1066 struct i915_dependency *dep, *p;
@@ -973,7 +1069,7 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
973 1069
974 GEM_BUG_ON(prio == I915_PRIORITY_INVALID); 1070 GEM_BUG_ON(prio == I915_PRIORITY_INVALID);
975 1071
976 if (i915_gem_request_completed(request)) 1072 if (i915_request_completed(request))
977 return; 1073 return;
978 1074
979 if (prio <= READ_ONCE(request->priotree.priority)) 1075 if (prio <= READ_ONCE(request->priotree.priority))
@@ -992,7 +1088,7 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
992 * static void update_priorities(struct i915_priotree *pt, prio) { 1088 * static void update_priorities(struct i915_priotree *pt, prio) {
993 * list_for_each_entry(dep, &pt->signalers_list, signal_link) 1089 * list_for_each_entry(dep, &pt->signalers_list, signal_link)
994 * update_priorities(dep->signal, prio) 1090 * update_priorities(dep->signal, prio)
995 * insert_request(pt); 1091 * queue_request(pt);
996 * } 1092 * }
997 * but that may have unlimited recursion depth and so runs a very 1093 * but that may have unlimited recursion depth and so runs a very
998 * real risk of overunning the kernel stack. Instead, we build 1094 * real risk of overunning the kernel stack. Instead, we build
@@ -1055,8 +1151,9 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
1055 pt->priority = prio; 1151 pt->priority = prio;
1056 if (!list_empty(&pt->link)) { 1152 if (!list_empty(&pt->link)) {
1057 __list_del_entry(&pt->link); 1153 __list_del_entry(&pt->link);
1058 insert_request(engine, pt, prio); 1154 queue_request(engine, pt, prio);
1059 } 1155 }
1156 submit_queue(engine, prio);
1060 } 1157 }
1061 1158
1062 spin_unlock_irq(&engine->timeline->lock); 1159 spin_unlock_irq(&engine->timeline->lock);
@@ -1158,7 +1255,7 @@ static void execlists_context_unpin(struct intel_engine_cs *engine,
1158 i915_gem_context_put(ctx); 1255 i915_gem_context_put(ctx);
1159} 1256}
1160 1257
1161static int execlists_request_alloc(struct drm_i915_gem_request *request) 1258static int execlists_request_alloc(struct i915_request *request)
1162{ 1259{
1163 struct intel_engine_cs *engine = request->engine; 1260 struct intel_engine_cs *engine = request->engine;
1164 struct intel_context *ce = &request->ctx->engine[engine->id]; 1261 struct intel_context *ce = &request->ctx->engine[engine->id];
@@ -1590,7 +1687,7 @@ static void reset_irq(struct intel_engine_cs *engine)
1590} 1687}
1591 1688
1592static void reset_common_ring(struct intel_engine_cs *engine, 1689static void reset_common_ring(struct intel_engine_cs *engine,
1593 struct drm_i915_gem_request *request) 1690 struct i915_request *request)
1594{ 1691{
1595 struct intel_engine_execlists * const execlists = &engine->execlists; 1692 struct intel_engine_execlists * const execlists = &engine->execlists;
1596 struct intel_context *ce; 1693 struct intel_context *ce;
@@ -1599,9 +1696,10 @@ static void reset_common_ring(struct intel_engine_cs *engine,
1599 GEM_TRACE("%s seqno=%x\n", 1696 GEM_TRACE("%s seqno=%x\n",
1600 engine->name, request ? request->global_seqno : 0); 1697 engine->name, request ? request->global_seqno : 0);
1601 1698
1602 reset_irq(engine); 1699 /* See execlists_cancel_requests() for the irq/spinlock split. */
1700 local_irq_save(flags);
1603 1701
1604 spin_lock_irqsave(&engine->timeline->lock, flags); 1702 reset_irq(engine);
1605 1703
1606 /* 1704 /*
1607 * Catch up with any missed context-switch interrupts. 1705 * Catch up with any missed context-switch interrupts.
@@ -1615,14 +1713,17 @@ static void reset_common_ring(struct intel_engine_cs *engine,
1615 execlists_cancel_port_requests(execlists); 1713 execlists_cancel_port_requests(execlists);
1616 1714
1617 /* Push back any incomplete requests for replay after the reset. */ 1715 /* Push back any incomplete requests for replay after the reset. */
1716 spin_lock(&engine->timeline->lock);
1618 __unwind_incomplete_requests(engine); 1717 __unwind_incomplete_requests(engine);
1619 1718 spin_unlock(&engine->timeline->lock);
1620 spin_unlock_irqrestore(&engine->timeline->lock, flags);
1621 1719
1622 /* Mark all CS interrupts as complete */ 1720 /* Mark all CS interrupts as complete */
1623 execlists->active = 0; 1721 execlists->active = 0;
1624 1722
1625 /* If the request was innocent, we leave the request in the ELSP 1723 local_irq_restore(flags);
1724
1725 /*
1726 * If the request was innocent, we leave the request in the ELSP
1626 * and will try to replay it on restarting. The context image may 1727 * and will try to replay it on restarting. The context image may
1627 * have been corrupted by the reset, in which case we may have 1728 * have been corrupted by the reset, in which case we may have
1628 * to service a new GPU hang, but more likely we can continue on 1729 * to service a new GPU hang, but more likely we can continue on
@@ -1635,7 +1736,8 @@ static void reset_common_ring(struct intel_engine_cs *engine,
1635 if (!request || request->fence.error != -EIO) 1736 if (!request || request->fence.error != -EIO)
1636 return; 1737 return;
1637 1738
1638 /* We want a simple context + ring to execute the breadcrumb update. 1739 /*
1740 * We want a simple context + ring to execute the breadcrumb update.
1639 * We cannot rely on the context being intact across the GPU hang, 1741 * We cannot rely on the context being intact across the GPU hang,
1640 * so clear it and rebuild just what we need for the breadcrumb. 1742 * so clear it and rebuild just what we need for the breadcrumb.
1641 * All pending requests for this context will be zapped, and any 1743 * All pending requests for this context will be zapped, and any
@@ -1658,15 +1760,15 @@ static void reset_common_ring(struct intel_engine_cs *engine,
1658 unwind_wa_tail(request); 1760 unwind_wa_tail(request);
1659} 1761}
1660 1762
1661static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req) 1763static int intel_logical_ring_emit_pdps(struct i915_request *rq)
1662{ 1764{
1663 struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt; 1765 struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
1664 struct intel_engine_cs *engine = req->engine; 1766 struct intel_engine_cs *engine = rq->engine;
1665 const int num_lri_cmds = GEN8_3LVL_PDPES * 2; 1767 const int num_lri_cmds = GEN8_3LVL_PDPES * 2;
1666 u32 *cs; 1768 u32 *cs;
1667 int i; 1769 int i;
1668 1770
1669 cs = intel_ring_begin(req, num_lri_cmds * 2 + 2); 1771 cs = intel_ring_begin(rq, num_lri_cmds * 2 + 2);
1670 if (IS_ERR(cs)) 1772 if (IS_ERR(cs))
1671 return PTR_ERR(cs); 1773 return PTR_ERR(cs);
1672 1774
@@ -1681,12 +1783,12 @@ static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
1681 } 1783 }
1682 1784
1683 *cs++ = MI_NOOP; 1785 *cs++ = MI_NOOP;
1684 intel_ring_advance(req, cs); 1786 intel_ring_advance(rq, cs);
1685 1787
1686 return 0; 1788 return 0;
1687} 1789}
1688 1790
1689static int gen8_emit_bb_start(struct drm_i915_gem_request *req, 1791static int gen8_emit_bb_start(struct i915_request *rq,
1690 u64 offset, u32 len, 1792 u64 offset, u32 len,
1691 const unsigned int flags) 1793 const unsigned int flags)
1692{ 1794{
@@ -1699,18 +1801,18 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1699 * it is unsafe in case of lite-restore (because the ctx is 1801 * it is unsafe in case of lite-restore (because the ctx is
1700 * not idle). PML4 is allocated during ppgtt init so this is 1802 * not idle). PML4 is allocated during ppgtt init so this is
1701 * not needed in 48-bit.*/ 1803 * not needed in 48-bit.*/
1702 if (req->ctx->ppgtt && 1804 if (rq->ctx->ppgtt &&
1703 (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings) && 1805 (intel_engine_flag(rq->engine) & rq->ctx->ppgtt->pd_dirty_rings) &&
1704 !i915_vm_is_48bit(&req->ctx->ppgtt->base) && 1806 !i915_vm_is_48bit(&rq->ctx->ppgtt->base) &&
1705 !intel_vgpu_active(req->i915)) { 1807 !intel_vgpu_active(rq->i915)) {
1706 ret = intel_logical_ring_emit_pdps(req); 1808 ret = intel_logical_ring_emit_pdps(rq);
1707 if (ret) 1809 if (ret)
1708 return ret; 1810 return ret;
1709 1811
1710 req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine); 1812 rq->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(rq->engine);
1711 } 1813 }
1712 1814
1713 cs = intel_ring_begin(req, 4); 1815 cs = intel_ring_begin(rq, 4);
1714 if (IS_ERR(cs)) 1816 if (IS_ERR(cs))
1715 return PTR_ERR(cs); 1817 return PTR_ERR(cs);
1716 1818
@@ -1739,7 +1841,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1739 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0); 1841 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1740 *cs++ = lower_32_bits(offset); 1842 *cs++ = lower_32_bits(offset);
1741 *cs++ = upper_32_bits(offset); 1843 *cs++ = upper_32_bits(offset);
1742 intel_ring_advance(req, cs); 1844 intel_ring_advance(rq, cs);
1743 1845
1744 return 0; 1846 return 0;
1745} 1847}
@@ -1758,7 +1860,7 @@ static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1758 I915_WRITE_IMR(engine, ~engine->irq_keep_mask); 1860 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1759} 1861}
1760 1862
1761static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode) 1863static int gen8_emit_flush(struct i915_request *request, u32 mode)
1762{ 1864{
1763 u32 cmd, *cs; 1865 u32 cmd, *cs;
1764 1866
@@ -1790,7 +1892,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode)
1790 return 0; 1892 return 0;
1791} 1893}
1792 1894
1793static int gen8_emit_flush_render(struct drm_i915_gem_request *request, 1895static int gen8_emit_flush_render(struct i915_request *request,
1794 u32 mode) 1896 u32 mode)
1795{ 1897{
1796 struct intel_engine_cs *engine = request->engine; 1898 struct intel_engine_cs *engine = request->engine;
@@ -1865,7 +1967,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1865 * used as a workaround for not being allowed to do lite 1967 * used as a workaround for not being allowed to do lite
1866 * restore with HEAD==TAIL (WaIdleLiteRestore). 1968 * restore with HEAD==TAIL (WaIdleLiteRestore).
1867 */ 1969 */
1868static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs) 1970static void gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
1869{ 1971{
1870 /* Ensure there's always at least one preemption point per-request. */ 1972 /* Ensure there's always at least one preemption point per-request. */
1871 *cs++ = MI_ARB_CHECK; 1973 *cs++ = MI_ARB_CHECK;
@@ -1873,7 +1975,7 @@ static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs)
1873 request->wa_tail = intel_ring_offset(request, cs); 1975 request->wa_tail = intel_ring_offset(request, cs);
1874} 1976}
1875 1977
1876static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs) 1978static void gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
1877{ 1979{
1878 /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ 1980 /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
1879 BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); 1981 BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
@@ -1889,8 +1991,7 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
1889} 1991}
1890static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; 1992static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
1891 1993
1892static void gen8_emit_breadcrumb_rcs(struct drm_i915_gem_request *request, 1994static void gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs)
1893 u32 *cs)
1894{ 1995{
1895 /* We're using qword write, seqno should be aligned to 8 bytes. */ 1996 /* We're using qword write, seqno should be aligned to 8 bytes. */
1896 BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); 1997 BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
@@ -1906,15 +2007,15 @@ static void gen8_emit_breadcrumb_rcs(struct drm_i915_gem_request *request,
1906} 2007}
1907static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS; 2008static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
1908 2009
1909static int gen8_init_rcs_context(struct drm_i915_gem_request *req) 2010static int gen8_init_rcs_context(struct i915_request *rq)
1910{ 2011{
1911 int ret; 2012 int ret;
1912 2013
1913 ret = intel_ring_workarounds_emit(req); 2014 ret = intel_ring_workarounds_emit(rq);
1914 if (ret) 2015 if (ret)
1915 return ret; 2016 return ret;
1916 2017
1917 ret = intel_rcs_context_init_mocs(req); 2018 ret = intel_rcs_context_init_mocs(rq);
1918 /* 2019 /*
1919 * Failing to program the MOCS is non-fatal.The system will not 2020 * Failing to program the MOCS is non-fatal.The system will not
1920 * run at peak performance. So generate an error and carry on. 2021 * run at peak performance. So generate an error and carry on.
@@ -1922,7 +2023,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
1922 if (ret) 2023 if (ret)
1923 DRM_ERROR("MOCS failed to program: expect performance issues.\n"); 2024 DRM_ERROR("MOCS failed to program: expect performance issues.\n");
1924 2025
1925 return i915_gem_render_state_emit(req); 2026 return i915_gem_render_state_emit(rq);
1926} 2027}
1927 2028
1928/** 2029/**
@@ -1996,8 +2097,17 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1996 2097
1997 engine->set_default_submission = execlists_set_default_submission; 2098 engine->set_default_submission = execlists_set_default_submission;
1998 2099
1999 engine->irq_enable = gen8_logical_ring_enable_irq; 2100 if (INTEL_GEN(engine->i915) < 11) {
2000 engine->irq_disable = gen8_logical_ring_disable_irq; 2101 engine->irq_enable = gen8_logical_ring_enable_irq;
2102 engine->irq_disable = gen8_logical_ring_disable_irq;
2103 } else {
2104 /*
2105 * TODO: On Gen11 interrupt masks need to be clear
2106 * to allow C6 entry. Keep interrupts enabled at
2107 * and take the hit of generating extra interrupts
2108 * until a more refined solution exists.
2109 */
2110 }
2001 engine->emit_bb_start = gen8_emit_bb_start; 2111 engine->emit_bb_start = gen8_emit_bb_start;
2002} 2112}
2003 2113
@@ -2049,8 +2159,15 @@ static int logical_ring_init(struct intel_engine_cs *engine)
2049 if (ret) 2159 if (ret)
2050 goto error; 2160 goto error;
2051 2161
2052 engine->execlists.elsp = 2162 if (HAS_LOGICAL_RING_ELSQ(engine->i915)) {
2053 engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); 2163 engine->execlists.submit_reg = engine->i915->regs +
2164 i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
2165 engine->execlists.ctrl_reg = engine->i915->regs +
2166 i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
2167 } else {
2168 engine->execlists.submit_reg = engine->i915->regs +
2169 i915_mmio_reg_offset(RING_ELSP(engine));
2170 }
2054 2171
2055 engine->execlists.preempt_complete_status = ~0u; 2172 engine->execlists.preempt_complete_status = ~0u;
2056 if (engine->i915->preempt_context) 2173 if (engine->i915->preempt_context)
@@ -2136,7 +2253,7 @@ make_rpcs(struct drm_i915_private *dev_priv)
2136 2253
2137 if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) { 2254 if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
2138 rpcs |= GEN8_RPCS_SS_CNT_ENABLE; 2255 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2139 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) << 2256 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask[0]) <<
2140 GEN8_RPCS_SS_CNT_SHIFT; 2257 GEN8_RPCS_SS_CNT_SHIFT;
2141 rpcs |= GEN8_RPCS_ENABLE; 2258 rpcs |= GEN8_RPCS_ENABLE;
2142 } 2259 }
@@ -2160,6 +2277,10 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2160 default: 2277 default:
2161 MISSING_CASE(INTEL_GEN(engine->i915)); 2278 MISSING_CASE(INTEL_GEN(engine->i915));
2162 /* fall through */ 2279 /* fall through */
2280 case 11:
2281 indirect_ctx_offset =
2282 GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
2283 break;
2163 case 10: 2284 case 10:
2164 indirect_ctx_offset = 2285 indirect_ctx_offset =
2165 GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2286 GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
@@ -2319,7 +2440,7 @@ populate_lr_context(struct i915_gem_context *ctx,
2319 if (!engine->default_state) 2440 if (!engine->default_state)
2320 regs[CTX_CONTEXT_CONTROL + 1] |= 2441 regs[CTX_CONTEXT_CONTROL + 1] |=
2321 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); 2442 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
2322 if (ctx == ctx->i915->preempt_context) 2443 if (ctx == ctx->i915->preempt_context && INTEL_GEN(engine->i915) < 11)
2323 regs[CTX_CONTEXT_CONTROL + 1] |= 2444 regs[CTX_CONTEXT_CONTROL + 1] |=
2324 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 2445 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
2325 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT); 2446 CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT);
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 636ced41225d..59d7b86012e9 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -42,6 +42,9 @@
42#define RING_CONTEXT_STATUS_BUF_LO(engine, i) _MMIO((engine)->mmio_base + 0x370 + (i) * 8) 42#define RING_CONTEXT_STATUS_BUF_LO(engine, i) _MMIO((engine)->mmio_base + 0x370 + (i) * 8)
43#define RING_CONTEXT_STATUS_BUF_HI(engine, i) _MMIO((engine)->mmio_base + 0x370 + (i) * 8 + 4) 43#define RING_CONTEXT_STATUS_BUF_HI(engine, i) _MMIO((engine)->mmio_base + 0x370 + (i) * 8 + 4)
44#define RING_CONTEXT_STATUS_PTR(engine) _MMIO((engine)->mmio_base + 0x3a0) 44#define RING_CONTEXT_STATUS_PTR(engine) _MMIO((engine)->mmio_base + 0x3a0)
45#define RING_EXECLIST_SQ_CONTENTS(engine) _MMIO((engine)->mmio_base + 0x510)
46#define RING_EXECLIST_CONTROL(engine) _MMIO((engine)->mmio_base + 0x550)
47#define EL_CTRL_LOAD (1 << 0)
45 48
46/* The docs specify that the write pointer wraps around after 5h, "After status 49/* The docs specify that the write pointer wraps around after 5h, "After status
47 * is written out to the last available status QW at offset 5h, this pointer 50 * is written out to the last available status QW at offset 5h, this pointer
diff --git a/drivers/gpu/drm/i915/intel_lrc_reg.h b/drivers/gpu/drm/i915/intel_lrc_reg.h
index a53336e2fc97..169a2239d6c7 100644
--- a/drivers/gpu/drm/i915/intel_lrc_reg.h
+++ b/drivers/gpu/drm/i915/intel_lrc_reg.h
@@ -63,5 +63,6 @@
63#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 63#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17
64#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 64#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26
65#define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x19 65#define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x19
66#define GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x1A
66 67
67#endif /* _INTEL_LRC_REG_H_ */ 68#endif /* _INTEL_LRC_REG_H_ */
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index abb7a8c1e340..c0b34b7943b9 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -265,7 +265,7 @@ int intel_mocs_init_engine(struct intel_engine_cs *engine)
265 265
266/** 266/**
267 * emit_mocs_control_table() - emit the mocs control table 267 * emit_mocs_control_table() - emit the mocs control table
268 * @req: Request to set up the MOCS table for. 268 * @rq: Request to set up the MOCS table for.
269 * @table: The values to program into the control regs. 269 * @table: The values to program into the control regs.
270 * 270 *
271 * This function simply emits a MI_LOAD_REGISTER_IMM command for the 271 * This function simply emits a MI_LOAD_REGISTER_IMM command for the
@@ -273,17 +273,17 @@ int intel_mocs_init_engine(struct intel_engine_cs *engine)
273 * 273 *
274 * Return: 0 on success, otherwise the error status. 274 * Return: 0 on success, otherwise the error status.
275 */ 275 */
276static int emit_mocs_control_table(struct drm_i915_gem_request *req, 276static int emit_mocs_control_table(struct i915_request *rq,
277 const struct drm_i915_mocs_table *table) 277 const struct drm_i915_mocs_table *table)
278{ 278{
279 enum intel_engine_id engine = req->engine->id; 279 enum intel_engine_id engine = rq->engine->id;
280 unsigned int index; 280 unsigned int index;
281 u32 *cs; 281 u32 *cs;
282 282
283 if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) 283 if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES))
284 return -ENODEV; 284 return -ENODEV;
285 285
286 cs = intel_ring_begin(req, 2 + 2 * GEN9_NUM_MOCS_ENTRIES); 286 cs = intel_ring_begin(rq, 2 + 2 * GEN9_NUM_MOCS_ENTRIES);
287 if (IS_ERR(cs)) 287 if (IS_ERR(cs))
288 return PTR_ERR(cs); 288 return PTR_ERR(cs);
289 289
@@ -308,7 +308,7 @@ static int emit_mocs_control_table(struct drm_i915_gem_request *req,
308 } 308 }
309 309
310 *cs++ = MI_NOOP; 310 *cs++ = MI_NOOP;
311 intel_ring_advance(req, cs); 311 intel_ring_advance(rq, cs);
312 312
313 return 0; 313 return 0;
314} 314}
@@ -323,7 +323,7 @@ static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table,
323 323
324/** 324/**
325 * emit_mocs_l3cc_table() - emit the mocs control table 325 * emit_mocs_l3cc_table() - emit the mocs control table
326 * @req: Request to set up the MOCS table for. 326 * @rq: Request to set up the MOCS table for.
327 * @table: The values to program into the control regs. 327 * @table: The values to program into the control regs.
328 * 328 *
329 * This function simply emits a MI_LOAD_REGISTER_IMM command for the 329 * This function simply emits a MI_LOAD_REGISTER_IMM command for the
@@ -332,7 +332,7 @@ static inline u32 l3cc_combine(const struct drm_i915_mocs_table *table,
332 * 332 *
333 * Return: 0 on success, otherwise the error status. 333 * Return: 0 on success, otherwise the error status.
334 */ 334 */
335static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req, 335static int emit_mocs_l3cc_table(struct i915_request *rq,
336 const struct drm_i915_mocs_table *table) 336 const struct drm_i915_mocs_table *table)
337{ 337{
338 unsigned int i; 338 unsigned int i;
@@ -341,7 +341,7 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
341 if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES)) 341 if (WARN_ON(table->size > GEN9_NUM_MOCS_ENTRIES))
342 return -ENODEV; 342 return -ENODEV;
343 343
344 cs = intel_ring_begin(req, 2 + GEN9_NUM_MOCS_ENTRIES); 344 cs = intel_ring_begin(rq, 2 + GEN9_NUM_MOCS_ENTRIES);
345 if (IS_ERR(cs)) 345 if (IS_ERR(cs))
346 return PTR_ERR(cs); 346 return PTR_ERR(cs);
347 347
@@ -370,7 +370,7 @@ static int emit_mocs_l3cc_table(struct drm_i915_gem_request *req,
370 } 370 }
371 371
372 *cs++ = MI_NOOP; 372 *cs++ = MI_NOOP;
373 intel_ring_advance(req, cs); 373 intel_ring_advance(rq, cs);
374 374
375 return 0; 375 return 0;
376} 376}
@@ -417,7 +417,7 @@ void intel_mocs_init_l3cc_table(struct drm_i915_private *dev_priv)
417 417
418/** 418/**
419 * intel_rcs_context_init_mocs() - program the MOCS register. 419 * intel_rcs_context_init_mocs() - program the MOCS register.
420 * @req: Request to set up the MOCS tables for. 420 * @rq: Request to set up the MOCS tables for.
421 * 421 *
422 * This function will emit a batch buffer with the values required for 422 * This function will emit a batch buffer with the values required for
423 * programming the MOCS register values for all the currently supported 423 * programming the MOCS register values for all the currently supported
@@ -431,19 +431,19 @@ void intel_mocs_init_l3cc_table(struct drm_i915_private *dev_priv)
431 * 431 *
432 * Return: 0 on success, otherwise the error status. 432 * Return: 0 on success, otherwise the error status.
433 */ 433 */
434int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req) 434int intel_rcs_context_init_mocs(struct i915_request *rq)
435{ 435{
436 struct drm_i915_mocs_table t; 436 struct drm_i915_mocs_table t;
437 int ret; 437 int ret;
438 438
439 if (get_mocs_settings(req->i915, &t)) { 439 if (get_mocs_settings(rq->i915, &t)) {
440 /* Program the RCS control registers */ 440 /* Program the RCS control registers */
441 ret = emit_mocs_control_table(req, &t); 441 ret = emit_mocs_control_table(rq, &t);
442 if (ret) 442 if (ret)
443 return ret; 443 return ret;
444 444
445 /* Now program the l3cc registers */ 445 /* Now program the l3cc registers */
446 ret = emit_mocs_l3cc_table(req, &t); 446 ret = emit_mocs_l3cc_table(rq, &t);
447 if (ret) 447 if (ret)
448 return ret; 448 return ret;
449 } 449 }
diff --git a/drivers/gpu/drm/i915/intel_mocs.h b/drivers/gpu/drm/i915/intel_mocs.h
index ce4a5dfa5f94..d1751f91c1a4 100644
--- a/drivers/gpu/drm/i915/intel_mocs.h
+++ b/drivers/gpu/drm/i915/intel_mocs.h
@@ -52,7 +52,7 @@
52#include <drm/drmP.h> 52#include <drm/drmP.h>
53#include "i915_drv.h" 53#include "i915_drv.h"
54 54
55int intel_rcs_context_init_mocs(struct drm_i915_gem_request *req); 55int intel_rcs_context_init_mocs(struct i915_request *rq);
56void intel_mocs_init_l3cc_table(struct drm_i915_private *dev_priv); 56void intel_mocs_init_l3cc_table(struct drm_i915_private *dev_priv);
57int intel_mocs_init_engine(struct intel_engine_cs *engine); 57int intel_mocs_init_engine(struct intel_engine_cs *engine);
58 58
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 89f568e739ee..36671a937fa4 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -234,50 +234,50 @@ static void intel_overlay_unmap_regs(struct intel_overlay *overlay,
234} 234}
235 235
236static void intel_overlay_submit_request(struct intel_overlay *overlay, 236static void intel_overlay_submit_request(struct intel_overlay *overlay,
237 struct drm_i915_gem_request *req, 237 struct i915_request *rq,
238 i915_gem_retire_fn retire) 238 i915_gem_retire_fn retire)
239{ 239{
240 GEM_BUG_ON(i915_gem_active_peek(&overlay->last_flip, 240 GEM_BUG_ON(i915_gem_active_peek(&overlay->last_flip,
241 &overlay->i915->drm.struct_mutex)); 241 &overlay->i915->drm.struct_mutex));
242 i915_gem_active_set_retire_fn(&overlay->last_flip, retire, 242 i915_gem_active_set_retire_fn(&overlay->last_flip, retire,
243 &overlay->i915->drm.struct_mutex); 243 &overlay->i915->drm.struct_mutex);
244 i915_gem_active_set(&overlay->last_flip, req); 244 i915_gem_active_set(&overlay->last_flip, rq);
245 i915_add_request(req); 245 i915_request_add(rq);
246} 246}
247 247
248static int intel_overlay_do_wait_request(struct intel_overlay *overlay, 248static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
249 struct drm_i915_gem_request *req, 249 struct i915_request *rq,
250 i915_gem_retire_fn retire) 250 i915_gem_retire_fn retire)
251{ 251{
252 intel_overlay_submit_request(overlay, req, retire); 252 intel_overlay_submit_request(overlay, rq, retire);
253 return i915_gem_active_retire(&overlay->last_flip, 253 return i915_gem_active_retire(&overlay->last_flip,
254 &overlay->i915->drm.struct_mutex); 254 &overlay->i915->drm.struct_mutex);
255} 255}
256 256
257static struct drm_i915_gem_request *alloc_request(struct intel_overlay *overlay) 257static struct i915_request *alloc_request(struct intel_overlay *overlay)
258{ 258{
259 struct drm_i915_private *dev_priv = overlay->i915; 259 struct drm_i915_private *dev_priv = overlay->i915;
260 struct intel_engine_cs *engine = dev_priv->engine[RCS]; 260 struct intel_engine_cs *engine = dev_priv->engine[RCS];
261 261
262 return i915_gem_request_alloc(engine, dev_priv->kernel_context); 262 return i915_request_alloc(engine, dev_priv->kernel_context);
263} 263}
264 264
265/* overlay needs to be disable in OCMD reg */ 265/* overlay needs to be disable in OCMD reg */
266static int intel_overlay_on(struct intel_overlay *overlay) 266static int intel_overlay_on(struct intel_overlay *overlay)
267{ 267{
268 struct drm_i915_private *dev_priv = overlay->i915; 268 struct drm_i915_private *dev_priv = overlay->i915;
269 struct drm_i915_gem_request *req; 269 struct i915_request *rq;
270 u32 *cs; 270 u32 *cs;
271 271
272 WARN_ON(overlay->active); 272 WARN_ON(overlay->active);
273 273
274 req = alloc_request(overlay); 274 rq = alloc_request(overlay);
275 if (IS_ERR(req)) 275 if (IS_ERR(rq))
276 return PTR_ERR(req); 276 return PTR_ERR(rq);
277 277
278 cs = intel_ring_begin(req, 4); 278 cs = intel_ring_begin(rq, 4);
279 if (IS_ERR(cs)) { 279 if (IS_ERR(cs)) {
280 i915_add_request(req); 280 i915_request_add(rq);
281 return PTR_ERR(cs); 281 return PTR_ERR(cs);
282 } 282 }
283 283
@@ -290,9 +290,9 @@ static int intel_overlay_on(struct intel_overlay *overlay)
290 *cs++ = overlay->flip_addr | OFC_UPDATE; 290 *cs++ = overlay->flip_addr | OFC_UPDATE;
291 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP; 291 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
292 *cs++ = MI_NOOP; 292 *cs++ = MI_NOOP;
293 intel_ring_advance(req, cs); 293 intel_ring_advance(rq, cs);
294 294
295 return intel_overlay_do_wait_request(overlay, req, NULL); 295 return intel_overlay_do_wait_request(overlay, rq, NULL);
296} 296}
297 297
298static void intel_overlay_flip_prepare(struct intel_overlay *overlay, 298static void intel_overlay_flip_prepare(struct intel_overlay *overlay,
@@ -322,7 +322,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
322 bool load_polyphase_filter) 322 bool load_polyphase_filter)
323{ 323{
324 struct drm_i915_private *dev_priv = overlay->i915; 324 struct drm_i915_private *dev_priv = overlay->i915;
325 struct drm_i915_gem_request *req; 325 struct i915_request *rq;
326 u32 flip_addr = overlay->flip_addr; 326 u32 flip_addr = overlay->flip_addr;
327 u32 tmp, *cs; 327 u32 tmp, *cs;
328 328
@@ -336,23 +336,23 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
336 if (tmp & (1 << 17)) 336 if (tmp & (1 << 17))
337 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); 337 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
338 338
339 req = alloc_request(overlay); 339 rq = alloc_request(overlay);
340 if (IS_ERR(req)) 340 if (IS_ERR(rq))
341 return PTR_ERR(req); 341 return PTR_ERR(rq);
342 342
343 cs = intel_ring_begin(req, 2); 343 cs = intel_ring_begin(rq, 2);
344 if (IS_ERR(cs)) { 344 if (IS_ERR(cs)) {
345 i915_add_request(req); 345 i915_request_add(rq);
346 return PTR_ERR(cs); 346 return PTR_ERR(cs);
347 } 347 }
348 348
349 *cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE; 349 *cs++ = MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE;
350 *cs++ = flip_addr; 350 *cs++ = flip_addr;
351 intel_ring_advance(req, cs); 351 intel_ring_advance(rq, cs);
352 352
353 intel_overlay_flip_prepare(overlay, vma); 353 intel_overlay_flip_prepare(overlay, vma);
354 354
355 intel_overlay_submit_request(overlay, req, NULL); 355 intel_overlay_submit_request(overlay, rq, NULL);
356 356
357 return 0; 357 return 0;
358} 358}
@@ -373,7 +373,7 @@ static void intel_overlay_release_old_vma(struct intel_overlay *overlay)
373} 373}
374 374
375static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active, 375static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active,
376 struct drm_i915_gem_request *req) 376 struct i915_request *rq)
377{ 377{
378 struct intel_overlay *overlay = 378 struct intel_overlay *overlay =
379 container_of(active, typeof(*overlay), last_flip); 379 container_of(active, typeof(*overlay), last_flip);
@@ -382,7 +382,7 @@ static void intel_overlay_release_old_vid_tail(struct i915_gem_active *active,
382} 382}
383 383
384static void intel_overlay_off_tail(struct i915_gem_active *active, 384static void intel_overlay_off_tail(struct i915_gem_active *active,
385 struct drm_i915_gem_request *req) 385 struct i915_request *rq)
386{ 386{
387 struct intel_overlay *overlay = 387 struct intel_overlay *overlay =
388 container_of(active, typeof(*overlay), last_flip); 388 container_of(active, typeof(*overlay), last_flip);
@@ -401,7 +401,7 @@ static void intel_overlay_off_tail(struct i915_gem_active *active,
401/* overlay needs to be disabled in OCMD reg */ 401/* overlay needs to be disabled in OCMD reg */
402static int intel_overlay_off(struct intel_overlay *overlay) 402static int intel_overlay_off(struct intel_overlay *overlay)
403{ 403{
404 struct drm_i915_gem_request *req; 404 struct i915_request *rq;
405 u32 *cs, flip_addr = overlay->flip_addr; 405 u32 *cs, flip_addr = overlay->flip_addr;
406 406
407 WARN_ON(!overlay->active); 407 WARN_ON(!overlay->active);
@@ -412,13 +412,13 @@ static int intel_overlay_off(struct intel_overlay *overlay)
412 * of the hw. Do it in both cases */ 412 * of the hw. Do it in both cases */
413 flip_addr |= OFC_UPDATE; 413 flip_addr |= OFC_UPDATE;
414 414
415 req = alloc_request(overlay); 415 rq = alloc_request(overlay);
416 if (IS_ERR(req)) 416 if (IS_ERR(rq))
417 return PTR_ERR(req); 417 return PTR_ERR(rq);
418 418
419 cs = intel_ring_begin(req, 6); 419 cs = intel_ring_begin(rq, 6);
420 if (IS_ERR(cs)) { 420 if (IS_ERR(cs)) {
421 i915_add_request(req); 421 i915_request_add(rq);
422 return PTR_ERR(cs); 422 return PTR_ERR(cs);
423 } 423 }
424 424
@@ -432,11 +432,11 @@ static int intel_overlay_off(struct intel_overlay *overlay)
432 *cs++ = flip_addr; 432 *cs++ = flip_addr;
433 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP; 433 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
434 434
435 intel_ring_advance(req, cs); 435 intel_ring_advance(rq, cs);
436 436
437 intel_overlay_flip_prepare(overlay, NULL); 437 intel_overlay_flip_prepare(overlay, NULL);
438 438
439 return intel_overlay_do_wait_request(overlay, req, 439 return intel_overlay_do_wait_request(overlay, rq,
440 intel_overlay_off_tail); 440 intel_overlay_off_tail);
441} 441}
442 442
@@ -468,23 +468,23 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
468 468
469 if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) { 469 if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
470 /* synchronous slowpath */ 470 /* synchronous slowpath */
471 struct drm_i915_gem_request *req; 471 struct i915_request *rq;
472 472
473 req = alloc_request(overlay); 473 rq = alloc_request(overlay);
474 if (IS_ERR(req)) 474 if (IS_ERR(rq))
475 return PTR_ERR(req); 475 return PTR_ERR(rq);
476 476
477 cs = intel_ring_begin(req, 2); 477 cs = intel_ring_begin(rq, 2);
478 if (IS_ERR(cs)) { 478 if (IS_ERR(cs)) {
479 i915_add_request(req); 479 i915_request_add(rq);
480 return PTR_ERR(cs); 480 return PTR_ERR(cs);
481 } 481 }
482 482
483 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP; 483 *cs++ = MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP;
484 *cs++ = MI_NOOP; 484 *cs++ = MI_NOOP;
485 intel_ring_advance(req, cs); 485 intel_ring_advance(rq, cs);
486 486
487 ret = intel_overlay_do_wait_request(overlay, req, 487 ret = intel_overlay_do_wait_request(overlay, rq,
488 intel_overlay_release_old_vid_tail); 488 intel_overlay_release_old_vid_tail);
489 if (ret) 489 if (ret)
490 return ret; 490 return ret;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index abf80e462833..b8da4dcdd584 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -6360,7 +6360,7 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
6360 mutex_unlock(&dev_priv->pcu_lock); 6360 mutex_unlock(&dev_priv->pcu_lock);
6361} 6361}
6362 6362
6363void gen6_rps_boost(struct drm_i915_gem_request *rq, 6363void gen6_rps_boost(struct i915_request *rq,
6364 struct intel_rps_client *rps_client) 6364 struct intel_rps_client *rps_client)
6365{ 6365{
6366 struct intel_rps *rps = &rq->i915->gt_pm.rps; 6366 struct intel_rps *rps = &rq->i915->gt_pm.rps;
@@ -6376,7 +6376,7 @@ void gen6_rps_boost(struct drm_i915_gem_request *rq,
6376 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) 6376 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags))
6377 return; 6377 return;
6378 6378
6379 /* Serializes with i915_gem_request_retire() */ 6379 /* Serializes with i915_request_retire() */
6380 boost = false; 6380 boost = false;
6381 spin_lock_irqsave(&rq->lock, flags); 6381 spin_lock_irqsave(&rq->lock, flags);
6382 if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) { 6382 if (!rq->waitboost && !dma_fence_is_signaled_locked(&rq->fence)) {
@@ -6715,7 +6715,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6715 6715
6716 /* 6716 /*
6717 * 3b: Enable Coarse Power Gating only when RC6 is enabled. 6717 * 3b: Enable Coarse Power Gating only when RC6 is enabled.
6718 * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6. 6718 * WaRsDisableCoarsePowerGating:skl,cnl - Render/Media PG need to be disabled with RC6.
6719 */ 6719 */
6720 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv)) 6720 if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6721 I915_WRITE(GEN9_PG_ENABLE, 0); 6721 I915_WRITE(GEN9_PG_ENABLE, 0);
@@ -8026,7 +8026,10 @@ void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
8026 dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */ 8026 dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
8027 intel_disable_gt_powersave(dev_priv); 8027 intel_disable_gt_powersave(dev_priv);
8028 8028
8029 gen6_reset_rps_interrupts(dev_priv); 8029 if (INTEL_GEN(dev_priv) < 11)
8030 gen6_reset_rps_interrupts(dev_priv);
8031 else
8032 WARN_ON_ONCE(1);
8030} 8033}
8031 8034
8032static inline void intel_disable_llc_pstate(struct drm_i915_private *i915) 8035static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
@@ -8139,6 +8142,8 @@ static void intel_enable_rps(struct drm_i915_private *dev_priv)
8139 cherryview_enable_rps(dev_priv); 8142 cherryview_enable_rps(dev_priv);
8140 } else if (IS_VALLEYVIEW(dev_priv)) { 8143 } else if (IS_VALLEYVIEW(dev_priv)) {
8141 valleyview_enable_rps(dev_priv); 8144 valleyview_enable_rps(dev_priv);
8145 } else if (WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11)) {
8146 /* TODO */
8142 } else if (INTEL_GEN(dev_priv) >= 9) { 8147 } else if (INTEL_GEN(dev_priv) >= 9) {
8143 gen9_enable_rps(dev_priv); 8148 gen9_enable_rps(dev_priv);
8144 } else if (IS_BROADWELL(dev_priv)) { 8149 } else if (IS_BROADWELL(dev_priv)) {
@@ -8487,7 +8492,7 @@ static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8487 if (!HAS_PCH_CNP(dev_priv)) 8492 if (!HAS_PCH_CNP(dev_priv))
8488 return; 8493 return;
8489 8494
8490 /* Display WA #1181: cnp */ 8495 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
8491 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) | 8496 I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8492 CNP_PWM_CGE_GATING_DISABLE); 8497 CNP_PWM_CGE_GATING_DISABLE);
8493} 8498}
@@ -8517,7 +8522,13 @@ static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8517 val |= SARBUNIT_CLKGATE_DIS; 8522 val |= SARBUNIT_CLKGATE_DIS;
8518 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val); 8523 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8519 8524
8525 /* Wa_2201832410:cnl */
8526 val = I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE);
8527 val |= GWUNIT_CLKGATE_DIS;
8528 I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE, val);
8529
8520 /* WaDisableVFclkgate:cnl */ 8530 /* WaDisableVFclkgate:cnl */
8531 /* WaVFUnitClockGatingDisable:cnl */
8521 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE); 8532 val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8522 val |= VFUNIT_CLKGATE_DIS; 8533 val |= VFUNIT_CLKGATE_DIS;
8523 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val); 8534 I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 2ef374f936b9..23175c5c4a50 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -56,6 +56,111 @@
56#include "intel_drv.h" 56#include "intel_drv.h"
57#include "i915_drv.h" 57#include "i915_drv.h"
58 58
59static inline enum intel_display_power_domain
60psr_aux_domain(struct intel_dp *intel_dp)
61{
62 /* CNL HW requires corresponding AUX IOs to be powered up for PSR.
63 * However, for non-A AUX ports the corresponding non-EDP transcoders
64 * would have already enabled power well 2 and DC_OFF. This means we can
65 * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a
66 * specific AUX_IO reference without powering up any extra wells.
67 * Note that PSR is enabled only on Port A even though this function
68 * returns the correct domain for other ports too.
69 */
70 return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A :
71 intel_dp->aux_power_domain;
72}
73
74static void psr_aux_io_power_get(struct intel_dp *intel_dp)
75{
76 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
77 struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
78
79 if (INTEL_GEN(dev_priv) < 10)
80 return;
81
82 intel_display_power_get(dev_priv, psr_aux_domain(intel_dp));
83}
84
85static void psr_aux_io_power_put(struct intel_dp *intel_dp)
86{
87 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
88 struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
89
90 if (INTEL_GEN(dev_priv) < 10)
91 return;
92
93 intel_display_power_put(dev_priv, psr_aux_domain(intel_dp));
94}
95
96static bool intel_dp_get_y_cord_status(struct intel_dp *intel_dp)
97{
98 uint8_t psr_caps = 0;
99
100 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_CAPS, &psr_caps) != 1)
101 return false;
102 return psr_caps & DP_PSR2_SU_Y_COORDINATE_REQUIRED;
103}
104
105static bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp)
106{
107 uint8_t dprx = 0;
108
109 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST,
110 &dprx) != 1)
111 return false;
112 return dprx & DP_VSC_SDP_EXT_FOR_COLORIMETRY_SUPPORTED;
113}
114
115static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
116{
117 uint8_t alpm_caps = 0;
118
119 if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
120 &alpm_caps) != 1)
121 return false;
122 return alpm_caps & DP_ALPM_CAP;
123}
124
125void intel_psr_init_dpcd(struct intel_dp *intel_dp)
126{
127 struct drm_i915_private *dev_priv =
128 to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
129
130 drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
131 sizeof(intel_dp->psr_dpcd));
132
133 if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
134 dev_priv->psr.sink_support = true;
135 DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
136 }
137
138 if (INTEL_GEN(dev_priv) >= 9 &&
139 (intel_dp->psr_dpcd[0] & DP_PSR2_IS_SUPPORTED)) {
140 uint8_t frame_sync_cap;
141
142 dev_priv->psr.sink_support = true;
143 if (drm_dp_dpcd_readb(&intel_dp->aux,
144 DP_SINK_DEVICE_AUX_FRAME_SYNC_CAP,
145 &frame_sync_cap) != 1)
146 frame_sync_cap = 0;
147 dev_priv->psr.aux_frame_sync = frame_sync_cap & DP_AUX_FRAME_SYNC_CAP;
148 /* PSR2 needs frame sync as well */
149 dev_priv->psr.psr2_support = dev_priv->psr.aux_frame_sync;
150 DRM_DEBUG_KMS("PSR2 %s on sink",
151 dev_priv->psr.psr2_support ? "supported" : "not supported");
152
153 if (dev_priv->psr.psr2_support) {
154 dev_priv->psr.y_cord_support =
155 intel_dp_get_y_cord_status(intel_dp);
156 dev_priv->psr.colorimetry_support =
157 intel_dp_get_colorimetry_status(intel_dp);
158 dev_priv->psr.alpm =
159 intel_dp_get_alpm_status(intel_dp);
160 }
161 }
162}
163
59static bool vlv_is_psr_active_on_pipe(struct drm_device *dev, int pipe) 164static bool vlv_is_psr_active_on_pipe(struct drm_device *dev, int pipe)
60{ 165{
61 struct drm_i915_private *dev_priv = to_i915(dev); 166 struct drm_i915_private *dev_priv = to_i915(dev);
@@ -341,6 +446,50 @@ static void hsw_psr_activate(struct intel_dp *intel_dp)
341 hsw_activate_psr1(intel_dp); 446 hsw_activate_psr1(intel_dp);
342} 447}
343 448
449static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
450 struct intel_crtc_state *crtc_state)
451{
452 struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
453 struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
454 int crtc_hdisplay = crtc_state->base.adjusted_mode.crtc_hdisplay;
455 int crtc_vdisplay = crtc_state->base.adjusted_mode.crtc_vdisplay;
456 int psr_max_h = 0, psr_max_v = 0;
457
458 /*
459 * FIXME psr2_support is messed up. It's both computed
460 * dynamically during PSR enable, and extracted from sink
461 * caps during eDP detection.
462 */
463 if (!dev_priv->psr.psr2_support)
464 return false;
465
466 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
467 psr_max_h = 4096;
468 psr_max_v = 2304;
469 } else if (IS_GEN9(dev_priv)) {
470 psr_max_h = 3640;
471 psr_max_v = 2304;
472 }
473
474 if (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v) {
475 DRM_DEBUG_KMS("PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
476 crtc_hdisplay, crtc_vdisplay,
477 psr_max_h, psr_max_v);
478 return false;
479 }
480
481 /*
482 * FIXME:enable psr2 only for y-cordinate psr2 panels
483 * After gtc implementation , remove this restriction.
484 */
485 if (!dev_priv->psr.y_cord_support) {
486 DRM_DEBUG_KMS("PSR2 not enabled, panel does not support Y coordinate\n");
487 return false;
488 }
489
490 return true;
491}
492
344void intel_psr_compute_config(struct intel_dp *intel_dp, 493void intel_psr_compute_config(struct intel_dp *intel_dp,
345 struct intel_crtc_state *crtc_state) 494 struct intel_crtc_state *crtc_state)
346{ 495{
@@ -403,34 +552,14 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
403 return; 552 return;
404 } 553 }
405 554
406 /* 555 if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
407 * FIXME psr2_support is messed up. It's both computed 556 DRM_DEBUG_KMS("PSR condition failed: panel lacks power state control\n");
408 * dynamically during PSR enable, and extracted from sink
409 * caps during eDP detection.
410 */
411 if (!dev_priv->psr.psr2_support) {
412 crtc_state->has_psr = true;
413 return;
414 }
415
416 /* PSR2 is restricted to work with panel resolutions upto 3200x2000 */
417 if (adjusted_mode->crtc_hdisplay > 3200 ||
418 adjusted_mode->crtc_vdisplay > 2000) {
419 DRM_DEBUG_KMS("PSR2 disabled, panel resolution too big\n");
420 return;
421 }
422
423 /*
424 * FIXME:enable psr2 only for y-cordinate psr2 panels
425 * After gtc implementation , remove this restriction.
426 */
427 if (!dev_priv->psr.y_cord_support) {
428 DRM_DEBUG_KMS("PSR2 disabled, panel does not support Y coordinate\n");
429 return; 557 return;
430 } 558 }
431 559
432 crtc_state->has_psr = true; 560 crtc_state->has_psr = true;
433 crtc_state->has_psr2 = true; 561 crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
562 DRM_DEBUG_KMS("Enabling PSR%s\n", crtc_state->has_psr2 ? "2" : "");
434} 563}
435 564
436static void intel_psr_activate(struct intel_dp *intel_dp) 565static void intel_psr_activate(struct intel_dp *intel_dp)
@@ -459,6 +588,8 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp,
459 enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; 588 enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
460 u32 chicken; 589 u32 chicken;
461 590
591 psr_aux_io_power_get(intel_dp);
592
462 if (dev_priv->psr.psr2_support) { 593 if (dev_priv->psr.psr2_support) {
463 chicken = PSR2_VSC_ENABLE_PROG_HEADER; 594 chicken = PSR2_VSC_ENABLE_PROG_HEADER;
464 if (dev_priv->psr.y_cord_support) 595 if (dev_priv->psr.y_cord_support)
@@ -617,6 +748,8 @@ static void hsw_psr_disable(struct intel_dp *intel_dp,
617 else 748 else
618 WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); 749 WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
619 } 750 }
751
752 psr_aux_io_power_put(intel_dp);
620} 753}
621 754
622/** 755/**
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 5718f37160c5..1d599524a759 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -66,7 +66,7 @@ unsigned int intel_ring_update_space(struct intel_ring *ring)
66} 66}
67 67
68static int 68static int
69gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) 69gen2_render_ring_flush(struct i915_request *rq, u32 mode)
70{ 70{
71 u32 cmd, *cs; 71 u32 cmd, *cs;
72 72
@@ -75,19 +75,19 @@ gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
75 if (mode & EMIT_INVALIDATE) 75 if (mode & EMIT_INVALIDATE)
76 cmd |= MI_READ_FLUSH; 76 cmd |= MI_READ_FLUSH;
77 77
78 cs = intel_ring_begin(req, 2); 78 cs = intel_ring_begin(rq, 2);
79 if (IS_ERR(cs)) 79 if (IS_ERR(cs))
80 return PTR_ERR(cs); 80 return PTR_ERR(cs);
81 81
82 *cs++ = cmd; 82 *cs++ = cmd;
83 *cs++ = MI_NOOP; 83 *cs++ = MI_NOOP;
84 intel_ring_advance(req, cs); 84 intel_ring_advance(rq, cs);
85 85
86 return 0; 86 return 0;
87} 87}
88 88
89static int 89static int
90gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) 90gen4_render_ring_flush(struct i915_request *rq, u32 mode)
91{ 91{
92 u32 cmd, *cs; 92 u32 cmd, *cs;
93 93
@@ -122,17 +122,17 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
122 cmd = MI_FLUSH; 122 cmd = MI_FLUSH;
123 if (mode & EMIT_INVALIDATE) { 123 if (mode & EMIT_INVALIDATE) {
124 cmd |= MI_EXE_FLUSH; 124 cmd |= MI_EXE_FLUSH;
125 if (IS_G4X(req->i915) || IS_GEN5(req->i915)) 125 if (IS_G4X(rq->i915) || IS_GEN5(rq->i915))
126 cmd |= MI_INVALIDATE_ISP; 126 cmd |= MI_INVALIDATE_ISP;
127 } 127 }
128 128
129 cs = intel_ring_begin(req, 2); 129 cs = intel_ring_begin(rq, 2);
130 if (IS_ERR(cs)) 130 if (IS_ERR(cs))
131 return PTR_ERR(cs); 131 return PTR_ERR(cs);
132 132
133 *cs++ = cmd; 133 *cs++ = cmd;
134 *cs++ = MI_NOOP; 134 *cs++ = MI_NOOP;
135 intel_ring_advance(req, cs); 135 intel_ring_advance(rq, cs);
136 136
137 return 0; 137 return 0;
138} 138}
@@ -175,13 +175,13 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
175 * really our business. That leaves only stall at scoreboard. 175 * really our business. That leaves only stall at scoreboard.
176 */ 176 */
177static int 177static int
178intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req) 178intel_emit_post_sync_nonzero_flush(struct i915_request *rq)
179{ 179{
180 u32 scratch_addr = 180 u32 scratch_addr =
181 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; 181 i915_ggtt_offset(rq->engine->scratch) + 2 * CACHELINE_BYTES;
182 u32 *cs; 182 u32 *cs;
183 183
184 cs = intel_ring_begin(req, 6); 184 cs = intel_ring_begin(rq, 6);
185 if (IS_ERR(cs)) 185 if (IS_ERR(cs))
186 return PTR_ERR(cs); 186 return PTR_ERR(cs);
187 187
@@ -191,9 +191,9 @@ intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
191 *cs++ = 0; /* low dword */ 191 *cs++ = 0; /* low dword */
192 *cs++ = 0; /* high dword */ 192 *cs++ = 0; /* high dword */
193 *cs++ = MI_NOOP; 193 *cs++ = MI_NOOP;
194 intel_ring_advance(req, cs); 194 intel_ring_advance(rq, cs);
195 195
196 cs = intel_ring_begin(req, 6); 196 cs = intel_ring_begin(rq, 6);
197 if (IS_ERR(cs)) 197 if (IS_ERR(cs))
198 return PTR_ERR(cs); 198 return PTR_ERR(cs);
199 199
@@ -203,21 +203,21 @@ intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
203 *cs++ = 0; 203 *cs++ = 0;
204 *cs++ = 0; 204 *cs++ = 0;
205 *cs++ = MI_NOOP; 205 *cs++ = MI_NOOP;
206 intel_ring_advance(req, cs); 206 intel_ring_advance(rq, cs);
207 207
208 return 0; 208 return 0;
209} 209}
210 210
211static int 211static int
212gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) 212gen6_render_ring_flush(struct i915_request *rq, u32 mode)
213{ 213{
214 u32 scratch_addr = 214 u32 scratch_addr =
215 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; 215 i915_ggtt_offset(rq->engine->scratch) + 2 * CACHELINE_BYTES;
216 u32 *cs, flags = 0; 216 u32 *cs, flags = 0;
217 int ret; 217 int ret;
218 218
219 /* Force SNB workarounds for PIPE_CONTROL flushes */ 219 /* Force SNB workarounds for PIPE_CONTROL flushes */
220 ret = intel_emit_post_sync_nonzero_flush(req); 220 ret = intel_emit_post_sync_nonzero_flush(rq);
221 if (ret) 221 if (ret)
222 return ret; 222 return ret;
223 223
@@ -247,7 +247,7 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
247 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; 247 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
248 } 248 }
249 249
250 cs = intel_ring_begin(req, 4); 250 cs = intel_ring_begin(rq, 4);
251 if (IS_ERR(cs)) 251 if (IS_ERR(cs))
252 return PTR_ERR(cs); 252 return PTR_ERR(cs);
253 253
@@ -255,17 +255,17 @@ gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
255 *cs++ = flags; 255 *cs++ = flags;
256 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 256 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
257 *cs++ = 0; 257 *cs++ = 0;
258 intel_ring_advance(req, cs); 258 intel_ring_advance(rq, cs);
259 259
260 return 0; 260 return 0;
261} 261}
262 262
263static int 263static int
264gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req) 264gen7_render_ring_cs_stall_wa(struct i915_request *rq)
265{ 265{
266 u32 *cs; 266 u32 *cs;
267 267
268 cs = intel_ring_begin(req, 4); 268 cs = intel_ring_begin(rq, 4);
269 if (IS_ERR(cs)) 269 if (IS_ERR(cs))
270 return PTR_ERR(cs); 270 return PTR_ERR(cs);
271 271
@@ -273,16 +273,16 @@ gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
273 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 273 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
274 *cs++ = 0; 274 *cs++ = 0;
275 *cs++ = 0; 275 *cs++ = 0;
276 intel_ring_advance(req, cs); 276 intel_ring_advance(rq, cs);
277 277
278 return 0; 278 return 0;
279} 279}
280 280
281static int 281static int
282gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode) 282gen7_render_ring_flush(struct i915_request *rq, u32 mode)
283{ 283{
284 u32 scratch_addr = 284 u32 scratch_addr =
285 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES; 285 i915_ggtt_offset(rq->engine->scratch) + 2 * CACHELINE_BYTES;
286 u32 *cs, flags = 0; 286 u32 *cs, flags = 0;
287 287
288 /* 288 /*
@@ -324,10 +324,10 @@ gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
324 /* Workaround: we must issue a pipe_control with CS-stall bit 324 /* Workaround: we must issue a pipe_control with CS-stall bit
325 * set before a pipe_control command that has the state cache 325 * set before a pipe_control command that has the state cache
326 * invalidate bit set. */ 326 * invalidate bit set. */
327 gen7_render_ring_cs_stall_wa(req); 327 gen7_render_ring_cs_stall_wa(rq);
328 } 328 }
329 329
330 cs = intel_ring_begin(req, 4); 330 cs = intel_ring_begin(rq, 4);
331 if (IS_ERR(cs)) 331 if (IS_ERR(cs))
332 return PTR_ERR(cs); 332 return PTR_ERR(cs);
333 333
@@ -335,7 +335,7 @@ gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
335 *cs++ = flags; 335 *cs++ = flags;
336 *cs++ = scratch_addr; 336 *cs++ = scratch_addr;
337 *cs++ = 0; 337 *cs++ = 0;
338 intel_ring_advance(req, cs); 338 intel_ring_advance(rq, cs);
339 339
340 return 0; 340 return 0;
341} 341}
@@ -531,7 +531,7 @@ out:
531} 531}
532 532
533static void reset_ring_common(struct intel_engine_cs *engine, 533static void reset_ring_common(struct intel_engine_cs *engine,
534 struct drm_i915_gem_request *request) 534 struct i915_request *request)
535{ 535{
536 /* 536 /*
537 * RC6 must be prevented until the reset is complete and the engine 537 * RC6 must be prevented until the reset is complete and the engine
@@ -595,15 +595,15 @@ static void reset_ring_common(struct intel_engine_cs *engine,
595 } 595 }
596} 596}
597 597
598static int intel_rcs_ctx_init(struct drm_i915_gem_request *req) 598static int intel_rcs_ctx_init(struct i915_request *rq)
599{ 599{
600 int ret; 600 int ret;
601 601
602 ret = intel_ring_workarounds_emit(req); 602 ret = intel_ring_workarounds_emit(rq);
603 if (ret != 0) 603 if (ret != 0)
604 return ret; 604 return ret;
605 605
606 ret = i915_gem_render_state_emit(req); 606 ret = i915_gem_render_state_emit(rq);
607 if (ret) 607 if (ret)
608 return ret; 608 return ret;
609 609
@@ -661,9 +661,9 @@ static int init_render_ring(struct intel_engine_cs *engine)
661 return init_workarounds_ring(engine); 661 return init_workarounds_ring(engine);
662} 662}
663 663
664static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs) 664static u32 *gen6_signal(struct i915_request *rq, u32 *cs)
665{ 665{
666 struct drm_i915_private *dev_priv = req->i915; 666 struct drm_i915_private *dev_priv = rq->i915;
667 struct intel_engine_cs *engine; 667 struct intel_engine_cs *engine;
668 enum intel_engine_id id; 668 enum intel_engine_id id;
669 int num_rings = 0; 669 int num_rings = 0;
@@ -674,11 +674,11 @@ static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs)
674 if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK)) 674 if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
675 continue; 675 continue;
676 676
677 mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id]; 677 mbox_reg = rq->engine->semaphore.mbox.signal[engine->hw_id];
678 if (i915_mmio_reg_valid(mbox_reg)) { 678 if (i915_mmio_reg_valid(mbox_reg)) {
679 *cs++ = MI_LOAD_REGISTER_IMM(1); 679 *cs++ = MI_LOAD_REGISTER_IMM(1);
680 *cs++ = i915_mmio_reg_offset(mbox_reg); 680 *cs++ = i915_mmio_reg_offset(mbox_reg);
681 *cs++ = req->global_seqno; 681 *cs++ = rq->global_seqno;
682 num_rings++; 682 num_rings++;
683 } 683 }
684 } 684 }
@@ -690,7 +690,7 @@ static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs)
690 690
691static void cancel_requests(struct intel_engine_cs *engine) 691static void cancel_requests(struct intel_engine_cs *engine)
692{ 692{
693 struct drm_i915_gem_request *request; 693 struct i915_request *request;
694 unsigned long flags; 694 unsigned long flags;
695 695
696 spin_lock_irqsave(&engine->timeline->lock, flags); 696 spin_lock_irqsave(&engine->timeline->lock, flags);
@@ -698,7 +698,7 @@ static void cancel_requests(struct intel_engine_cs *engine)
698 /* Mark all submitted requests as skipped. */ 698 /* Mark all submitted requests as skipped. */
699 list_for_each_entry(request, &engine->timeline->requests, link) { 699 list_for_each_entry(request, &engine->timeline->requests, link) {
700 GEM_BUG_ON(!request->global_seqno); 700 GEM_BUG_ON(!request->global_seqno);
701 if (!i915_gem_request_completed(request)) 701 if (!i915_request_completed(request))
702 dma_fence_set_error(&request->fence, -EIO); 702 dma_fence_set_error(&request->fence, -EIO);
703 } 703 }
704 /* Remaining _unready_ requests will be nop'ed when submitted */ 704 /* Remaining _unready_ requests will be nop'ed when submitted */
@@ -706,48 +706,46 @@ static void cancel_requests(struct intel_engine_cs *engine)
706 spin_unlock_irqrestore(&engine->timeline->lock, flags); 706 spin_unlock_irqrestore(&engine->timeline->lock, flags);
707} 707}
708 708
709static void i9xx_submit_request(struct drm_i915_gem_request *request) 709static void i9xx_submit_request(struct i915_request *request)
710{ 710{
711 struct drm_i915_private *dev_priv = request->i915; 711 struct drm_i915_private *dev_priv = request->i915;
712 712
713 i915_gem_request_submit(request); 713 i915_request_submit(request);
714 714
715 I915_WRITE_TAIL(request->engine, 715 I915_WRITE_TAIL(request->engine,
716 intel_ring_set_tail(request->ring, request->tail)); 716 intel_ring_set_tail(request->ring, request->tail));
717} 717}
718 718
719static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs) 719static void i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
720{ 720{
721 *cs++ = MI_STORE_DWORD_INDEX; 721 *cs++ = MI_STORE_DWORD_INDEX;
722 *cs++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT; 722 *cs++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
723 *cs++ = req->global_seqno; 723 *cs++ = rq->global_seqno;
724 *cs++ = MI_USER_INTERRUPT; 724 *cs++ = MI_USER_INTERRUPT;
725 725
726 req->tail = intel_ring_offset(req, cs); 726 rq->tail = intel_ring_offset(rq, cs);
727 assert_ring_tail_valid(req->ring, req->tail); 727 assert_ring_tail_valid(rq->ring, rq->tail);
728} 728}
729 729
730static const int i9xx_emit_breadcrumb_sz = 4; 730static const int i9xx_emit_breadcrumb_sz = 4;
731 731
732static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs) 732static void gen6_sema_emit_breadcrumb(struct i915_request *rq, u32 *cs)
733{ 733{
734 return i9xx_emit_breadcrumb(req, 734 return i9xx_emit_breadcrumb(rq, rq->engine->semaphore.signal(rq, cs));
735 req->engine->semaphore.signal(req, cs));
736} 735}
737 736
738static int 737static int
739gen6_ring_sync_to(struct drm_i915_gem_request *req, 738gen6_ring_sync_to(struct i915_request *rq, struct i915_request *signal)
740 struct drm_i915_gem_request *signal)
741{ 739{
742 u32 dw1 = MI_SEMAPHORE_MBOX | 740 u32 dw1 = MI_SEMAPHORE_MBOX |
743 MI_SEMAPHORE_COMPARE | 741 MI_SEMAPHORE_COMPARE |
744 MI_SEMAPHORE_REGISTER; 742 MI_SEMAPHORE_REGISTER;
745 u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id]; 743 u32 wait_mbox = signal->engine->semaphore.mbox.wait[rq->engine->hw_id];
746 u32 *cs; 744 u32 *cs;
747 745
748 WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID); 746 WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
749 747
750 cs = intel_ring_begin(req, 4); 748 cs = intel_ring_begin(rq, 4);
751 if (IS_ERR(cs)) 749 if (IS_ERR(cs))
752 return PTR_ERR(cs); 750 return PTR_ERR(cs);
753 751
@@ -759,7 +757,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req,
759 *cs++ = signal->global_seqno - 1; 757 *cs++ = signal->global_seqno - 1;
760 *cs++ = 0; 758 *cs++ = 0;
761 *cs++ = MI_NOOP; 759 *cs++ = MI_NOOP;
762 intel_ring_advance(req, cs); 760 intel_ring_advance(rq, cs);
763 761
764 return 0; 762 return 0;
765} 763}
@@ -858,17 +856,17 @@ i8xx_irq_disable(struct intel_engine_cs *engine)
858} 856}
859 857
860static int 858static int
861bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode) 859bsd_ring_flush(struct i915_request *rq, u32 mode)
862{ 860{
863 u32 *cs; 861 u32 *cs;
864 862
865 cs = intel_ring_begin(req, 2); 863 cs = intel_ring_begin(rq, 2);
866 if (IS_ERR(cs)) 864 if (IS_ERR(cs))
867 return PTR_ERR(cs); 865 return PTR_ERR(cs);
868 866
869 *cs++ = MI_FLUSH; 867 *cs++ = MI_FLUSH;
870 *cs++ = MI_NOOP; 868 *cs++ = MI_NOOP;
871 intel_ring_advance(req, cs); 869 intel_ring_advance(rq, cs);
872 return 0; 870 return 0;
873} 871}
874 872
@@ -911,20 +909,20 @@ hsw_vebox_irq_disable(struct intel_engine_cs *engine)
911} 909}
912 910
913static int 911static int
914i965_emit_bb_start(struct drm_i915_gem_request *req, 912i965_emit_bb_start(struct i915_request *rq,
915 u64 offset, u32 length, 913 u64 offset, u32 length,
916 unsigned int dispatch_flags) 914 unsigned int dispatch_flags)
917{ 915{
918 u32 *cs; 916 u32 *cs;
919 917
920 cs = intel_ring_begin(req, 2); 918 cs = intel_ring_begin(rq, 2);
921 if (IS_ERR(cs)) 919 if (IS_ERR(cs))
922 return PTR_ERR(cs); 920 return PTR_ERR(cs);
923 921
924 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags & 922 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &
925 I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965); 923 I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965);
926 *cs++ = offset; 924 *cs++ = offset;
927 intel_ring_advance(req, cs); 925 intel_ring_advance(rq, cs);
928 926
929 return 0; 927 return 0;
930} 928}
@@ -934,13 +932,13 @@ i965_emit_bb_start(struct drm_i915_gem_request *req,
934#define I830_TLB_ENTRIES (2) 932#define I830_TLB_ENTRIES (2)
935#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) 933#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
936static int 934static int
937i830_emit_bb_start(struct drm_i915_gem_request *req, 935i830_emit_bb_start(struct i915_request *rq,
938 u64 offset, u32 len, 936 u64 offset, u32 len,
939 unsigned int dispatch_flags) 937 unsigned int dispatch_flags)
940{ 938{
941 u32 *cs, cs_offset = i915_ggtt_offset(req->engine->scratch); 939 u32 *cs, cs_offset = i915_ggtt_offset(rq->engine->scratch);
942 940
943 cs = intel_ring_begin(req, 6); 941 cs = intel_ring_begin(rq, 6);
944 if (IS_ERR(cs)) 942 if (IS_ERR(cs))
945 return PTR_ERR(cs); 943 return PTR_ERR(cs);
946 944
@@ -951,13 +949,13 @@ i830_emit_bb_start(struct drm_i915_gem_request *req,
951 *cs++ = cs_offset; 949 *cs++ = cs_offset;
952 *cs++ = 0xdeadbeef; 950 *cs++ = 0xdeadbeef;
953 *cs++ = MI_NOOP; 951 *cs++ = MI_NOOP;
954 intel_ring_advance(req, cs); 952 intel_ring_advance(rq, cs);
955 953
956 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) { 954 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
957 if (len > I830_BATCH_LIMIT) 955 if (len > I830_BATCH_LIMIT)
958 return -ENOSPC; 956 return -ENOSPC;
959 957
960 cs = intel_ring_begin(req, 6 + 2); 958 cs = intel_ring_begin(rq, 6 + 2);
961 if (IS_ERR(cs)) 959 if (IS_ERR(cs))
962 return PTR_ERR(cs); 960 return PTR_ERR(cs);
963 961
@@ -974,39 +972,39 @@ i830_emit_bb_start(struct drm_i915_gem_request *req,
974 972
975 *cs++ = MI_FLUSH; 973 *cs++ = MI_FLUSH;
976 *cs++ = MI_NOOP; 974 *cs++ = MI_NOOP;
977 intel_ring_advance(req, cs); 975 intel_ring_advance(rq, cs);
978 976
979 /* ... and execute it. */ 977 /* ... and execute it. */
980 offset = cs_offset; 978 offset = cs_offset;
981 } 979 }
982 980
983 cs = intel_ring_begin(req, 2); 981 cs = intel_ring_begin(rq, 2);
984 if (IS_ERR(cs)) 982 if (IS_ERR(cs))
985 return PTR_ERR(cs); 983 return PTR_ERR(cs);
986 984
987 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 985 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
988 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 986 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
989 MI_BATCH_NON_SECURE); 987 MI_BATCH_NON_SECURE);
990 intel_ring_advance(req, cs); 988 intel_ring_advance(rq, cs);
991 989
992 return 0; 990 return 0;
993} 991}
994 992
995static int 993static int
996i915_emit_bb_start(struct drm_i915_gem_request *req, 994i915_emit_bb_start(struct i915_request *rq,
997 u64 offset, u32 len, 995 u64 offset, u32 len,
998 unsigned int dispatch_flags) 996 unsigned int dispatch_flags)
999{ 997{
1000 u32 *cs; 998 u32 *cs;
1001 999
1002 cs = intel_ring_begin(req, 2); 1000 cs = intel_ring_begin(rq, 2);
1003 if (IS_ERR(cs)) 1001 if (IS_ERR(cs))
1004 return PTR_ERR(cs); 1002 return PTR_ERR(cs);
1005 1003
1006 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1004 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1007 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1005 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1008 MI_BATCH_NON_SECURE); 1006 MI_BATCH_NON_SECURE);
1009 intel_ring_advance(req, cs); 1007 intel_ring_advance(rq, cs);
1010 1008
1011 return 0; 1009 return 0;
1012} 1010}
@@ -1377,7 +1375,7 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
1377 intel_ring_reset(engine->buffer, 0); 1375 intel_ring_reset(engine->buffer, 0);
1378} 1376}
1379 1377
1380static inline int mi_set_context(struct drm_i915_gem_request *rq, u32 flags) 1378static inline int mi_set_context(struct i915_request *rq, u32 flags)
1381{ 1379{
1382 struct drm_i915_private *i915 = rq->i915; 1380 struct drm_i915_private *i915 = rq->i915;
1383 struct intel_engine_cs *engine = rq->engine; 1381 struct intel_engine_cs *engine = rq->engine;
@@ -1463,7 +1461,7 @@ static inline int mi_set_context(struct drm_i915_gem_request *rq, u32 flags)
1463 return 0; 1461 return 0;
1464} 1462}
1465 1463
1466static int remap_l3(struct drm_i915_gem_request *rq, int slice) 1464static int remap_l3(struct i915_request *rq, int slice)
1467{ 1465{
1468 u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice]; 1466 u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice];
1469 int i; 1467 int i;
@@ -1491,7 +1489,7 @@ static int remap_l3(struct drm_i915_gem_request *rq, int slice)
1491 return 0; 1489 return 0;
1492} 1490}
1493 1491
1494static int switch_context(struct drm_i915_gem_request *rq) 1492static int switch_context(struct i915_request *rq)
1495{ 1493{
1496 struct intel_engine_cs *engine = rq->engine; 1494 struct intel_engine_cs *engine = rq->engine;
1497 struct i915_gem_context *to_ctx = rq->ctx; 1495 struct i915_gem_context *to_ctx = rq->ctx;
@@ -1561,7 +1559,7 @@ err:
1561 return ret; 1559 return ret;
1562} 1560}
1563 1561
1564static int ring_request_alloc(struct drm_i915_gem_request *request) 1562static int ring_request_alloc(struct i915_request *request)
1565{ 1563{
1566 int ret; 1564 int ret;
1567 1565
@@ -1587,7 +1585,7 @@ static int ring_request_alloc(struct drm_i915_gem_request *request)
1587 1585
1588static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes) 1586static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes)
1589{ 1587{
1590 struct drm_i915_gem_request *target; 1588 struct i915_request *target;
1591 long timeout; 1589 long timeout;
1592 1590
1593 lockdep_assert_held(&ring->vma->vm->i915->drm.struct_mutex); 1591 lockdep_assert_held(&ring->vma->vm->i915->drm.struct_mutex);
@@ -1605,13 +1603,13 @@ static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes)
1605 if (WARN_ON(&target->ring_link == &ring->request_list)) 1603 if (WARN_ON(&target->ring_link == &ring->request_list))
1606 return -ENOSPC; 1604 return -ENOSPC;
1607 1605
1608 timeout = i915_wait_request(target, 1606 timeout = i915_request_wait(target,
1609 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, 1607 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
1610 MAX_SCHEDULE_TIMEOUT); 1608 MAX_SCHEDULE_TIMEOUT);
1611 if (timeout < 0) 1609 if (timeout < 0)
1612 return timeout; 1610 return timeout;
1613 1611
1614 i915_gem_request_retire_upto(target); 1612 i915_request_retire_upto(target);
1615 1613
1616 intel_ring_update_space(ring); 1614 intel_ring_update_space(ring);
1617 GEM_BUG_ON(ring->space < bytes); 1615 GEM_BUG_ON(ring->space < bytes);
@@ -1634,10 +1632,9 @@ int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes)
1634 return 0; 1632 return 0;
1635} 1633}
1636 1634
1637u32 *intel_ring_begin(struct drm_i915_gem_request *req, 1635u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
1638 unsigned int num_dwords)
1639{ 1636{
1640 struct intel_ring *ring = req->ring; 1637 struct intel_ring *ring = rq->ring;
1641 const unsigned int remain_usable = ring->effective_size - ring->emit; 1638 const unsigned int remain_usable = ring->effective_size - ring->emit;
1642 const unsigned int bytes = num_dwords * sizeof(u32); 1639 const unsigned int bytes = num_dwords * sizeof(u32);
1643 unsigned int need_wrap = 0; 1640 unsigned int need_wrap = 0;
@@ -1647,7 +1644,7 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1647 /* Packets must be qword aligned. */ 1644 /* Packets must be qword aligned. */
1648 GEM_BUG_ON(num_dwords & 1); 1645 GEM_BUG_ON(num_dwords & 1);
1649 1646
1650 total_bytes = bytes + req->reserved_space; 1647 total_bytes = bytes + rq->reserved_space;
1651 GEM_BUG_ON(total_bytes > ring->effective_size); 1648 GEM_BUG_ON(total_bytes > ring->effective_size);
1652 1649
1653 if (unlikely(total_bytes > remain_usable)) { 1650 if (unlikely(total_bytes > remain_usable)) {
@@ -1668,7 +1665,7 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1668 * wrap and only need to effectively wait for the 1665 * wrap and only need to effectively wait for the
1669 * reserved size from the start of ringbuffer. 1666 * reserved size from the start of ringbuffer.
1670 */ 1667 */
1671 total_bytes = req->reserved_space + remain_actual; 1668 total_bytes = rq->reserved_space + remain_actual;
1672 } 1669 }
1673 } 1670 }
1674 1671
@@ -1682,9 +1679,9 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1682 * overallocation and the assumption is that then we never need 1679 * overallocation and the assumption is that then we never need
1683 * to wait (which has the risk of failing with EINTR). 1680 * to wait (which has the risk of failing with EINTR).
1684 * 1681 *
1685 * See also i915_gem_request_alloc() and i915_add_request(). 1682 * See also i915_request_alloc() and i915_request_add().
1686 */ 1683 */
1687 GEM_BUG_ON(!req->reserved_space); 1684 GEM_BUG_ON(!rq->reserved_space);
1688 1685
1689 ret = wait_for_space(ring, total_bytes); 1686 ret = wait_for_space(ring, total_bytes);
1690 if (unlikely(ret)) 1687 if (unlikely(ret))
@@ -1713,29 +1710,28 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1713} 1710}
1714 1711
1715/* Align the ring tail to a cacheline boundary */ 1712/* Align the ring tail to a cacheline boundary */
1716int intel_ring_cacheline_align(struct drm_i915_gem_request *req) 1713int intel_ring_cacheline_align(struct i915_request *rq)
1717{ 1714{
1718 int num_dwords = 1715 int num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
1719 (req->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
1720 u32 *cs; 1716 u32 *cs;
1721 1717
1722 if (num_dwords == 0) 1718 if (num_dwords == 0)
1723 return 0; 1719 return 0;
1724 1720
1725 num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords; 1721 num_dwords = CACHELINE_BYTES / sizeof(u32) - num_dwords;
1726 cs = intel_ring_begin(req, num_dwords); 1722 cs = intel_ring_begin(rq, num_dwords);
1727 if (IS_ERR(cs)) 1723 if (IS_ERR(cs))
1728 return PTR_ERR(cs); 1724 return PTR_ERR(cs);
1729 1725
1730 while (num_dwords--) 1726 while (num_dwords--)
1731 *cs++ = MI_NOOP; 1727 *cs++ = MI_NOOP;
1732 1728
1733 intel_ring_advance(req, cs); 1729 intel_ring_advance(rq, cs);
1734 1730
1735 return 0; 1731 return 0;
1736} 1732}
1737 1733
1738static void gen6_bsd_submit_request(struct drm_i915_gem_request *request) 1734static void gen6_bsd_submit_request(struct i915_request *request)
1739{ 1735{
1740 struct drm_i915_private *dev_priv = request->i915; 1736 struct drm_i915_private *dev_priv = request->i915;
1741 1737
@@ -1772,11 +1768,11 @@ static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
1772 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 1768 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1773} 1769}
1774 1770
1775static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode) 1771static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode)
1776{ 1772{
1777 u32 cmd, *cs; 1773 u32 cmd, *cs;
1778 1774
1779 cs = intel_ring_begin(req, 4); 1775 cs = intel_ring_begin(rq, 4);
1780 if (IS_ERR(cs)) 1776 if (IS_ERR(cs))
1781 return PTR_ERR(cs); 1777 return PTR_ERR(cs);
1782 1778
@@ -1802,18 +1798,18 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1802 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1798 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1803 *cs++ = 0; 1799 *cs++ = 0;
1804 *cs++ = MI_NOOP; 1800 *cs++ = MI_NOOP;
1805 intel_ring_advance(req, cs); 1801 intel_ring_advance(rq, cs);
1806 return 0; 1802 return 0;
1807} 1803}
1808 1804
1809static int 1805static int
1810hsw_emit_bb_start(struct drm_i915_gem_request *req, 1806hsw_emit_bb_start(struct i915_request *rq,
1811 u64 offset, u32 len, 1807 u64 offset, u32 len,
1812 unsigned int dispatch_flags) 1808 unsigned int dispatch_flags)
1813{ 1809{
1814 u32 *cs; 1810 u32 *cs;
1815 1811
1816 cs = intel_ring_begin(req, 2); 1812 cs = intel_ring_begin(rq, 2);
1817 if (IS_ERR(cs)) 1813 if (IS_ERR(cs))
1818 return PTR_ERR(cs); 1814 return PTR_ERR(cs);
1819 1815
@@ -1823,19 +1819,19 @@ hsw_emit_bb_start(struct drm_i915_gem_request *req,
1823 MI_BATCH_RESOURCE_STREAMER : 0); 1819 MI_BATCH_RESOURCE_STREAMER : 0);
1824 /* bit0-7 is the length on GEN6+ */ 1820 /* bit0-7 is the length on GEN6+ */
1825 *cs++ = offset; 1821 *cs++ = offset;
1826 intel_ring_advance(req, cs); 1822 intel_ring_advance(rq, cs);
1827 1823
1828 return 0; 1824 return 0;
1829} 1825}
1830 1826
1831static int 1827static int
1832gen6_emit_bb_start(struct drm_i915_gem_request *req, 1828gen6_emit_bb_start(struct i915_request *rq,
1833 u64 offset, u32 len, 1829 u64 offset, u32 len,
1834 unsigned int dispatch_flags) 1830 unsigned int dispatch_flags)
1835{ 1831{
1836 u32 *cs; 1832 u32 *cs;
1837 1833
1838 cs = intel_ring_begin(req, 2); 1834 cs = intel_ring_begin(rq, 2);
1839 if (IS_ERR(cs)) 1835 if (IS_ERR(cs))
1840 return PTR_ERR(cs); 1836 return PTR_ERR(cs);
1841 1837
@@ -1843,18 +1839,18 @@ gen6_emit_bb_start(struct drm_i915_gem_request *req,
1843 0 : MI_BATCH_NON_SECURE_I965); 1839 0 : MI_BATCH_NON_SECURE_I965);
1844 /* bit0-7 is the length on GEN6+ */ 1840 /* bit0-7 is the length on GEN6+ */
1845 *cs++ = offset; 1841 *cs++ = offset;
1846 intel_ring_advance(req, cs); 1842 intel_ring_advance(rq, cs);
1847 1843
1848 return 0; 1844 return 0;
1849} 1845}
1850 1846
1851/* Blitter support (SandyBridge+) */ 1847/* Blitter support (SandyBridge+) */
1852 1848
1853static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode) 1849static int gen6_ring_flush(struct i915_request *rq, u32 mode)
1854{ 1850{
1855 u32 cmd, *cs; 1851 u32 cmd, *cs;
1856 1852
1857 cs = intel_ring_begin(req, 4); 1853 cs = intel_ring_begin(rq, 4);
1858 if (IS_ERR(cs)) 1854 if (IS_ERR(cs))
1859 return PTR_ERR(cs); 1855 return PTR_ERR(cs);
1860 1856
@@ -1879,7 +1875,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1879 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1875 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1880 *cs++ = 0; 1876 *cs++ = 0;
1881 *cs++ = MI_NOOP; 1877 *cs++ = MI_NOOP;
1882 intel_ring_advance(req, cs); 1878 intel_ring_advance(rq, cs);
1883 1879
1884 return 0; 1880 return 0;
1885} 1881}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 51523ad049de..0320c2c4cfba 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -3,10 +3,12 @@
3#define _INTEL_RINGBUFFER_H_ 3#define _INTEL_RINGBUFFER_H_
4 4
5#include <linux/hashtable.h> 5#include <linux/hashtable.h>
6
6#include "i915_gem_batch_pool.h" 7#include "i915_gem_batch_pool.h"
7#include "i915_gem_request.h"
8#include "i915_gem_timeline.h" 8#include "i915_gem_timeline.h"
9
9#include "i915_pmu.h" 10#include "i915_pmu.h"
11#include "i915_request.h"
10#include "i915_selftest.h" 12#include "i915_selftest.h"
11 13
12struct drm_printer; 14struct drm_printer;
@@ -90,7 +92,7 @@ hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
90 92
91#define instdone_subslice_mask(dev_priv__) \ 93#define instdone_subslice_mask(dev_priv__) \
92 (INTEL_GEN(dev_priv__) == 7 ? \ 94 (INTEL_GEN(dev_priv__) == 7 ? \
93 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask) 95 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask[0])
94 96
95#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \ 97#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
96 for ((slice__) = 0, (subslice__) = 0; \ 98 for ((slice__) = 0, (subslice__) = 0; \
@@ -115,7 +117,7 @@ struct intel_engine_hangcheck {
115 unsigned long action_timestamp; 117 unsigned long action_timestamp;
116 int deadlock; 118 int deadlock;
117 struct intel_instdone instdone; 119 struct intel_instdone instdone;
118 struct drm_i915_gem_request *active_request; 120 struct i915_request *active_request;
119 bool stalled; 121 bool stalled;
120}; 122};
121 123
@@ -156,7 +158,10 @@ struct i915_ctx_workarounds {
156 struct i915_vma *vma; 158 struct i915_vma *vma;
157}; 159};
158 160
159struct drm_i915_gem_request; 161struct i915_request;
162
163#define I915_MAX_VCS 4
164#define I915_MAX_VECS 2
160 165
161/* 166/*
162 * Engine IDs definitions. 167 * Engine IDs definitions.
@@ -167,8 +172,12 @@ enum intel_engine_id {
167 BCS, 172 BCS,
168 VCS, 173 VCS,
169 VCS2, 174 VCS2,
175 VCS3,
176 VCS4,
170#define _VCS(n) (VCS + (n)) 177#define _VCS(n) (VCS + (n))
171 VECS 178 VECS,
179 VECS2
180#define _VECS(n) (VECS + (n))
172}; 181};
173 182
174struct i915_priolist { 183struct i915_priolist {
@@ -200,9 +209,17 @@ struct intel_engine_execlists {
200 bool no_priolist; 209 bool no_priolist;
201 210
202 /** 211 /**
203 * @elsp: the ExecList Submission Port register 212 * @submit_reg: gen-specific execlist submission register
213 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to
214 * the ExecList Submission Queue Contents register array for Gen11+
204 */ 215 */
205 u32 __iomem *elsp; 216 u32 __iomem *submit_reg;
217
218 /**
219 * @ctrl_reg: the enhanced execlists control register, used to load the
220 * submit queue on the HW and to request preemptions to idle
221 */
222 u32 __iomem *ctrl_reg;
206 223
207 /** 224 /**
208 * @port: execlist port states 225 * @port: execlist port states
@@ -218,7 +235,7 @@ struct intel_engine_execlists {
218 /** 235 /**
219 * @request_count: combined request and submission count 236 * @request_count: combined request and submission count
220 */ 237 */
221 struct drm_i915_gem_request *request_count; 238 struct i915_request *request_count;
222#define EXECLIST_COUNT_BITS 2 239#define EXECLIST_COUNT_BITS 2
223#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS) 240#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
224#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS) 241#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
@@ -256,6 +273,16 @@ struct intel_engine_execlists {
256 unsigned int port_mask; 273 unsigned int port_mask;
257 274
258 /** 275 /**
276 * @queue_priority: Highest pending priority.
277 *
278 * When we add requests into the queue, or adjust the priority of
279 * executing requests, we compute the maximum priority of those
280 * pending requests. We can then use this value to determine if
281 * we need to preempt the executing requests to service the queue.
282 */
283 int queue_priority;
284
285 /**
259 * @queue: queue of requests, in priority lists 286 * @queue: queue of requests, in priority lists
260 */ 287 */
261 struct rb_root queue; 288 struct rb_root queue;
@@ -337,9 +364,9 @@ struct intel_engine_cs {
337 364
338 spinlock_t rb_lock; /* protects the rb and wraps irq_lock */ 365 spinlock_t rb_lock; /* protects the rb and wraps irq_lock */
339 struct rb_root waiters; /* sorted by retirement, priority */ 366 struct rb_root waiters; /* sorted by retirement, priority */
340 struct rb_root signals; /* sorted by retirement */ 367 struct list_head signals; /* sorted by retirement */
341 struct task_struct *signaler; /* used for fence signalling */ 368 struct task_struct *signaler; /* used for fence signalling */
342 struct drm_i915_gem_request __rcu *first_signal; 369
343 struct timer_list fake_irq; /* used after a missed interrupt */ 370 struct timer_list fake_irq; /* used after a missed interrupt */
344 struct timer_list hangcheck; /* detect missed interrupts */ 371 struct timer_list hangcheck; /* detect missed interrupts */
345 372
@@ -391,7 +418,7 @@ struct intel_engine_cs {
391 418
392 int (*init_hw)(struct intel_engine_cs *engine); 419 int (*init_hw)(struct intel_engine_cs *engine);
393 void (*reset_hw)(struct intel_engine_cs *engine, 420 void (*reset_hw)(struct intel_engine_cs *engine,
394 struct drm_i915_gem_request *req); 421 struct i915_request *rq);
395 422
396 void (*park)(struct intel_engine_cs *engine); 423 void (*park)(struct intel_engine_cs *engine);
397 void (*unpark)(struct intel_engine_cs *engine); 424 void (*unpark)(struct intel_engine_cs *engine);
@@ -402,22 +429,20 @@ struct intel_engine_cs {
402 struct i915_gem_context *ctx); 429 struct i915_gem_context *ctx);
403 void (*context_unpin)(struct intel_engine_cs *engine, 430 void (*context_unpin)(struct intel_engine_cs *engine,
404 struct i915_gem_context *ctx); 431 struct i915_gem_context *ctx);
405 int (*request_alloc)(struct drm_i915_gem_request *req); 432 int (*request_alloc)(struct i915_request *rq);
406 int (*init_context)(struct drm_i915_gem_request *req); 433 int (*init_context)(struct i915_request *rq);
407 434
408 int (*emit_flush)(struct drm_i915_gem_request *request, 435 int (*emit_flush)(struct i915_request *request, u32 mode);
409 u32 mode);
410#define EMIT_INVALIDATE BIT(0) 436#define EMIT_INVALIDATE BIT(0)
411#define EMIT_FLUSH BIT(1) 437#define EMIT_FLUSH BIT(1)
412#define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH) 438#define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
413 int (*emit_bb_start)(struct drm_i915_gem_request *req, 439 int (*emit_bb_start)(struct i915_request *rq,
414 u64 offset, u32 length, 440 u64 offset, u32 length,
415 unsigned int dispatch_flags); 441 unsigned int dispatch_flags);
416#define I915_DISPATCH_SECURE BIT(0) 442#define I915_DISPATCH_SECURE BIT(0)
417#define I915_DISPATCH_PINNED BIT(1) 443#define I915_DISPATCH_PINNED BIT(1)
418#define I915_DISPATCH_RS BIT(2) 444#define I915_DISPATCH_RS BIT(2)
419 void (*emit_breadcrumb)(struct drm_i915_gem_request *req, 445 void (*emit_breadcrumb)(struct i915_request *rq, u32 *cs);
420 u32 *cs);
421 int emit_breadcrumb_sz; 446 int emit_breadcrumb_sz;
422 447
423 /* Pass the request to the hardware queue (e.g. directly into 448 /* Pass the request to the hardware queue (e.g. directly into
@@ -426,7 +451,7 @@ struct intel_engine_cs {
426 * This is called from an atomic context with irqs disabled; must 451 * This is called from an atomic context with irqs disabled; must
427 * be irq safe. 452 * be irq safe.
428 */ 453 */
429 void (*submit_request)(struct drm_i915_gem_request *req); 454 void (*submit_request)(struct i915_request *rq);
430 455
431 /* Call when the priority on a request has changed and it and its 456 /* Call when the priority on a request has changed and it and its
432 * dependencies may need rescheduling. Note the request itself may 457 * dependencies may need rescheduling. Note the request itself may
@@ -434,8 +459,7 @@ struct intel_engine_cs {
434 * 459 *
435 * Called under the struct_mutex. 460 * Called under the struct_mutex.
436 */ 461 */
437 void (*schedule)(struct drm_i915_gem_request *request, 462 void (*schedule)(struct i915_request *request, int priority);
438 int priority);
439 463
440 /* 464 /*
441 * Cancel all requests on the hardware, or queued for execution. 465 * Cancel all requests on the hardware, or queued for execution.
@@ -503,9 +527,9 @@ struct intel_engine_cs {
503 } mbox; 527 } mbox;
504 528
505 /* AKA wait() */ 529 /* AKA wait() */
506 int (*sync_to)(struct drm_i915_gem_request *req, 530 int (*sync_to)(struct i915_request *rq,
507 struct drm_i915_gem_request *signal); 531 struct i915_request *signal);
508 u32 *(*signal)(struct drm_i915_gem_request *req, u32 *cs); 532 u32 *(*signal)(struct i915_request *rq, u32 *cs);
509 } semaphore; 533 } semaphore;
510 534
511 struct intel_engine_execlists execlists; 535 struct intel_engine_execlists execlists;
@@ -726,14 +750,12 @@ void intel_engine_cleanup(struct intel_engine_cs *engine);
726 750
727void intel_legacy_submission_resume(struct drm_i915_private *dev_priv); 751void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
728 752
729int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req); 753int __must_check intel_ring_cacheline_align(struct i915_request *rq);
730 754
731int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes); 755int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes);
732u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req, 756u32 __must_check *intel_ring_begin(struct i915_request *rq, unsigned int n);
733 unsigned int n);
734 757
735static inline void 758static inline void intel_ring_advance(struct i915_request *rq, u32 *cs)
736intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
737{ 759{
738 /* Dummy function. 760 /* Dummy function.
739 * 761 *
@@ -743,22 +765,20 @@ intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
743 * reserved for the command packet (i.e. the value passed to 765 * reserved for the command packet (i.e. the value passed to
744 * intel_ring_begin()). 766 * intel_ring_begin()).
745 */ 767 */
746 GEM_BUG_ON((req->ring->vaddr + req->ring->emit) != cs); 768 GEM_BUG_ON((rq->ring->vaddr + rq->ring->emit) != cs);
747} 769}
748 770
749static inline u32 771static inline u32 intel_ring_wrap(const struct intel_ring *ring, u32 pos)
750intel_ring_wrap(const struct intel_ring *ring, u32 pos)
751{ 772{
752 return pos & (ring->size - 1); 773 return pos & (ring->size - 1);
753} 774}
754 775
755static inline u32 776static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr)
756intel_ring_offset(const struct drm_i915_gem_request *req, void *addr)
757{ 777{
758 /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */ 778 /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
759 u32 offset = addr - req->ring->vaddr; 779 u32 offset = addr - rq->ring->vaddr;
760 GEM_BUG_ON(offset > req->ring->size); 780 GEM_BUG_ON(offset > rq->ring->size);
761 return intel_ring_wrap(req->ring, offset); 781 return intel_ring_wrap(rq->ring, offset);
762} 782}
763 783
764static inline void 784static inline void
@@ -796,7 +816,7 @@ intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
796{ 816{
797 /* Whilst writes to the tail are strictly order, there is no 817 /* Whilst writes to the tail are strictly order, there is no
798 * serialisation between readers and the writers. The tail may be 818 * serialisation between readers and the writers. The tail may be
799 * read by i915_gem_request_retire() just as it is being updated 819 * read by i915_request_retire() just as it is being updated
800 * by execlists, as although the breadcrumb is complete, the context 820 * by execlists, as although the breadcrumb is complete, the context
801 * switch hasn't been seen. 821 * switch hasn't been seen.
802 */ 822 */
@@ -838,7 +858,7 @@ static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
838} 858}
839 859
840int init_workarounds_ring(struct intel_engine_cs *engine); 860int init_workarounds_ring(struct intel_engine_cs *engine);
841int intel_ring_workarounds_emit(struct drm_i915_gem_request *req); 861int intel_ring_workarounds_emit(struct i915_request *rq);
842 862
843void intel_engine_get_instdone(struct intel_engine_cs *engine, 863void intel_engine_get_instdone(struct intel_engine_cs *engine,
844 struct intel_instdone *instdone); 864 struct intel_instdone *instdone);
@@ -866,7 +886,7 @@ static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine)
866int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); 886int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
867 887
868static inline void intel_wait_init(struct intel_wait *wait, 888static inline void intel_wait_init(struct intel_wait *wait,
869 struct drm_i915_gem_request *rq) 889 struct i915_request *rq)
870{ 890{
871 wait->tsk = current; 891 wait->tsk = current;
872 wait->request = rq; 892 wait->request = rq;
@@ -892,9 +912,9 @@ intel_wait_update_seqno(struct intel_wait *wait, u32 seqno)
892 912
893static inline bool 913static inline bool
894intel_wait_update_request(struct intel_wait *wait, 914intel_wait_update_request(struct intel_wait *wait,
895 const struct drm_i915_gem_request *rq) 915 const struct i915_request *rq)
896{ 916{
897 return intel_wait_update_seqno(wait, i915_gem_request_global_seqno(rq)); 917 return intel_wait_update_seqno(wait, i915_request_global_seqno(rq));
898} 918}
899 919
900static inline bool 920static inline bool
@@ -905,9 +925,9 @@ intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno)
905 925
906static inline bool 926static inline bool
907intel_wait_check_request(const struct intel_wait *wait, 927intel_wait_check_request(const struct intel_wait *wait,
908 const struct drm_i915_gem_request *rq) 928 const struct i915_request *rq)
909{ 929{
910 return intel_wait_check_seqno(wait, i915_gem_request_global_seqno(rq)); 930 return intel_wait_check_seqno(wait, i915_request_global_seqno(rq));
911} 931}
912 932
913static inline bool intel_wait_complete(const struct intel_wait *wait) 933static inline bool intel_wait_complete(const struct intel_wait *wait)
@@ -919,9 +939,8 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine,
919 struct intel_wait *wait); 939 struct intel_wait *wait);
920void intel_engine_remove_wait(struct intel_engine_cs *engine, 940void intel_engine_remove_wait(struct intel_engine_cs *engine,
921 struct intel_wait *wait); 941 struct intel_wait *wait);
922void intel_engine_enable_signaling(struct drm_i915_gem_request *request, 942void intel_engine_enable_signaling(struct i915_request *request, bool wakeup);
923 bool wakeup); 943void intel_engine_cancel_signaling(struct i915_request *request);
924void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
925 944
926static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine) 945static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
927{ 946{
@@ -940,7 +959,6 @@ void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
940 959
941void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); 960void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
942void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); 961void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
943bool intel_breadcrumbs_busy(struct intel_engine_cs *engine);
944 962
945static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) 963static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
946{ 964{
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index b7924feb9f27..53ea564f971e 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -130,6 +130,8 @@ intel_display_power_domain_str(enum intel_display_power_domain domain)
130 return "AUX_D"; 130 return "AUX_D";
131 case POWER_DOMAIN_AUX_F: 131 case POWER_DOMAIN_AUX_F:
132 return "AUX_F"; 132 return "AUX_F";
133 case POWER_DOMAIN_AUX_IO_A:
134 return "AUX_IO_A";
133 case POWER_DOMAIN_GMBUS: 135 case POWER_DOMAIN_GMBUS:
134 return "GMBUS"; 136 return "GMBUS";
135 case POWER_DOMAIN_INIT: 137 case POWER_DOMAIN_INIT:
@@ -1853,6 +1855,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
1853 BIT_ULL(POWER_DOMAIN_INIT)) 1855 BIT_ULL(POWER_DOMAIN_INIT))
1854#define CNL_DISPLAY_AUX_A_POWER_DOMAINS ( \ 1856#define CNL_DISPLAY_AUX_A_POWER_DOMAINS ( \
1855 BIT_ULL(POWER_DOMAIN_AUX_A) | \ 1857 BIT_ULL(POWER_DOMAIN_AUX_A) | \
1858 BIT_ULL(POWER_DOMAIN_AUX_IO_A) | \
1856 BIT_ULL(POWER_DOMAIN_INIT)) 1859 BIT_ULL(POWER_DOMAIN_INIT))
1857#define CNL_DISPLAY_AUX_B_POWER_DOMAINS ( \ 1860#define CNL_DISPLAY_AUX_B_POWER_DOMAINS ( \
1858 BIT_ULL(POWER_DOMAIN_AUX_B) | \ 1861 BIT_ULL(POWER_DOMAIN_AUX_B) | \
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 0c14d1c04cbd..96e213ec202d 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1705,7 +1705,15 @@ static void intel_sdvo_enable_hotplug(struct intel_encoder *encoder)
1705 struct intel_sdvo *intel_sdvo = to_sdvo(encoder); 1705 struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
1706 1706
1707 intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG, 1707 intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG,
1708 &intel_sdvo->hotplug_active, 2); 1708 &intel_sdvo->hotplug_active, 2);
1709}
1710
1711static bool intel_sdvo_hotplug(struct intel_encoder *encoder,
1712 struct intel_connector *connector)
1713{
1714 intel_sdvo_enable_hotplug(encoder);
1715
1716 return intel_encoder_hotplug(encoder, connector);
1709} 1717}
1710 1718
1711static bool 1719static bool
@@ -2516,7 +2524,7 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
2516 * Some SDVO devices have one-shot hotplug interrupts. 2524 * Some SDVO devices have one-shot hotplug interrupts.
2517 * Ensure that they get re-enabled when an interrupt happens. 2525 * Ensure that they get re-enabled when an interrupt happens.
2518 */ 2526 */
2519 intel_encoder->hot_plug = intel_sdvo_enable_hotplug; 2527 intel_encoder->hotplug = intel_sdvo_hotplug;
2520 intel_sdvo_enable_hotplug(intel_encoder); 2528 intel_sdvo_enable_hotplug(intel_encoder);
2521 } else { 2529 } else {
2522 intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; 2530 intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 9f1bac6398fb..e5bf0d37bf43 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -361,7 +361,7 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
361 goto err_out; 361 goto err_out;
362 362
363 if (USES_HUC(dev_priv)) { 363 if (USES_HUC(dev_priv)) {
364 ret = intel_huc_init_hw(huc); 364 ret = intel_huc_fw_upload(huc);
365 if (ret) 365 if (ret)
366 goto err_out; 366 goto err_out;
367 } 367 }
@@ -445,3 +445,48 @@ void intel_uc_fini_hw(struct drm_i915_private *dev_priv)
445 if (USES_GUC_SUBMISSION(dev_priv)) 445 if (USES_GUC_SUBMISSION(dev_priv))
446 gen9_disable_guc_interrupts(dev_priv); 446 gen9_disable_guc_interrupts(dev_priv);
447} 447}
448
449int intel_uc_suspend(struct drm_i915_private *i915)
450{
451 struct intel_guc *guc = &i915->guc;
452 int err;
453
454 if (!USES_GUC(i915))
455 return 0;
456
457 if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
458 return 0;
459
460 err = intel_guc_suspend(guc);
461 if (err) {
462 DRM_DEBUG_DRIVER("Failed to suspend GuC, err=%d", err);
463 return err;
464 }
465
466 gen9_disable_guc_interrupts(i915);
467
468 return 0;
469}
470
471int intel_uc_resume(struct drm_i915_private *i915)
472{
473 struct intel_guc *guc = &i915->guc;
474 int err;
475
476 if (!USES_GUC(i915))
477 return 0;
478
479 if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
480 return 0;
481
482 if (i915_modparams.guc_log_level)
483 gen9_enable_guc_interrupts(i915);
484
485 err = intel_guc_resume(guc);
486 if (err) {
487 DRM_DEBUG_DRIVER("Failed to resume GuC, err=%d", err);
488 return err;
489 }
490
491 return 0;
492}
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index f2984e01e257..f76d51d1ce70 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -39,6 +39,8 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv);
39void intel_uc_fini_hw(struct drm_i915_private *dev_priv); 39void intel_uc_fini_hw(struct drm_i915_private *dev_priv);
40int intel_uc_init(struct drm_i915_private *dev_priv); 40int intel_uc_init(struct drm_i915_private *dev_priv);
41void intel_uc_fini(struct drm_i915_private *dev_priv); 41void intel_uc_fini(struct drm_i915_private *dev_priv);
42int intel_uc_suspend(struct drm_i915_private *dev_priv);
43int intel_uc_resume(struct drm_i915_private *dev_priv);
42 44
43static inline bool intel_uc_is_using_guc(void) 45static inline bool intel_uc_is_using_guc(void)
44{ 46{
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 5ae9a62712ca..4df7c2ef8576 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -37,6 +37,12 @@ static const char * const forcewake_domain_names[] = {
37 "render", 37 "render",
38 "blitter", 38 "blitter",
39 "media", 39 "media",
40 "vdbox0",
41 "vdbox1",
42 "vdbox2",
43 "vdbox3",
44 "vebox0",
45 "vebox1",
40}; 46};
41 47
42const char * 48const char *
@@ -774,6 +780,9 @@ void assert_forcewakes_active(struct drm_i915_private *dev_priv,
774/* We give fast paths for the really cool registers */ 780/* We give fast paths for the really cool registers */
775#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000) 781#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
776 782
783#define GEN11_NEEDS_FORCE_WAKE(reg) \
784 ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000))
785
777#define __gen6_reg_read_fw_domains(offset) \ 786#define __gen6_reg_read_fw_domains(offset) \
778({ \ 787({ \
779 enum forcewake_domains __fwd; \ 788 enum forcewake_domains __fwd; \
@@ -826,6 +835,14 @@ find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
826 if (!entry) 835 if (!entry)
827 return 0; 836 return 0;
828 837
838 /*
839 * The list of FW domains depends on the SKU in gen11+ so we
840 * can't determine it statically. We use FORCEWAKE_ALL and
841 * translate it here to the list of available domains.
842 */
843 if (entry->domains == FORCEWAKE_ALL)
844 return dev_priv->uncore.fw_domains;
845
829 WARN(entry->domains & ~dev_priv->uncore.fw_domains, 846 WARN(entry->domains & ~dev_priv->uncore.fw_domains,
830 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n", 847 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
831 entry->domains & ~dev_priv->uncore.fw_domains, offset); 848 entry->domains & ~dev_priv->uncore.fw_domains, offset);
@@ -860,6 +877,14 @@ static const struct intel_forcewake_range __vlv_fw_ranges[] = {
860 __fwd; \ 877 __fwd; \
861}) 878})
862 879
880#define __gen11_fwtable_reg_read_fw_domains(offset) \
881({ \
882 enum forcewake_domains __fwd = 0; \
883 if (GEN11_NEEDS_FORCE_WAKE((offset))) \
884 __fwd = find_fw_domain(dev_priv, offset); \
885 __fwd; \
886})
887
863/* *Must* be sorted by offset! See intel_shadow_table_check(). */ 888/* *Must* be sorted by offset! See intel_shadow_table_check(). */
864static const i915_reg_t gen8_shadowed_regs[] = { 889static const i915_reg_t gen8_shadowed_regs[] = {
865 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */ 890 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */
@@ -871,6 +896,20 @@ static const i915_reg_t gen8_shadowed_regs[] = {
871 /* TODO: Other registers are not yet used */ 896 /* TODO: Other registers are not yet used */
872}; 897};
873 898
899static const i915_reg_t gen11_shadowed_regs[] = {
900 RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */
901 GEN6_RPNSWREQ, /* 0xA008 */
902 GEN6_RC_VIDEO_FREQ, /* 0xA00C */
903 RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */
904 RING_TAIL(GEN11_BSD_RING_BASE), /* 0x1C0000 (base) */
905 RING_TAIL(GEN11_BSD2_RING_BASE), /* 0x1C4000 (base) */
906 RING_TAIL(GEN11_VEBOX_RING_BASE), /* 0x1C8000 (base) */
907 RING_TAIL(GEN11_BSD3_RING_BASE), /* 0x1D0000 (base) */
908 RING_TAIL(GEN11_BSD4_RING_BASE), /* 0x1D4000 (base) */
909 RING_TAIL(GEN11_VEBOX2_RING_BASE), /* 0x1D8000 (base) */
910 /* TODO: Other registers are not yet used */
911};
912
874static int mmio_reg_cmp(u32 key, const i915_reg_t *reg) 913static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
875{ 914{
876 u32 offset = i915_mmio_reg_offset(*reg); 915 u32 offset = i915_mmio_reg_offset(*reg);
@@ -883,14 +922,17 @@ static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
883 return 0; 922 return 0;
884} 923}
885 924
886static bool is_gen8_shadowed(u32 offset) 925#define __is_genX_shadowed(x) \
887{ 926static bool is_gen##x##_shadowed(u32 offset) \
888 const i915_reg_t *regs = gen8_shadowed_regs; 927{ \
889 928 const i915_reg_t *regs = gen##x##_shadowed_regs; \
890 return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs), 929 return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \
891 mmio_reg_cmp); 930 mmio_reg_cmp); \
892} 931}
893 932
933__is_genX_shadowed(8)
934__is_genX_shadowed(11)
935
894#define __gen8_reg_write_fw_domains(offset) \ 936#define __gen8_reg_write_fw_domains(offset) \
895({ \ 937({ \
896 enum forcewake_domains __fwd; \ 938 enum forcewake_domains __fwd; \
@@ -929,6 +971,14 @@ static const struct intel_forcewake_range __chv_fw_ranges[] = {
929 __fwd; \ 971 __fwd; \
930}) 972})
931 973
974#define __gen11_fwtable_reg_write_fw_domains(offset) \
975({ \
976 enum forcewake_domains __fwd = 0; \
977 if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \
978 __fwd = find_fw_domain(dev_priv, offset); \
979 __fwd; \
980})
981
932/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ 982/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
933static const struct intel_forcewake_range __gen9_fw_ranges[] = { 983static const struct intel_forcewake_range __gen9_fw_ranges[] = {
934 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), 984 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
@@ -965,6 +1015,40 @@ static const struct intel_forcewake_range __gen9_fw_ranges[] = {
965 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA), 1015 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
966}; 1016};
967 1017
1018/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
1019static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1020 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
1021 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1022 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1023 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
1024 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1025 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
1026 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1027 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER),
1028 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1029 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
1030 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1031 GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER),
1032 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1033 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
1034 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL),
1035 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
1036 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1037 GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER),
1038 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1039 GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER),
1040 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1041 GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER),
1042 GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1043 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1044 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1),
1045 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0),
1046 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER),
1047 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1048 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3),
1049 GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1)
1050};
1051
968static void 1052static void
969ilk_dummy_write(struct drm_i915_private *dev_priv) 1053ilk_dummy_write(struct drm_i915_private *dev_priv)
970{ 1054{
@@ -1095,7 +1179,12 @@ func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) {
1095} 1179}
1096#define __gen6_read(x) __gen_read(gen6, x) 1180#define __gen6_read(x) __gen_read(gen6, x)
1097#define __fwtable_read(x) __gen_read(fwtable, x) 1181#define __fwtable_read(x) __gen_read(fwtable, x)
1182#define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x)
1098 1183
1184__gen11_fwtable_read(8)
1185__gen11_fwtable_read(16)
1186__gen11_fwtable_read(32)
1187__gen11_fwtable_read(64)
1099__fwtable_read(8) 1188__fwtable_read(8)
1100__fwtable_read(16) 1189__fwtable_read(16)
1101__fwtable_read(32) 1190__fwtable_read(32)
@@ -1105,6 +1194,7 @@ __gen6_read(16)
1105__gen6_read(32) 1194__gen6_read(32)
1106__gen6_read(64) 1195__gen6_read(64)
1107 1196
1197#undef __gen11_fwtable_read
1108#undef __fwtable_read 1198#undef __fwtable_read
1109#undef __gen6_read 1199#undef __gen6_read
1110#undef GEN6_READ_FOOTER 1200#undef GEN6_READ_FOOTER
@@ -1181,7 +1271,11 @@ func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, boo
1181} 1271}
1182#define __gen8_write(x) __gen_write(gen8, x) 1272#define __gen8_write(x) __gen_write(gen8, x)
1183#define __fwtable_write(x) __gen_write(fwtable, x) 1273#define __fwtable_write(x) __gen_write(fwtable, x)
1274#define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x)
1184 1275
1276__gen11_fwtable_write(8)
1277__gen11_fwtable_write(16)
1278__gen11_fwtable_write(32)
1185__fwtable_write(8) 1279__fwtable_write(8)
1186__fwtable_write(16) 1280__fwtable_write(16)
1187__fwtable_write(32) 1281__fwtable_write(32)
@@ -1192,6 +1286,7 @@ __gen6_write(8)
1192__gen6_write(16) 1286__gen6_write(16)
1193__gen6_write(32) 1287__gen6_write(32)
1194 1288
1289#undef __gen11_fwtable_write
1195#undef __fwtable_write 1290#undef __fwtable_write
1196#undef __gen8_write 1291#undef __gen8_write
1197#undef __gen6_write 1292#undef __gen6_write
@@ -1240,6 +1335,13 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
1240 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER)); 1335 BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
1241 BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER)); 1336 BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
1242 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA)); 1337 BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
1338 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
1339 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
1340 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
1341 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
1342 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
1343 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
1344
1243 1345
1244 d->mask = BIT(domain_id); 1346 d->mask = BIT(domain_id);
1245 1347
@@ -1267,7 +1369,34 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1267 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL); 1369 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
1268 } 1370 }
1269 1371
1270 if (INTEL_GEN(dev_priv) >= 9) { 1372 if (INTEL_GEN(dev_priv) >= 11) {
1373 int i;
1374
1375 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1376 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1377 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1378 FORCEWAKE_RENDER_GEN9,
1379 FORCEWAKE_ACK_RENDER_GEN9);
1380 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
1381 FORCEWAKE_BLITTER_GEN9,
1382 FORCEWAKE_ACK_BLITTER_GEN9);
1383 for (i = 0; i < I915_MAX_VCS; i++) {
1384 if (!HAS_ENGINE(dev_priv, _VCS(i)))
1385 continue;
1386
1387 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
1388 FORCEWAKE_MEDIA_VDBOX_GEN11(i),
1389 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
1390 }
1391 for (i = 0; i < I915_MAX_VECS; i++) {
1392 if (!HAS_ENGINE(dev_priv, _VECS(i)))
1393 continue;
1394
1395 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
1396 FORCEWAKE_MEDIA_VEBOX_GEN11(i),
1397 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
1398 }
1399 } else if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) {
1271 dev_priv->uncore.funcs.force_wake_get = 1400 dev_priv->uncore.funcs.force_wake_get =
1272 fw_domains_get_with_fallback; 1401 fw_domains_get_with_fallback;
1273 dev_priv->uncore.funcs.force_wake_put = fw_domains_put; 1402 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
@@ -1422,10 +1551,14 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
1422 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8); 1551 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
1423 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6); 1552 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1424 } 1553 }
1425 } else { 1554 } else if (IS_GEN(dev_priv, 9, 10)) {
1426 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); 1555 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1427 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable); 1556 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1428 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable); 1557 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1558 } else {
1559 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges);
1560 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable);
1561 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable);
1429 } 1562 }
1430 1563
1431 iosf_mbi_register_pmic_bus_access_notifier( 1564 iosf_mbi_register_pmic_bus_access_notifier(
@@ -1994,7 +2127,9 @@ intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
1994 u32 offset = i915_mmio_reg_offset(reg); 2127 u32 offset = i915_mmio_reg_offset(reg);
1995 enum forcewake_domains fw_domains; 2128 enum forcewake_domains fw_domains;
1996 2129
1997 if (HAS_FWTABLE(dev_priv)) { 2130 if (INTEL_GEN(dev_priv) >= 11) {
2131 fw_domains = __gen11_fwtable_reg_read_fw_domains(offset);
2132 } else if (HAS_FWTABLE(dev_priv)) {
1998 fw_domains = __fwtable_reg_read_fw_domains(offset); 2133 fw_domains = __fwtable_reg_read_fw_domains(offset);
1999 } else if (INTEL_GEN(dev_priv) >= 6) { 2134 } else if (INTEL_GEN(dev_priv) >= 6) {
2000 fw_domains = __gen6_reg_read_fw_domains(offset); 2135 fw_domains = __gen6_reg_read_fw_domains(offset);
@@ -2015,7 +2150,9 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
2015 u32 offset = i915_mmio_reg_offset(reg); 2150 u32 offset = i915_mmio_reg_offset(reg);
2016 enum forcewake_domains fw_domains; 2151 enum forcewake_domains fw_domains;
2017 2152
2018 if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) { 2153 if (INTEL_GEN(dev_priv) >= 11) {
2154 fw_domains = __gen11_fwtable_reg_write_fw_domains(offset);
2155 } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
2019 fw_domains = __fwtable_reg_write_fw_domains(offset); 2156 fw_domains = __fwtable_reg_write_fw_domains(offset);
2020 } else if (IS_GEN8(dev_priv)) { 2157 } else if (IS_GEN8(dev_priv)) {
2021 fw_domains = __gen8_reg_write_fw_domains(offset); 2158 fw_domains = __gen8_reg_write_fw_domains(offset);
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
index 53ef77d0c97c..dfdf444e4bcc 100644
--- a/drivers/gpu/drm/i915/intel_uncore.h
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -37,17 +37,28 @@ enum forcewake_domain_id {
37 FW_DOMAIN_ID_RENDER = 0, 37 FW_DOMAIN_ID_RENDER = 0,
38 FW_DOMAIN_ID_BLITTER, 38 FW_DOMAIN_ID_BLITTER,
39 FW_DOMAIN_ID_MEDIA, 39 FW_DOMAIN_ID_MEDIA,
40 FW_DOMAIN_ID_MEDIA_VDBOX0,
41 FW_DOMAIN_ID_MEDIA_VDBOX1,
42 FW_DOMAIN_ID_MEDIA_VDBOX2,
43 FW_DOMAIN_ID_MEDIA_VDBOX3,
44 FW_DOMAIN_ID_MEDIA_VEBOX0,
45 FW_DOMAIN_ID_MEDIA_VEBOX1,
40 46
41 FW_DOMAIN_ID_COUNT 47 FW_DOMAIN_ID_COUNT
42}; 48};
43 49
44enum forcewake_domains { 50enum forcewake_domains {
45 FORCEWAKE_RENDER = BIT(FW_DOMAIN_ID_RENDER), 51 FORCEWAKE_RENDER = BIT(FW_DOMAIN_ID_RENDER),
46 FORCEWAKE_BLITTER = BIT(FW_DOMAIN_ID_BLITTER), 52 FORCEWAKE_BLITTER = BIT(FW_DOMAIN_ID_BLITTER),
47 FORCEWAKE_MEDIA = BIT(FW_DOMAIN_ID_MEDIA), 53 FORCEWAKE_MEDIA = BIT(FW_DOMAIN_ID_MEDIA),
48 FORCEWAKE_ALL = (FORCEWAKE_RENDER | 54 FORCEWAKE_MEDIA_VDBOX0 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX0),
49 FORCEWAKE_BLITTER | 55 FORCEWAKE_MEDIA_VDBOX1 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX1),
50 FORCEWAKE_MEDIA) 56 FORCEWAKE_MEDIA_VDBOX2 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX2),
57 FORCEWAKE_MEDIA_VDBOX3 = BIT(FW_DOMAIN_ID_MEDIA_VDBOX3),
58 FORCEWAKE_MEDIA_VEBOX0 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX0),
59 FORCEWAKE_MEDIA_VEBOX1 = BIT(FW_DOMAIN_ID_MEDIA_VEBOX1),
60
61 FORCEWAKE_ALL = BIT(FW_DOMAIN_ID_COUNT) - 1
51}; 62};
52 63
53struct intel_uncore_funcs { 64struct intel_uncore_funcs {
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index 52b1bd17bf46..05bbef363fff 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -964,7 +964,7 @@ static int gpu_write(struct i915_vma *vma,
964 u32 dword, 964 u32 dword,
965 u32 value) 965 u32 value)
966{ 966{
967 struct drm_i915_gem_request *rq; 967 struct i915_request *rq;
968 struct i915_vma *batch; 968 struct i915_vma *batch;
969 int flags = 0; 969 int flags = 0;
970 int err; 970 int err;
@@ -975,7 +975,7 @@ static int gpu_write(struct i915_vma *vma,
975 if (err) 975 if (err)
976 return err; 976 return err;
977 977
978 rq = i915_gem_request_alloc(engine, ctx); 978 rq = i915_request_alloc(engine, ctx);
979 if (IS_ERR(rq)) 979 if (IS_ERR(rq))
980 return PTR_ERR(rq); 980 return PTR_ERR(rq);
981 981
@@ -1003,7 +1003,7 @@ static int gpu_write(struct i915_vma *vma,
1003 reservation_object_unlock(vma->resv); 1003 reservation_object_unlock(vma->resv);
1004 1004
1005err_request: 1005err_request:
1006 __i915_add_request(rq, err == 0); 1006 __i915_request_add(rq, err == 0);
1007 1007
1008 return err; 1008 return err;
1009} 1009}
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index 7a0d1e17c1ad..340a98c0c804 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -178,7 +178,7 @@ static int gpu_set(struct drm_i915_gem_object *obj,
178 u32 v) 178 u32 v)
179{ 179{
180 struct drm_i915_private *i915 = to_i915(obj->base.dev); 180 struct drm_i915_private *i915 = to_i915(obj->base.dev);
181 struct drm_i915_gem_request *rq; 181 struct i915_request *rq;
182 struct i915_vma *vma; 182 struct i915_vma *vma;
183 u32 *cs; 183 u32 *cs;
184 int err; 184 int err;
@@ -191,7 +191,7 @@ static int gpu_set(struct drm_i915_gem_object *obj,
191 if (IS_ERR(vma)) 191 if (IS_ERR(vma))
192 return PTR_ERR(vma); 192 return PTR_ERR(vma);
193 193
194 rq = i915_gem_request_alloc(i915->engine[RCS], i915->kernel_context); 194 rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
195 if (IS_ERR(rq)) { 195 if (IS_ERR(rq)) {
196 i915_vma_unpin(vma); 196 i915_vma_unpin(vma);
197 return PTR_ERR(rq); 197 return PTR_ERR(rq);
@@ -199,7 +199,7 @@ static int gpu_set(struct drm_i915_gem_object *obj,
199 199
200 cs = intel_ring_begin(rq, 4); 200 cs = intel_ring_begin(rq, 4);
201 if (IS_ERR(cs)) { 201 if (IS_ERR(cs)) {
202 __i915_add_request(rq, false); 202 __i915_request_add(rq, false);
203 i915_vma_unpin(vma); 203 i915_vma_unpin(vma);
204 return PTR_ERR(cs); 204 return PTR_ERR(cs);
205 } 205 }
@@ -229,7 +229,7 @@ static int gpu_set(struct drm_i915_gem_object *obj,
229 reservation_object_add_excl_fence(obj->resv, &rq->fence); 229 reservation_object_add_excl_fence(obj->resv, &rq->fence);
230 reservation_object_unlock(obj->resv); 230 reservation_object_unlock(obj->resv);
231 231
232 __i915_add_request(rq, true); 232 __i915_request_add(rq, true);
233 233
234 return 0; 234 return 0;
235} 235}
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 6da2a2f29c54..7ecaed50d0b9 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -114,7 +114,7 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
114 struct drm_i915_private *i915 = to_i915(obj->base.dev); 114 struct drm_i915_private *i915 = to_i915(obj->base.dev);
115 struct i915_address_space *vm = 115 struct i915_address_space *vm =
116 ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base; 116 ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
117 struct drm_i915_gem_request *rq; 117 struct i915_request *rq;
118 struct i915_vma *vma; 118 struct i915_vma *vma;
119 struct i915_vma *batch; 119 struct i915_vma *batch;
120 unsigned int flags; 120 unsigned int flags;
@@ -152,7 +152,7 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
152 goto err_vma; 152 goto err_vma;
153 } 153 }
154 154
155 rq = i915_gem_request_alloc(engine, ctx); 155 rq = i915_request_alloc(engine, ctx);
156 if (IS_ERR(rq)) { 156 if (IS_ERR(rq)) {
157 err = PTR_ERR(rq); 157 err = PTR_ERR(rq);
158 goto err_batch; 158 goto err_batch;
@@ -180,12 +180,12 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
180 reservation_object_add_excl_fence(obj->resv, &rq->fence); 180 reservation_object_add_excl_fence(obj->resv, &rq->fence);
181 reservation_object_unlock(obj->resv); 181 reservation_object_unlock(obj->resv);
182 182
183 __i915_add_request(rq, true); 183 __i915_request_add(rq, true);
184 184
185 return 0; 185 return 0;
186 186
187err_request: 187err_request:
188 __i915_add_request(rq, false); 188 __i915_request_add(rq, false);
189err_batch: 189err_batch:
190 i915_vma_unpin(batch); 190 i915_vma_unpin(batch);
191err_vma: 191err_vma:
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index e1ddad635d73..ab9d7bee0aae 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -407,7 +407,7 @@ static int igt_evict_contexts(void *arg)
407 mutex_lock(&i915->drm.struct_mutex); 407 mutex_lock(&i915->drm.struct_mutex);
408 onstack_fence_init(&fence); 408 onstack_fence_init(&fence);
409 do { 409 do {
410 struct drm_i915_gem_request *rq; 410 struct i915_request *rq;
411 struct i915_gem_context *ctx; 411 struct i915_gem_context *ctx;
412 412
413 ctx = live_context(i915, file); 413 ctx = live_context(i915, file);
@@ -416,7 +416,7 @@ static int igt_evict_contexts(void *arg)
416 416
417 /* We will need some GGTT space for the rq's context */ 417 /* We will need some GGTT space for the rq's context */
418 igt_evict_ctl.fail_if_busy = true; 418 igt_evict_ctl.fail_if_busy = true;
419 rq = i915_gem_request_alloc(engine, ctx); 419 rq = i915_request_alloc(engine, ctx);
420 igt_evict_ctl.fail_if_busy = false; 420 igt_evict_ctl.fail_if_busy = false;
421 421
422 if (IS_ERR(rq)) { 422 if (IS_ERR(rq)) {
@@ -437,7 +437,7 @@ static int igt_evict_contexts(void *arg)
437 if (err < 0) 437 if (err < 0)
438 break; 438 break;
439 439
440 i915_add_request(rq); 440 i915_request_add(rq);
441 count++; 441 count++;
442 err = 0; 442 err = 0;
443 } while(1); 443 } while(1);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 3c64815e910b..fbdb2419d418 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -436,7 +436,7 @@ out:
436static int make_obj_busy(struct drm_i915_gem_object *obj) 436static int make_obj_busy(struct drm_i915_gem_object *obj)
437{ 437{
438 struct drm_i915_private *i915 = to_i915(obj->base.dev); 438 struct drm_i915_private *i915 = to_i915(obj->base.dev);
439 struct drm_i915_gem_request *rq; 439 struct i915_request *rq;
440 struct i915_vma *vma; 440 struct i915_vma *vma;
441 int err; 441 int err;
442 442
@@ -448,14 +448,14 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
448 if (err) 448 if (err)
449 return err; 449 return err;
450 450
451 rq = i915_gem_request_alloc(i915->engine[RCS], i915->kernel_context); 451 rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
452 if (IS_ERR(rq)) { 452 if (IS_ERR(rq)) {
453 i915_vma_unpin(vma); 453 i915_vma_unpin(vma);
454 return PTR_ERR(rq); 454 return PTR_ERR(rq);
455 } 455 }
456 456
457 i915_vma_move_to_active(vma, rq, 0); 457 i915_vma_move_to_active(vma, rq, 0);
458 i915_add_request(rq); 458 i915_request_add(rq);
459 459
460 i915_gem_object_set_active_reference(obj); 460 i915_gem_object_set_active_reference(obj);
461 i915_vma_unpin(vma); 461 i915_vma_unpin(vma);
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index 088f45bc6199..9c76f0305b6a 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -11,7 +11,7 @@
11 */ 11 */
12selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */ 12selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */
13selftest(uncore, intel_uncore_live_selftests) 13selftest(uncore, intel_uncore_live_selftests)
14selftest(requests, i915_gem_request_live_selftests) 14selftest(requests, i915_request_live_selftests)
15selftest(objects, i915_gem_object_live_selftests) 15selftest(objects, i915_gem_object_live_selftests)
16selftest(dmabuf, i915_gem_dmabuf_live_selftests) 16selftest(dmabuf, i915_gem_dmabuf_live_selftests)
17selftest(coherency, i915_gem_coherency_live_selftests) 17selftest(coherency, i915_gem_coherency_live_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index 19c6fce837df..9a48aa441743 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -16,7 +16,7 @@ selftest(syncmap, i915_syncmap_mock_selftests)
16selftest(uncore, intel_uncore_mock_selftests) 16selftest(uncore, intel_uncore_mock_selftests)
17selftest(breadcrumbs, intel_breadcrumbs_mock_selftests) 17selftest(breadcrumbs, intel_breadcrumbs_mock_selftests)
18selftest(timelines, i915_gem_timeline_mock_selftests) 18selftest(timelines, i915_gem_timeline_mock_selftests)
19selftest(requests, i915_gem_request_mock_selftests) 19selftest(requests, i915_request_mock_selftests)
20selftest(objects, i915_gem_object_mock_selftests) 20selftest(objects, i915_gem_object_mock_selftests)
21selftest(dmabuf, i915_gem_dmabuf_mock_selftests) 21selftest(dmabuf, i915_gem_dmabuf_mock_selftests)
22selftest(vma, i915_vma_mock_selftests) 22selftest(vma, i915_vma_mock_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 647bf2bbd799..94bc2e1898a4 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -32,7 +32,7 @@
32static int igt_add_request(void *arg) 32static int igt_add_request(void *arg)
33{ 33{
34 struct drm_i915_private *i915 = arg; 34 struct drm_i915_private *i915 = arg;
35 struct drm_i915_gem_request *request; 35 struct i915_request *request;
36 int err = -ENOMEM; 36 int err = -ENOMEM;
37 37
38 /* Basic preliminary test to create a request and let it loose! */ 38 /* Basic preliminary test to create a request and let it loose! */
@@ -44,7 +44,7 @@ static int igt_add_request(void *arg)
44 if (!request) 44 if (!request)
45 goto out_unlock; 45 goto out_unlock;
46 46
47 i915_add_request(request); 47 i915_request_add(request);
48 48
49 err = 0; 49 err = 0;
50out_unlock: 50out_unlock:
@@ -56,7 +56,7 @@ static int igt_wait_request(void *arg)
56{ 56{
57 const long T = HZ / 4; 57 const long T = HZ / 4;
58 struct drm_i915_private *i915 = arg; 58 struct drm_i915_private *i915 = arg;
59 struct drm_i915_gem_request *request; 59 struct i915_request *request;
60 int err = -EINVAL; 60 int err = -EINVAL;
61 61
62 /* Submit a request, then wait upon it */ 62 /* Submit a request, then wait upon it */
@@ -68,49 +68,49 @@ static int igt_wait_request(void *arg)
68 goto out_unlock; 68 goto out_unlock;
69 } 69 }
70 70
71 if (i915_wait_request(request, I915_WAIT_LOCKED, 0) != -ETIME) { 71 if (i915_request_wait(request, I915_WAIT_LOCKED, 0) != -ETIME) {
72 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n"); 72 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n");
73 goto out_unlock; 73 goto out_unlock;
74 } 74 }
75 75
76 if (i915_wait_request(request, I915_WAIT_LOCKED, T) != -ETIME) { 76 if (i915_request_wait(request, I915_WAIT_LOCKED, T) != -ETIME) {
77 pr_err("request wait succeeded (expected timeout before submit!)\n"); 77 pr_err("request wait succeeded (expected timeout before submit!)\n");
78 goto out_unlock; 78 goto out_unlock;
79 } 79 }
80 80
81 if (i915_gem_request_completed(request)) { 81 if (i915_request_completed(request)) {
82 pr_err("request completed before submit!!\n"); 82 pr_err("request completed before submit!!\n");
83 goto out_unlock; 83 goto out_unlock;
84 } 84 }
85 85
86 i915_add_request(request); 86 i915_request_add(request);
87 87
88 if (i915_wait_request(request, I915_WAIT_LOCKED, 0) != -ETIME) { 88 if (i915_request_wait(request, I915_WAIT_LOCKED, 0) != -ETIME) {
89 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n"); 89 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n");
90 goto out_unlock; 90 goto out_unlock;
91 } 91 }
92 92
93 if (i915_gem_request_completed(request)) { 93 if (i915_request_completed(request)) {
94 pr_err("request completed immediately!\n"); 94 pr_err("request completed immediately!\n");
95 goto out_unlock; 95 goto out_unlock;
96 } 96 }
97 97
98 if (i915_wait_request(request, I915_WAIT_LOCKED, T / 2) != -ETIME) { 98 if (i915_request_wait(request, I915_WAIT_LOCKED, T / 2) != -ETIME) {
99 pr_err("request wait succeeded (expected timeout!)\n"); 99 pr_err("request wait succeeded (expected timeout!)\n");
100 goto out_unlock; 100 goto out_unlock;
101 } 101 }
102 102
103 if (i915_wait_request(request, I915_WAIT_LOCKED, T) == -ETIME) { 103 if (i915_request_wait(request, I915_WAIT_LOCKED, T) == -ETIME) {
104 pr_err("request wait timed out!\n"); 104 pr_err("request wait timed out!\n");
105 goto out_unlock; 105 goto out_unlock;
106 } 106 }
107 107
108 if (!i915_gem_request_completed(request)) { 108 if (!i915_request_completed(request)) {
109 pr_err("request not complete after waiting!\n"); 109 pr_err("request not complete after waiting!\n");
110 goto out_unlock; 110 goto out_unlock;
111 } 111 }
112 112
113 if (i915_wait_request(request, I915_WAIT_LOCKED, T) == -ETIME) { 113 if (i915_request_wait(request, I915_WAIT_LOCKED, T) == -ETIME) {
114 pr_err("request wait timed out when already complete!\n"); 114 pr_err("request wait timed out when already complete!\n");
115 goto out_unlock; 115 goto out_unlock;
116 } 116 }
@@ -126,7 +126,7 @@ static int igt_fence_wait(void *arg)
126{ 126{
127 const long T = HZ / 4; 127 const long T = HZ / 4;
128 struct drm_i915_private *i915 = arg; 128 struct drm_i915_private *i915 = arg;
129 struct drm_i915_gem_request *request; 129 struct i915_request *request;
130 int err = -EINVAL; 130 int err = -EINVAL;
131 131
132 /* Submit a request, treat it as a fence and wait upon it */ 132 /* Submit a request, treat it as a fence and wait upon it */
@@ -145,7 +145,7 @@ static int igt_fence_wait(void *arg)
145 } 145 }
146 146
147 mutex_lock(&i915->drm.struct_mutex); 147 mutex_lock(&i915->drm.struct_mutex);
148 i915_add_request(request); 148 i915_request_add(request);
149 mutex_unlock(&i915->drm.struct_mutex); 149 mutex_unlock(&i915->drm.struct_mutex);
150 150
151 if (dma_fence_is_signaled(&request->fence)) { 151 if (dma_fence_is_signaled(&request->fence)) {
@@ -185,7 +185,7 @@ out_locked:
185static int igt_request_rewind(void *arg) 185static int igt_request_rewind(void *arg)
186{ 186{
187 struct drm_i915_private *i915 = arg; 187 struct drm_i915_private *i915 = arg;
188 struct drm_i915_gem_request *request, *vip; 188 struct i915_request *request, *vip;
189 struct i915_gem_context *ctx[2]; 189 struct i915_gem_context *ctx[2];
190 int err = -EINVAL; 190 int err = -EINVAL;
191 191
@@ -197,8 +197,8 @@ static int igt_request_rewind(void *arg)
197 goto err_context_0; 197 goto err_context_0;
198 } 198 }
199 199
200 i915_gem_request_get(request); 200 i915_request_get(request);
201 i915_add_request(request); 201 i915_request_add(request);
202 202
203 ctx[1] = mock_context(i915, "B"); 203 ctx[1] = mock_context(i915, "B");
204 vip = mock_request(i915->engine[RCS], ctx[1], 0); 204 vip = mock_request(i915->engine[RCS], ctx[1], 0);
@@ -210,35 +210,35 @@ static int igt_request_rewind(void *arg)
210 /* Simulate preemption by manual reordering */ 210 /* Simulate preemption by manual reordering */
211 if (!mock_cancel_request(request)) { 211 if (!mock_cancel_request(request)) {
212 pr_err("failed to cancel request (already executed)!\n"); 212 pr_err("failed to cancel request (already executed)!\n");
213 i915_add_request(vip); 213 i915_request_add(vip);
214 goto err_context_1; 214 goto err_context_1;
215 } 215 }
216 i915_gem_request_get(vip); 216 i915_request_get(vip);
217 i915_add_request(vip); 217 i915_request_add(vip);
218 rcu_read_lock(); 218 rcu_read_lock();
219 request->engine->submit_request(request); 219 request->engine->submit_request(request);
220 rcu_read_unlock(); 220 rcu_read_unlock();
221 221
222 mutex_unlock(&i915->drm.struct_mutex); 222 mutex_unlock(&i915->drm.struct_mutex);
223 223
224 if (i915_wait_request(vip, 0, HZ) == -ETIME) { 224 if (i915_request_wait(vip, 0, HZ) == -ETIME) {
225 pr_err("timed out waiting for high priority request, vip.seqno=%d, current seqno=%d\n", 225 pr_err("timed out waiting for high priority request, vip.seqno=%d, current seqno=%d\n",
226 vip->global_seqno, intel_engine_get_seqno(i915->engine[RCS])); 226 vip->global_seqno, intel_engine_get_seqno(i915->engine[RCS]));
227 goto err; 227 goto err;
228 } 228 }
229 229
230 if (i915_gem_request_completed(request)) { 230 if (i915_request_completed(request)) {
231 pr_err("low priority request already completed\n"); 231 pr_err("low priority request already completed\n");
232 goto err; 232 goto err;
233 } 233 }
234 234
235 err = 0; 235 err = 0;
236err: 236err:
237 i915_gem_request_put(vip); 237 i915_request_put(vip);
238 mutex_lock(&i915->drm.struct_mutex); 238 mutex_lock(&i915->drm.struct_mutex);
239err_context_1: 239err_context_1:
240 mock_context_close(ctx[1]); 240 mock_context_close(ctx[1]);
241 i915_gem_request_put(request); 241 i915_request_put(request);
242err_context_0: 242err_context_0:
243 mock_context_close(ctx[0]); 243 mock_context_close(ctx[0]);
244 mock_device_flush(i915); 244 mock_device_flush(i915);
@@ -246,7 +246,7 @@ err_context_0:
246 return err; 246 return err;
247} 247}
248 248
249int i915_gem_request_mock_selftests(void) 249int i915_request_mock_selftests(void)
250{ 250{
251 static const struct i915_subtest tests[] = { 251 static const struct i915_subtest tests[] = {
252 SUBTEST(igt_add_request), 252 SUBTEST(igt_add_request),
@@ -303,7 +303,7 @@ static int end_live_test(struct live_test *t)
303{ 303{
304 struct drm_i915_private *i915 = t->i915; 304 struct drm_i915_private *i915 = t->i915;
305 305
306 i915_gem_retire_requests(i915); 306 i915_retire_requests(i915);
307 307
308 if (wait_for(intel_engines_are_idle(i915), 10)) { 308 if (wait_for(intel_engines_are_idle(i915), 10)) {
309 pr_err("%s(%s): GPU not idle\n", t->func, t->name); 309 pr_err("%s(%s): GPU not idle\n", t->func, t->name);
@@ -343,7 +343,7 @@ static int live_nop_request(void *arg)
343 343
344 for_each_engine(engine, i915, id) { 344 for_each_engine(engine, i915, id) {
345 IGT_TIMEOUT(end_time); 345 IGT_TIMEOUT(end_time);
346 struct drm_i915_gem_request *request; 346 struct i915_request *request;
347 unsigned long n, prime; 347 unsigned long n, prime;
348 ktime_t times[2] = {}; 348 ktime_t times[2] = {};
349 349
@@ -355,8 +355,8 @@ static int live_nop_request(void *arg)
355 times[1] = ktime_get_raw(); 355 times[1] = ktime_get_raw();
356 356
357 for (n = 0; n < prime; n++) { 357 for (n = 0; n < prime; n++) {
358 request = i915_gem_request_alloc(engine, 358 request = i915_request_alloc(engine,
359 i915->kernel_context); 359 i915->kernel_context);
360 if (IS_ERR(request)) { 360 if (IS_ERR(request)) {
361 err = PTR_ERR(request); 361 err = PTR_ERR(request);
362 goto out_unlock; 362 goto out_unlock;
@@ -375,9 +375,9 @@ static int live_nop_request(void *arg)
375 * for latency. 375 * for latency.
376 */ 376 */
377 377
378 i915_add_request(request); 378 i915_request_add(request);
379 } 379 }
380 i915_wait_request(request, 380 i915_request_wait(request,
381 I915_WAIT_LOCKED, 381 I915_WAIT_LOCKED,
382 MAX_SCHEDULE_TIMEOUT); 382 MAX_SCHEDULE_TIMEOUT);
383 383
@@ -447,15 +447,14 @@ err:
447 return ERR_PTR(err); 447 return ERR_PTR(err);
448} 448}
449 449
450static struct drm_i915_gem_request * 450static struct i915_request *
451empty_request(struct intel_engine_cs *engine, 451empty_request(struct intel_engine_cs *engine,
452 struct i915_vma *batch) 452 struct i915_vma *batch)
453{ 453{
454 struct drm_i915_gem_request *request; 454 struct i915_request *request;
455 int err; 455 int err;
456 456
457 request = i915_gem_request_alloc(engine, 457 request = i915_request_alloc(engine, engine->i915->kernel_context);
458 engine->i915->kernel_context);
459 if (IS_ERR(request)) 458 if (IS_ERR(request))
460 return request; 459 return request;
461 460
@@ -467,7 +466,7 @@ empty_request(struct intel_engine_cs *engine,
467 goto out_request; 466 goto out_request;
468 467
469out_request: 468out_request:
470 __i915_add_request(request, err == 0); 469 __i915_request_add(request, err == 0);
471 return err ? ERR_PTR(err) : request; 470 return err ? ERR_PTR(err) : request;
472} 471}
473 472
@@ -495,7 +494,7 @@ static int live_empty_request(void *arg)
495 494
496 for_each_engine(engine, i915, id) { 495 for_each_engine(engine, i915, id) {
497 IGT_TIMEOUT(end_time); 496 IGT_TIMEOUT(end_time);
498 struct drm_i915_gem_request *request; 497 struct i915_request *request;
499 unsigned long n, prime; 498 unsigned long n, prime;
500 ktime_t times[2] = {}; 499 ktime_t times[2] = {};
501 500
@@ -509,7 +508,7 @@ static int live_empty_request(void *arg)
509 err = PTR_ERR(request); 508 err = PTR_ERR(request);
510 goto out_batch; 509 goto out_batch;
511 } 510 }
512 i915_wait_request(request, 511 i915_request_wait(request,
513 I915_WAIT_LOCKED, 512 I915_WAIT_LOCKED,
514 MAX_SCHEDULE_TIMEOUT); 513 MAX_SCHEDULE_TIMEOUT);
515 514
@@ -523,7 +522,7 @@ static int live_empty_request(void *arg)
523 goto out_batch; 522 goto out_batch;
524 } 523 }
525 } 524 }
526 i915_wait_request(request, 525 i915_request_wait(request,
527 I915_WAIT_LOCKED, 526 I915_WAIT_LOCKED,
528 MAX_SCHEDULE_TIMEOUT); 527 MAX_SCHEDULE_TIMEOUT);
529 528
@@ -633,7 +632,7 @@ static int live_all_engines(void *arg)
633{ 632{
634 struct drm_i915_private *i915 = arg; 633 struct drm_i915_private *i915 = arg;
635 struct intel_engine_cs *engine; 634 struct intel_engine_cs *engine;
636 struct drm_i915_gem_request *request[I915_NUM_ENGINES]; 635 struct i915_request *request[I915_NUM_ENGINES];
637 struct i915_vma *batch; 636 struct i915_vma *batch;
638 struct live_test t; 637 struct live_test t;
639 unsigned int id; 638 unsigned int id;
@@ -658,8 +657,7 @@ static int live_all_engines(void *arg)
658 } 657 }
659 658
660 for_each_engine(engine, i915, id) { 659 for_each_engine(engine, i915, id) {
661 request[id] = i915_gem_request_alloc(engine, 660 request[id] = i915_request_alloc(engine, i915->kernel_context);
662 i915->kernel_context);
663 if (IS_ERR(request[id])) { 661 if (IS_ERR(request[id])) {
664 err = PTR_ERR(request[id]); 662 err = PTR_ERR(request[id]);
665 pr_err("%s: Request allocation failed with err=%d\n", 663 pr_err("%s: Request allocation failed with err=%d\n",
@@ -680,12 +678,12 @@ static int live_all_engines(void *arg)
680 } 678 }
681 679
682 i915_vma_move_to_active(batch, request[id], 0); 680 i915_vma_move_to_active(batch, request[id], 0);
683 i915_gem_request_get(request[id]); 681 i915_request_get(request[id]);
684 i915_add_request(request[id]); 682 i915_request_add(request[id]);
685 } 683 }
686 684
687 for_each_engine(engine, i915, id) { 685 for_each_engine(engine, i915, id) {
688 if (i915_gem_request_completed(request[id])) { 686 if (i915_request_completed(request[id])) {
689 pr_err("%s(%s): request completed too early!\n", 687 pr_err("%s(%s): request completed too early!\n",
690 __func__, engine->name); 688 __func__, engine->name);
691 err = -EINVAL; 689 err = -EINVAL;
@@ -702,7 +700,7 @@ static int live_all_engines(void *arg)
702 for_each_engine(engine, i915, id) { 700 for_each_engine(engine, i915, id) {
703 long timeout; 701 long timeout;
704 702
705 timeout = i915_wait_request(request[id], 703 timeout = i915_request_wait(request[id],
706 I915_WAIT_LOCKED, 704 I915_WAIT_LOCKED,
707 MAX_SCHEDULE_TIMEOUT); 705 MAX_SCHEDULE_TIMEOUT);
708 if (timeout < 0) { 706 if (timeout < 0) {
@@ -712,8 +710,8 @@ static int live_all_engines(void *arg)
712 goto out_request; 710 goto out_request;
713 } 711 }
714 712
715 GEM_BUG_ON(!i915_gem_request_completed(request[id])); 713 GEM_BUG_ON(!i915_request_completed(request[id]));
716 i915_gem_request_put(request[id]); 714 i915_request_put(request[id]);
717 request[id] = NULL; 715 request[id] = NULL;
718 } 716 }
719 717
@@ -722,7 +720,7 @@ static int live_all_engines(void *arg)
722out_request: 720out_request:
723 for_each_engine(engine, i915, id) 721 for_each_engine(engine, i915, id)
724 if (request[id]) 722 if (request[id])
725 i915_gem_request_put(request[id]); 723 i915_request_put(request[id]);
726 i915_vma_unpin(batch); 724 i915_vma_unpin(batch);
727 i915_vma_put(batch); 725 i915_vma_put(batch);
728out_unlock: 726out_unlock:
@@ -733,8 +731,8 @@ out_unlock:
733static int live_sequential_engines(void *arg) 731static int live_sequential_engines(void *arg)
734{ 732{
735 struct drm_i915_private *i915 = arg; 733 struct drm_i915_private *i915 = arg;
736 struct drm_i915_gem_request *request[I915_NUM_ENGINES] = {}; 734 struct i915_request *request[I915_NUM_ENGINES] = {};
737 struct drm_i915_gem_request *prev = NULL; 735 struct i915_request *prev = NULL;
738 struct intel_engine_cs *engine; 736 struct intel_engine_cs *engine;
739 struct live_test t; 737 struct live_test t;
740 unsigned int id; 738 unsigned int id;
@@ -763,8 +761,7 @@ static int live_sequential_engines(void *arg)
763 goto out_unlock; 761 goto out_unlock;
764 } 762 }
765 763
766 request[id] = i915_gem_request_alloc(engine, 764 request[id] = i915_request_alloc(engine, i915->kernel_context);
767 i915->kernel_context);
768 if (IS_ERR(request[id])) { 765 if (IS_ERR(request[id])) {
769 err = PTR_ERR(request[id]); 766 err = PTR_ERR(request[id]);
770 pr_err("%s: Request allocation failed for %s with err=%d\n", 767 pr_err("%s: Request allocation failed for %s with err=%d\n",
@@ -773,10 +770,10 @@ static int live_sequential_engines(void *arg)
773 } 770 }
774 771
775 if (prev) { 772 if (prev) {
776 err = i915_gem_request_await_dma_fence(request[id], 773 err = i915_request_await_dma_fence(request[id],
777 &prev->fence); 774 &prev->fence);
778 if (err) { 775 if (err) {
779 i915_add_request(request[id]); 776 i915_request_add(request[id]);
780 pr_err("%s: Request await failed for %s with err=%d\n", 777 pr_err("%s: Request await failed for %s with err=%d\n",
781 __func__, engine->name, err); 778 __func__, engine->name, err);
782 goto out_request; 779 goto out_request;
@@ -794,8 +791,8 @@ static int live_sequential_engines(void *arg)
794 i915_gem_object_set_active_reference(batch->obj); 791 i915_gem_object_set_active_reference(batch->obj);
795 i915_vma_get(batch); 792 i915_vma_get(batch);
796 793
797 i915_gem_request_get(request[id]); 794 i915_request_get(request[id]);
798 i915_add_request(request[id]); 795 i915_request_add(request[id]);
799 796
800 prev = request[id]; 797 prev = request[id];
801 } 798 }
@@ -803,7 +800,7 @@ static int live_sequential_engines(void *arg)
803 for_each_engine(engine, i915, id) { 800 for_each_engine(engine, i915, id) {
804 long timeout; 801 long timeout;
805 802
806 if (i915_gem_request_completed(request[id])) { 803 if (i915_request_completed(request[id])) {
807 pr_err("%s(%s): request completed too early!\n", 804 pr_err("%s(%s): request completed too early!\n",
808 __func__, engine->name); 805 __func__, engine->name);
809 err = -EINVAL; 806 err = -EINVAL;
@@ -817,7 +814,7 @@ static int live_sequential_engines(void *arg)
817 goto out_request; 814 goto out_request;
818 } 815 }
819 816
820 timeout = i915_wait_request(request[id], 817 timeout = i915_request_wait(request[id],
821 I915_WAIT_LOCKED, 818 I915_WAIT_LOCKED,
822 MAX_SCHEDULE_TIMEOUT); 819 MAX_SCHEDULE_TIMEOUT);
823 if (timeout < 0) { 820 if (timeout < 0) {
@@ -827,7 +824,7 @@ static int live_sequential_engines(void *arg)
827 goto out_request; 824 goto out_request;
828 } 825 }
829 826
830 GEM_BUG_ON(!i915_gem_request_completed(request[id])); 827 GEM_BUG_ON(!i915_request_completed(request[id]));
831 } 828 }
832 829
833 err = end_live_test(&t); 830 err = end_live_test(&t);
@@ -849,14 +846,14 @@ out_request:
849 } 846 }
850 847
851 i915_vma_put(request[id]->batch); 848 i915_vma_put(request[id]->batch);
852 i915_gem_request_put(request[id]); 849 i915_request_put(request[id]);
853 } 850 }
854out_unlock: 851out_unlock:
855 mutex_unlock(&i915->drm.struct_mutex); 852 mutex_unlock(&i915->drm.struct_mutex);
856 return err; 853 return err;
857} 854}
858 855
859int i915_gem_request_live_selftests(struct drm_i915_private *i915) 856int i915_request_live_selftests(struct drm_i915_private *i915)
860{ 857{
861 static const struct i915_subtest tests[] = { 858 static const struct i915_subtest tests[] = {
862 SUBTEST(live_nop_request), 859 SUBTEST(live_nop_request),
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index d1d2c2456f69..df7898c8edcb 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -92,13 +92,13 @@ err_ctx:
92} 92}
93 93
94static u64 hws_address(const struct i915_vma *hws, 94static u64 hws_address(const struct i915_vma *hws,
95 const struct drm_i915_gem_request *rq) 95 const struct i915_request *rq)
96{ 96{
97 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context); 97 return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context);
98} 98}
99 99
100static int emit_recurse_batch(struct hang *h, 100static int emit_recurse_batch(struct hang *h,
101 struct drm_i915_gem_request *rq) 101 struct i915_request *rq)
102{ 102{
103 struct drm_i915_private *i915 = h->i915; 103 struct drm_i915_private *i915 = h->i915;
104 struct i915_address_space *vm = rq->ctx->ppgtt ? &rq->ctx->ppgtt->base : &i915->ggtt.base; 104 struct i915_address_space *vm = rq->ctx->ppgtt ? &rq->ctx->ppgtt->base : &i915->ggtt.base;
@@ -204,10 +204,10 @@ unpin_vma:
204 return err; 204 return err;
205} 205}
206 206
207static struct drm_i915_gem_request * 207static struct i915_request *
208hang_create_request(struct hang *h, struct intel_engine_cs *engine) 208hang_create_request(struct hang *h, struct intel_engine_cs *engine)
209{ 209{
210 struct drm_i915_gem_request *rq; 210 struct i915_request *rq;
211 int err; 211 int err;
212 212
213 if (i915_gem_object_is_active(h->obj)) { 213 if (i915_gem_object_is_active(h->obj)) {
@@ -232,21 +232,20 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine)
232 h->batch = vaddr; 232 h->batch = vaddr;
233 } 233 }
234 234
235 rq = i915_gem_request_alloc(engine, h->ctx); 235 rq = i915_request_alloc(engine, h->ctx);
236 if (IS_ERR(rq)) 236 if (IS_ERR(rq))
237 return rq; 237 return rq;
238 238
239 err = emit_recurse_batch(h, rq); 239 err = emit_recurse_batch(h, rq);
240 if (err) { 240 if (err) {
241 __i915_add_request(rq, false); 241 __i915_request_add(rq, false);
242 return ERR_PTR(err); 242 return ERR_PTR(err);
243 } 243 }
244 244
245 return rq; 245 return rq;
246} 246}
247 247
248static u32 hws_seqno(const struct hang *h, 248static u32 hws_seqno(const struct hang *h, const struct i915_request *rq)
249 const struct drm_i915_gem_request *rq)
250{ 249{
251 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]); 250 return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]);
252} 251}
@@ -319,7 +318,7 @@ static void hang_fini(struct hang *h)
319 flush_test(h->i915, I915_WAIT_LOCKED); 318 flush_test(h->i915, I915_WAIT_LOCKED);
320} 319}
321 320
322static bool wait_for_hang(struct hang *h, struct drm_i915_gem_request *rq) 321static bool wait_for_hang(struct hang *h, struct i915_request *rq)
323{ 322{
324 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq), 323 return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
325 rq->fence.seqno), 324 rq->fence.seqno),
@@ -332,7 +331,7 @@ static bool wait_for_hang(struct hang *h, struct drm_i915_gem_request *rq)
332static int igt_hang_sanitycheck(void *arg) 331static int igt_hang_sanitycheck(void *arg)
333{ 332{
334 struct drm_i915_private *i915 = arg; 333 struct drm_i915_private *i915 = arg;
335 struct drm_i915_gem_request *rq; 334 struct i915_request *rq;
336 struct intel_engine_cs *engine; 335 struct intel_engine_cs *engine;
337 enum intel_engine_id id; 336 enum intel_engine_id id;
338 struct hang h; 337 struct hang h;
@@ -359,17 +358,17 @@ static int igt_hang_sanitycheck(void *arg)
359 goto fini; 358 goto fini;
360 } 359 }
361 360
362 i915_gem_request_get(rq); 361 i915_request_get(rq);
363 362
364 *h.batch = MI_BATCH_BUFFER_END; 363 *h.batch = MI_BATCH_BUFFER_END;
365 i915_gem_chipset_flush(i915); 364 i915_gem_chipset_flush(i915);
366 365
367 __i915_add_request(rq, true); 366 __i915_request_add(rq, true);
368 367
369 timeout = i915_wait_request(rq, 368 timeout = i915_request_wait(rq,
370 I915_WAIT_LOCKED, 369 I915_WAIT_LOCKED,
371 MAX_SCHEDULE_TIMEOUT); 370 MAX_SCHEDULE_TIMEOUT);
372 i915_gem_request_put(rq); 371 i915_request_put(rq);
373 372
374 if (timeout < 0) { 373 if (timeout < 0) {
375 err = timeout; 374 err = timeout;
@@ -485,7 +484,7 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
485 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); 484 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
486 do { 485 do {
487 if (active) { 486 if (active) {
488 struct drm_i915_gem_request *rq; 487 struct i915_request *rq;
489 488
490 mutex_lock(&i915->drm.struct_mutex); 489 mutex_lock(&i915->drm.struct_mutex);
491 rq = hang_create_request(&h, engine); 490 rq = hang_create_request(&h, engine);
@@ -495,8 +494,8 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
495 break; 494 break;
496 } 495 }
497 496
498 i915_gem_request_get(rq); 497 i915_request_get(rq);
499 __i915_add_request(rq, true); 498 __i915_request_add(rq, true);
500 mutex_unlock(&i915->drm.struct_mutex); 499 mutex_unlock(&i915->drm.struct_mutex);
501 500
502 if (!wait_for_hang(&h, rq)) { 501 if (!wait_for_hang(&h, rq)) {
@@ -507,12 +506,12 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
507 intel_engine_dump(engine, &p, 506 intel_engine_dump(engine, &p,
508 "%s\n", engine->name); 507 "%s\n", engine->name);
509 508
510 i915_gem_request_put(rq); 509 i915_request_put(rq);
511 err = -EIO; 510 err = -EIO;
512 break; 511 break;
513 } 512 }
514 513
515 i915_gem_request_put(rq); 514 i915_request_put(rq);
516 } 515 }
517 516
518 engine->hangcheck.stalled = true; 517 engine->hangcheck.stalled = true;
@@ -577,7 +576,7 @@ static int igt_reset_active_engine(void *arg)
577static int active_engine(void *data) 576static int active_engine(void *data)
578{ 577{
579 struct intel_engine_cs *engine = data; 578 struct intel_engine_cs *engine = data;
580 struct drm_i915_gem_request *rq[2] = {}; 579 struct i915_request *rq[2] = {};
581 struct i915_gem_context *ctx[2]; 580 struct i915_gem_context *ctx[2];
582 struct drm_file *file; 581 struct drm_file *file;
583 unsigned long count = 0; 582 unsigned long count = 0;
@@ -606,29 +605,29 @@ static int active_engine(void *data)
606 605
607 while (!kthread_should_stop()) { 606 while (!kthread_should_stop()) {
608 unsigned int idx = count++ & 1; 607 unsigned int idx = count++ & 1;
609 struct drm_i915_gem_request *old = rq[idx]; 608 struct i915_request *old = rq[idx];
610 struct drm_i915_gem_request *new; 609 struct i915_request *new;
611 610
612 mutex_lock(&engine->i915->drm.struct_mutex); 611 mutex_lock(&engine->i915->drm.struct_mutex);
613 new = i915_gem_request_alloc(engine, ctx[idx]); 612 new = i915_request_alloc(engine, ctx[idx]);
614 if (IS_ERR(new)) { 613 if (IS_ERR(new)) {
615 mutex_unlock(&engine->i915->drm.struct_mutex); 614 mutex_unlock(&engine->i915->drm.struct_mutex);
616 err = PTR_ERR(new); 615 err = PTR_ERR(new);
617 break; 616 break;
618 } 617 }
619 618
620 rq[idx] = i915_gem_request_get(new); 619 rq[idx] = i915_request_get(new);
621 i915_add_request(new); 620 i915_request_add(new);
622 mutex_unlock(&engine->i915->drm.struct_mutex); 621 mutex_unlock(&engine->i915->drm.struct_mutex);
623 622
624 if (old) { 623 if (old) {
625 i915_wait_request(old, 0, MAX_SCHEDULE_TIMEOUT); 624 i915_request_wait(old, 0, MAX_SCHEDULE_TIMEOUT);
626 i915_gem_request_put(old); 625 i915_request_put(old);
627 } 626 }
628 } 627 }
629 628
630 for (count = 0; count < ARRAY_SIZE(rq); count++) 629 for (count = 0; count < ARRAY_SIZE(rq); count++)
631 i915_gem_request_put(rq[count]); 630 i915_request_put(rq[count]);
632 631
633err_file: 632err_file:
634 mock_file_free(engine->i915, file); 633 mock_file_free(engine->i915, file);
@@ -692,7 +691,7 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
692 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); 691 set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
693 do { 692 do {
694 if (active) { 693 if (active) {
695 struct drm_i915_gem_request *rq; 694 struct i915_request *rq;
696 695
697 mutex_lock(&i915->drm.struct_mutex); 696 mutex_lock(&i915->drm.struct_mutex);
698 rq = hang_create_request(&h, engine); 697 rq = hang_create_request(&h, engine);
@@ -702,8 +701,8 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
702 break; 701 break;
703 } 702 }
704 703
705 i915_gem_request_get(rq); 704 i915_request_get(rq);
706 __i915_add_request(rq, true); 705 __i915_request_add(rq, true);
707 mutex_unlock(&i915->drm.struct_mutex); 706 mutex_unlock(&i915->drm.struct_mutex);
708 707
709 if (!wait_for_hang(&h, rq)) { 708 if (!wait_for_hang(&h, rq)) {
@@ -714,12 +713,12 @@ static int __igt_reset_engine_others(struct drm_i915_private *i915,
714 intel_engine_dump(engine, &p, 713 intel_engine_dump(engine, &p,
715 "%s\n", engine->name); 714 "%s\n", engine->name);
716 715
717 i915_gem_request_put(rq); 716 i915_request_put(rq);
718 err = -EIO; 717 err = -EIO;
719 break; 718 break;
720 } 719 }
721 720
722 i915_gem_request_put(rq); 721 i915_request_put(rq);
723 } 722 }
724 723
725 engine->hangcheck.stalled = true; 724 engine->hangcheck.stalled = true;
@@ -814,7 +813,7 @@ static int igt_reset_active_engine_others(void *arg)
814 return __igt_reset_engine_others(arg, true); 813 return __igt_reset_engine_others(arg, true);
815} 814}
816 815
817static u32 fake_hangcheck(struct drm_i915_gem_request *rq) 816static u32 fake_hangcheck(struct i915_request *rq)
818{ 817{
819 u32 reset_count; 818 u32 reset_count;
820 819
@@ -832,7 +831,7 @@ static u32 fake_hangcheck(struct drm_i915_gem_request *rq)
832static int igt_wait_reset(void *arg) 831static int igt_wait_reset(void *arg)
833{ 832{
834 struct drm_i915_private *i915 = arg; 833 struct drm_i915_private *i915 = arg;
835 struct drm_i915_gem_request *rq; 834 struct i915_request *rq;
836 unsigned int reset_count; 835 unsigned int reset_count;
837 struct hang h; 836 struct hang h;
838 long timeout; 837 long timeout;
@@ -856,8 +855,8 @@ static int igt_wait_reset(void *arg)
856 goto fini; 855 goto fini;
857 } 856 }
858 857
859 i915_gem_request_get(rq); 858 i915_request_get(rq);
860 __i915_add_request(rq, true); 859 __i915_request_add(rq, true);
861 860
862 if (!wait_for_hang(&h, rq)) { 861 if (!wait_for_hang(&h, rq)) {
863 struct drm_printer p = drm_info_printer(i915->drm.dev); 862 struct drm_printer p = drm_info_printer(i915->drm.dev);
@@ -875,9 +874,9 @@ static int igt_wait_reset(void *arg)
875 874
876 reset_count = fake_hangcheck(rq); 875 reset_count = fake_hangcheck(rq);
877 876
878 timeout = i915_wait_request(rq, I915_WAIT_LOCKED, 10); 877 timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10);
879 if (timeout < 0) { 878 if (timeout < 0) {
880 pr_err("i915_wait_request failed on a stuck request: err=%ld\n", 879 pr_err("i915_request_wait failed on a stuck request: err=%ld\n",
881 timeout); 880 timeout);
882 err = timeout; 881 err = timeout;
883 goto out_rq; 882 goto out_rq;
@@ -891,7 +890,7 @@ static int igt_wait_reset(void *arg)
891 } 890 }
892 891
893out_rq: 892out_rq:
894 i915_gem_request_put(rq); 893 i915_request_put(rq);
895fini: 894fini:
896 hang_fini(&h); 895 hang_fini(&h);
897unlock: 896unlock:
@@ -922,7 +921,7 @@ static int igt_reset_queue(void *arg)
922 goto unlock; 921 goto unlock;
923 922
924 for_each_engine(engine, i915, id) { 923 for_each_engine(engine, i915, id) {
925 struct drm_i915_gem_request *prev; 924 struct i915_request *prev;
926 IGT_TIMEOUT(end_time); 925 IGT_TIMEOUT(end_time);
927 unsigned int count; 926 unsigned int count;
928 927
@@ -935,12 +934,12 @@ static int igt_reset_queue(void *arg)
935 goto fini; 934 goto fini;
936 } 935 }
937 936
938 i915_gem_request_get(prev); 937 i915_request_get(prev);
939 __i915_add_request(prev, true); 938 __i915_request_add(prev, true);
940 939
941 count = 0; 940 count = 0;
942 do { 941 do {
943 struct drm_i915_gem_request *rq; 942 struct i915_request *rq;
944 unsigned int reset_count; 943 unsigned int reset_count;
945 944
946 rq = hang_create_request(&h, engine); 945 rq = hang_create_request(&h, engine);
@@ -949,8 +948,8 @@ static int igt_reset_queue(void *arg)
949 goto fini; 948 goto fini;
950 } 949 }
951 950
952 i915_gem_request_get(rq); 951 i915_request_get(rq);
953 __i915_add_request(rq, true); 952 __i915_request_add(rq, true);
954 953
955 if (!wait_for_hang(&h, prev)) { 954 if (!wait_for_hang(&h, prev)) {
956 struct drm_printer p = drm_info_printer(i915->drm.dev); 955 struct drm_printer p = drm_info_printer(i915->drm.dev);
@@ -960,8 +959,8 @@ static int igt_reset_queue(void *arg)
960 intel_engine_dump(prev->engine, &p, 959 intel_engine_dump(prev->engine, &p,
961 "%s\n", prev->engine->name); 960 "%s\n", prev->engine->name);
962 961
963 i915_gem_request_put(rq); 962 i915_request_put(rq);
964 i915_gem_request_put(prev); 963 i915_request_put(prev);
965 964
966 i915_reset(i915, 0); 965 i915_reset(i915, 0);
967 i915_gem_set_wedged(i915); 966 i915_gem_set_wedged(i915);
@@ -980,8 +979,8 @@ static int igt_reset_queue(void *arg)
980 if (prev->fence.error != -EIO) { 979 if (prev->fence.error != -EIO) {
981 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n", 980 pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
982 prev->fence.error); 981 prev->fence.error);
983 i915_gem_request_put(rq); 982 i915_request_put(rq);
984 i915_gem_request_put(prev); 983 i915_request_put(prev);
985 err = -EINVAL; 984 err = -EINVAL;
986 goto fini; 985 goto fini;
987 } 986 }
@@ -989,21 +988,21 @@ static int igt_reset_queue(void *arg)
989 if (rq->fence.error) { 988 if (rq->fence.error) {
990 pr_err("Fence error status not zero [%d] after unrelated reset\n", 989 pr_err("Fence error status not zero [%d] after unrelated reset\n",
991 rq->fence.error); 990 rq->fence.error);
992 i915_gem_request_put(rq); 991 i915_request_put(rq);
993 i915_gem_request_put(prev); 992 i915_request_put(prev);
994 err = -EINVAL; 993 err = -EINVAL;
995 goto fini; 994 goto fini;
996 } 995 }
997 996
998 if (i915_reset_count(&i915->gpu_error) == reset_count) { 997 if (i915_reset_count(&i915->gpu_error) == reset_count) {
999 pr_err("No GPU reset recorded!\n"); 998 pr_err("No GPU reset recorded!\n");
1000 i915_gem_request_put(rq); 999 i915_request_put(rq);
1001 i915_gem_request_put(prev); 1000 i915_request_put(prev);
1002 err = -EINVAL; 1001 err = -EINVAL;
1003 goto fini; 1002 goto fini;
1004 } 1003 }
1005 1004
1006 i915_gem_request_put(prev); 1005 i915_request_put(prev);
1007 prev = rq; 1006 prev = rq;
1008 count++; 1007 count++;
1009 } while (time_before(jiffies, end_time)); 1008 } while (time_before(jiffies, end_time));
@@ -1012,7 +1011,7 @@ static int igt_reset_queue(void *arg)
1012 *h.batch = MI_BATCH_BUFFER_END; 1011 *h.batch = MI_BATCH_BUFFER_END;
1013 i915_gem_chipset_flush(i915); 1012 i915_gem_chipset_flush(i915);
1014 1013
1015 i915_gem_request_put(prev); 1014 i915_request_put(prev);
1016 1015
1017 err = flush_test(i915, I915_WAIT_LOCKED); 1016 err = flush_test(i915, I915_WAIT_LOCKED);
1018 if (err) 1017 if (err)
@@ -1036,7 +1035,7 @@ static int igt_handle_error(void *arg)
1036 struct drm_i915_private *i915 = arg; 1035 struct drm_i915_private *i915 = arg;
1037 struct intel_engine_cs *engine = i915->engine[RCS]; 1036 struct intel_engine_cs *engine = i915->engine[RCS];
1038 struct hang h; 1037 struct hang h;
1039 struct drm_i915_gem_request *rq; 1038 struct i915_request *rq;
1040 struct i915_gpu_state *error; 1039 struct i915_gpu_state *error;
1041 int err; 1040 int err;
1042 1041
@@ -1060,8 +1059,8 @@ static int igt_handle_error(void *arg)
1060 goto err_fini; 1059 goto err_fini;
1061 } 1060 }
1062 1061
1063 i915_gem_request_get(rq); 1062 i915_request_get(rq);
1064 __i915_add_request(rq, true); 1063 __i915_request_add(rq, true);
1065 1064
1066 if (!wait_for_hang(&h, rq)) { 1065 if (!wait_for_hang(&h, rq)) {
1067 struct drm_printer p = drm_info_printer(i915->drm.dev); 1066 struct drm_printer p = drm_info_printer(i915->drm.dev);
@@ -1098,7 +1097,7 @@ static int igt_handle_error(void *arg)
1098 } 1097 }
1099 1098
1100err_request: 1099err_request:
1101 i915_gem_request_put(rq); 1100 i915_request_put(rq);
1102err_fini: 1101err_fini:
1103 hang_fini(&h); 1102 hang_fini(&h);
1104err_unlock: 1103err_unlock:
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 2f6367643171..f76f2597df5c 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -61,20 +61,30 @@ static int intel_fw_table_check(const struct intel_forcewake_range *ranges,
61 61
62static int intel_shadow_table_check(void) 62static int intel_shadow_table_check(void)
63{ 63{
64 const i915_reg_t *reg = gen8_shadowed_regs; 64 struct {
65 unsigned int i; 65 const i915_reg_t *regs;
66 unsigned int size;
67 } reg_lists[] = {
68 { gen8_shadowed_regs, ARRAY_SIZE(gen8_shadowed_regs) },
69 { gen11_shadowed_regs, ARRAY_SIZE(gen11_shadowed_regs) },
70 };
71 const i915_reg_t *reg;
72 unsigned int i, j;
66 s32 prev; 73 s32 prev;
67 74
68 for (i = 0, prev = -1; i < ARRAY_SIZE(gen8_shadowed_regs); i++, reg++) { 75 for (j = 0; j < ARRAY_SIZE(reg_lists); ++j) {
69 u32 offset = i915_mmio_reg_offset(*reg); 76 reg = reg_lists[j].regs;
77 for (i = 0, prev = -1; i < reg_lists[j].size; i++, reg++) {
78 u32 offset = i915_mmio_reg_offset(*reg);
70 79
71 if (prev >= (s32)offset) { 80 if (prev >= (s32)offset) {
72 pr_err("%s: entry[%d]:(%x) is before previous (%x)\n", 81 pr_err("%s: entry[%d]:(%x) is before previous (%x)\n",
73 __func__, i, offset, prev); 82 __func__, i, offset, prev);
74 return -EINVAL; 83 return -EINVAL;
75 } 84 }
76 85
77 prev = offset; 86 prev = offset;
87 }
78 } 88 }
79 89
80 return 0; 90 return 0;
@@ -90,6 +100,7 @@ int intel_uncore_mock_selftests(void)
90 { __vlv_fw_ranges, ARRAY_SIZE(__vlv_fw_ranges), false }, 100 { __vlv_fw_ranges, ARRAY_SIZE(__vlv_fw_ranges), false },
91 { __chv_fw_ranges, ARRAY_SIZE(__chv_fw_ranges), false }, 101 { __chv_fw_ranges, ARRAY_SIZE(__chv_fw_ranges), false },
92 { __gen9_fw_ranges, ARRAY_SIZE(__gen9_fw_ranges), true }, 102 { __gen9_fw_ranges, ARRAY_SIZE(__gen9_fw_ranges), true },
103 { __gen11_fw_ranges, ARRAY_SIZE(__gen11_fw_ranges), true },
93 }; 104 };
94 int err, i; 105 int err, i;
95 106
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 55c0e2c15782..78a89efa1119 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -81,7 +81,7 @@ static void mock_context_unpin(struct intel_engine_cs *engine,
81 i915_gem_context_put(ctx); 81 i915_gem_context_put(ctx);
82} 82}
83 83
84static int mock_request_alloc(struct drm_i915_gem_request *request) 84static int mock_request_alloc(struct i915_request *request)
85{ 85{
86 struct mock_request *mock = container_of(request, typeof(*mock), base); 86 struct mock_request *mock = container_of(request, typeof(*mock), base);
87 87
@@ -91,24 +91,24 @@ static int mock_request_alloc(struct drm_i915_gem_request *request)
91 return 0; 91 return 0;
92} 92}
93 93
94static int mock_emit_flush(struct drm_i915_gem_request *request, 94static int mock_emit_flush(struct i915_request *request,
95 unsigned int flags) 95 unsigned int flags)
96{ 96{
97 return 0; 97 return 0;
98} 98}
99 99
100static void mock_emit_breadcrumb(struct drm_i915_gem_request *request, 100static void mock_emit_breadcrumb(struct i915_request *request,
101 u32 *flags) 101 u32 *flags)
102{ 102{
103} 103}
104 104
105static void mock_submit_request(struct drm_i915_gem_request *request) 105static void mock_submit_request(struct i915_request *request)
106{ 106{
107 struct mock_request *mock = container_of(request, typeof(*mock), base); 107 struct mock_request *mock = container_of(request, typeof(*mock), base);
108 struct mock_engine *engine = 108 struct mock_engine *engine =
109 container_of(request->engine, typeof(*engine), base); 109 container_of(request->engine, typeof(*engine), base);
110 110
111 i915_gem_request_submit(request); 111 i915_request_submit(request);
112 GEM_BUG_ON(!request->global_seqno); 112 GEM_BUG_ON(!request->global_seqno);
113 113
114 spin_lock_irq(&engine->hw_lock); 114 spin_lock_irq(&engine->hw_lock);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 3175db70cc6e..e6d4b882599a 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -43,7 +43,7 @@ void mock_device_flush(struct drm_i915_private *i915)
43 for_each_engine(engine, i915, id) 43 for_each_engine(engine, i915, id)
44 mock_engine_flush(engine); 44 mock_engine_flush(engine);
45 45
46 i915_gem_retire_requests(i915); 46 i915_retire_requests(i915);
47} 47}
48 48
49static void mock_device_release(struct drm_device *dev) 49static void mock_device_release(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/selftests/mock_request.c b/drivers/gpu/drm/i915/selftests/mock_request.c
index 8097e3693ec4..0dc29e242597 100644
--- a/drivers/gpu/drm/i915/selftests/mock_request.c
+++ b/drivers/gpu/drm/i915/selftests/mock_request.c
@@ -25,16 +25,16 @@
25#include "mock_engine.h" 25#include "mock_engine.h"
26#include "mock_request.h" 26#include "mock_request.h"
27 27
28struct drm_i915_gem_request * 28struct i915_request *
29mock_request(struct intel_engine_cs *engine, 29mock_request(struct intel_engine_cs *engine,
30 struct i915_gem_context *context, 30 struct i915_gem_context *context,
31 unsigned long delay) 31 unsigned long delay)
32{ 32{
33 struct drm_i915_gem_request *request; 33 struct i915_request *request;
34 struct mock_request *mock; 34 struct mock_request *mock;
35 35
36 /* NB the i915->requests slab cache is enlarged to fit mock_request */ 36 /* NB the i915->requests slab cache is enlarged to fit mock_request */
37 request = i915_gem_request_alloc(engine, context); 37 request = i915_request_alloc(engine, context);
38 if (IS_ERR(request)) 38 if (IS_ERR(request))
39 return NULL; 39 return NULL;
40 40
@@ -44,7 +44,7 @@ mock_request(struct intel_engine_cs *engine,
44 return &mock->base; 44 return &mock->base;
45} 45}
46 46
47bool mock_cancel_request(struct drm_i915_gem_request *request) 47bool mock_cancel_request(struct i915_request *request)
48{ 48{
49 struct mock_request *mock = container_of(request, typeof(*mock), base); 49 struct mock_request *mock = container_of(request, typeof(*mock), base);
50 struct mock_engine *engine = 50 struct mock_engine *engine =
@@ -57,7 +57,7 @@ bool mock_cancel_request(struct drm_i915_gem_request *request)
57 spin_unlock_irq(&engine->hw_lock); 57 spin_unlock_irq(&engine->hw_lock);
58 58
59 if (was_queued) 59 if (was_queued)
60 i915_gem_request_unsubmit(request); 60 i915_request_unsubmit(request);
61 61
62 return was_queued; 62 return was_queued;
63} 63}
diff --git a/drivers/gpu/drm/i915/selftests/mock_request.h b/drivers/gpu/drm/i915/selftests/mock_request.h
index 4dea74c8e96d..995fb728380c 100644
--- a/drivers/gpu/drm/i915/selftests/mock_request.h
+++ b/drivers/gpu/drm/i915/selftests/mock_request.h
@@ -27,20 +27,20 @@
27 27
28#include <linux/list.h> 28#include <linux/list.h>
29 29
30#include "../i915_gem_request.h" 30#include "../i915_request.h"
31 31
32struct mock_request { 32struct mock_request {
33 struct drm_i915_gem_request base; 33 struct i915_request base;
34 34
35 struct list_head link; 35 struct list_head link;
36 unsigned long delay; 36 unsigned long delay;
37}; 37};
38 38
39struct drm_i915_gem_request * 39struct i915_request *
40mock_request(struct intel_engine_cs *engine, 40mock_request(struct intel_engine_cs *engine,
41 struct i915_gem_context *context, 41 struct i915_gem_context *context,
42 unsigned long delay); 42 unsigned long delay);
43 43
44bool mock_cancel_request(struct drm_i915_gem_request *request); 44bool mock_cancel_request(struct i915_request *request);
45 45
46#endif /* !__MOCK_REQUEST__ */ 46#endif /* !__MOCK_REQUEST__ */
diff --git a/include/drm/i915_pciids.h b/include/drm/i915_pciids.h
index 0b2ba46fa00b..70f0c2535b87 100644
--- a/include/drm/i915_pciids.h
+++ b/include/drm/i915_pciids.h
@@ -431,4 +431,16 @@
431 INTEL_VGA_DEVICE(0x5A44, info), \ 431 INTEL_VGA_DEVICE(0x5A44, info), \
432 INTEL_VGA_DEVICE(0x5A4C, info) 432 INTEL_VGA_DEVICE(0x5A4C, info)
433 433
434/* ICL */
435#define INTEL_ICL_11_IDS(info) \
436 INTEL_VGA_DEVICE(0x8A50, info), \
437 INTEL_VGA_DEVICE(0x8A51, info), \
438 INTEL_VGA_DEVICE(0x8A5C, info), \
439 INTEL_VGA_DEVICE(0x8A5D, info), \
440 INTEL_VGA_DEVICE(0x8A52, info), \
441 INTEL_VGA_DEVICE(0x8A5A, info), \
442 INTEL_VGA_DEVICE(0x8A5B, info), \
443 INTEL_VGA_DEVICE(0x8A71, info), \
444 INTEL_VGA_DEVICE(0x8A70, info)
445
434#endif /* _I915_PCIIDS_H */ 446#endif /* _I915_PCIIDS_H */
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 29fa48e4755d..7f5634ce8e88 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -318,6 +318,7 @@ typedef struct _drm_i915_sarea {
318#define DRM_I915_PERF_OPEN 0x36 318#define DRM_I915_PERF_OPEN 0x36
319#define DRM_I915_PERF_ADD_CONFIG 0x37 319#define DRM_I915_PERF_ADD_CONFIG 0x37
320#define DRM_I915_PERF_REMOVE_CONFIG 0x38 320#define DRM_I915_PERF_REMOVE_CONFIG 0x38
321#define DRM_I915_QUERY 0x39
321 322
322#define DRM_IOCTL_I915_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t) 323#define DRM_IOCTL_I915_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
323#define DRM_IOCTL_I915_FLUSH DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH) 324#define DRM_IOCTL_I915_FLUSH DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
@@ -375,6 +376,7 @@ typedef struct _drm_i915_sarea {
375#define DRM_IOCTL_I915_PERF_OPEN DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_OPEN, struct drm_i915_perf_open_param) 376#define DRM_IOCTL_I915_PERF_OPEN DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_OPEN, struct drm_i915_perf_open_param)
376#define DRM_IOCTL_I915_PERF_ADD_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_ADD_CONFIG, struct drm_i915_perf_oa_config) 377#define DRM_IOCTL_I915_PERF_ADD_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_ADD_CONFIG, struct drm_i915_perf_oa_config)
377#define DRM_IOCTL_I915_PERF_REMOVE_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_REMOVE_CONFIG, __u64) 378#define DRM_IOCTL_I915_PERF_REMOVE_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_REMOVE_CONFIG, __u64)
379#define DRM_IOCTL_I915_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_QUERY, struct drm_i915_query)
378 380
379/* Allow drivers to submit batchbuffers directly to hardware, relying 381/* Allow drivers to submit batchbuffers directly to hardware, relying
380 * on the security mechanisms provided by hardware. 382 * on the security mechanisms provided by hardware.
@@ -1606,15 +1608,115 @@ struct drm_i915_perf_oa_config {
1606 __u32 n_flex_regs; 1608 __u32 n_flex_regs;
1607 1609
1608 /* 1610 /*
1609 * These fields are pointers to tuples of u32 values (register 1611 * These fields are pointers to tuples of u32 values (register address,
1610 * address, value). For example the expected length of the buffer 1612 * value). For example the expected length of the buffer pointed by
1611 * pointed by mux_regs_ptr is (2 * sizeof(u32) * n_mux_regs). 1613 * mux_regs_ptr is (2 * sizeof(u32) * n_mux_regs).
1612 */ 1614 */
1613 __u64 mux_regs_ptr; 1615 __u64 mux_regs_ptr;
1614 __u64 boolean_regs_ptr; 1616 __u64 boolean_regs_ptr;
1615 __u64 flex_regs_ptr; 1617 __u64 flex_regs_ptr;
1616}; 1618};
1617 1619
1620struct drm_i915_query_item {
1621 __u64 query_id;
1622#define DRM_I915_QUERY_TOPOLOGY_INFO 1
1623
1624 /*
1625 * When set to zero by userspace, this is filled with the size of the
1626 * data to be written at the data_ptr pointer. The kernel sets this
1627 * value to a negative value to signal an error on a particular query
1628 * item.
1629 */
1630 __s32 length;
1631
1632 /*
1633 * Unused for now. Must be cleared to zero.
1634 */
1635 __u32 flags;
1636
1637 /*
1638 * Data will be written at the location pointed by data_ptr when the
1639 * value of length matches the length of the data to be written by the
1640 * kernel.
1641 */
1642 __u64 data_ptr;
1643};
1644
1645struct drm_i915_query {
1646 __u32 num_items;
1647
1648 /*
1649 * Unused for now. Must be cleared to zero.
1650 */
1651 __u32 flags;
1652
1653 /*
1654 * This points to an array of num_items drm_i915_query_item structures.
1655 */
1656 __u64 items_ptr;
1657};
1658
1659/*
1660 * Data written by the kernel with query DRM_I915_QUERY_TOPOLOGY_INFO :
1661 *
1662 * data: contains the 3 pieces of information :
1663 *
1664 * - the slice mask with one bit per slice telling whether a slice is
1665 * available. The availability of slice X can be queried with the following
1666 * formula :
1667 *
1668 * (data[X / 8] >> (X % 8)) & 1
1669 *
1670 * - the subslice mask for each slice with one bit per subslice telling
1671 * whether a subslice is available. The availability of subslice Y in slice
1672 * X can be queried with the following formula :
1673 *
1674 * (data[subslice_offset +
1675 * X * subslice_stride +
1676 * Y / 8] >> (Y % 8)) & 1
1677 *
1678 * - the EU mask for each subslice in each slice with one bit per EU telling
1679 * whether an EU is available. The availability of EU Z in subslice Y in
1680 * slice X can be queried with the following formula :
1681 *
1682 * (data[eu_offset +
1683 * (X * max_subslices + Y) * eu_stride +
1684 * Z / 8] >> (Z % 8)) & 1
1685 */
1686struct drm_i915_query_topology_info {
1687 /*
1688 * Unused for now. Must be cleared to zero.
1689 */
1690 __u16 flags;
1691
1692 __u16 max_slices;
1693 __u16 max_subslices;
1694 __u16 max_eus_per_subslice;
1695
1696 /*
1697 * Offset in data[] at which the subslice masks are stored.
1698 */
1699 __u16 subslice_offset;
1700
1701 /*
1702 * Stride at which each of the subslice masks for each slice are
1703 * stored.
1704 */
1705 __u16 subslice_stride;
1706
1707 /*
1708 * Offset in data[] at which the EU masks are stored.
1709 */
1710 __u16 eu_offset;
1711
1712 /*
1713 * Stride at which each of the EU masks for each subslice are stored.
1714 */
1715 __u16 eu_stride;
1716
1717 __u8 data[];
1718};
1719
1618#if defined(__cplusplus) 1720#if defined(__cplusplus)
1619} 1721}
1620#endif 1722#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-21 23:26:52 -0400