aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2016-11-29 23:21:35 -0500
committerDave Airlie <airlied@redhat.com>2016-11-29 23:21:35 -0500
commit35838b470ab592071087a46804d373c5571dc032 (patch)
treeddad21fc37545addfbf642bf388ba6221375939e
parent63207455963053ca212e61c75f43b3502ea69f0e (diff)
parente9cbc4bd0140e1d4e0172e2fe8fe07ba278e5980 (diff)
Merge tag 'drm-intel-next-2016-11-21' of git://anongit.freedesktop.org/git/drm-intel into drm-next
Final 4.10 updates: - fine-tune fb flushing and tracking (Chris Wilson) - refactor state check dumper code for more conciseness (Tvrtko) - roll out dev_priv all over the place (Tvrkto) - finally remove __i915__ magic macro (Tvrtko) - more gvt bugfixes (Zhenyu&team) - better opregion CADL handling (Jani) - refactor/clean up wm programming (Maarten) - gpu scheduler + priority boosting for flips as first user (Chris Wilson) - make fbc use more atomic (Paulo) - initial kvm-gvt framework, but not yet complete (Zhenyu&team) * tag 'drm-intel-next-2016-11-21' of git://anongit.freedesktop.org/git/drm-intel: (127 commits) drm/i915: Update DRIVER_DATE to 20161121 drm/i915: Skip final clflush if LLC is coherent drm/i915: Always flush the dirty CPU cache when pinning the scanout drm/i915: Don't touch NULL sg on i915_gem_object_get_pages_gtt() error drm/i915: Check that each request phase is completed before retiring drm/i915: i915_pages_create_for_stolen should return err ptr drm/i915: Enable support for nonblocking modeset drm/i915: Be more careful to drop the GT wakeref drm/i915: Move frontbuffer CS write tracking from ggtt vma to object drm/i915: Only dump dp_m2_n2 configuration when drrs is used drm/i915: don't leak global_timeline drm/i915: add i915_address_space_fini drm/i915: Add a few more sanity checks for stolen handling drm/i915: Waterproof verification of gen9 forcewake table ranges drm/i915: Introduce enableddisabled helper drm/i915: Only dump possible panel fitter config for the platform drm/i915: Only dump scaler config where supported drm/i915: Compact a few pipe config debug lines drm/i915: Don't log pipe config kernel pointer and duplicated pipe name drm/i915: Dump FDI config only where applicable ...
Diffstat
-rw-r--r--arch/x86/include/asm/kvm_page_track.h14
-rw-r--r--arch/x86/kvm/mmu.c11
-rw-r--r--arch/x86/kvm/page_track.c31
-rw-r--r--arch/x86/kvm/x86.c2
-rw-r--r--drivers/gpu/drm/drm_edid.c26
-rw-r--r--drivers/gpu/drm/i915/Kconfig26
-rw-r--r--drivers/gpu/drm/i915/Makefile3
-rw-r--r--drivers/gpu/drm/i915/gvt/Makefile7
-rw-r--r--drivers/gpu/drm/i915/gvt/cfg_space.c12
-rw-r--r--drivers/gpu/drm/i915/gvt/cmd_parser.c7
-rw-r--r--drivers/gpu/drm/i915/gvt/edid.c3
-rw-r--r--drivers/gpu/drm/i915/gvt/edid.h2
-rw-r--r--drivers/gpu/drm/i915/gvt/execlist.c24
-rw-r--r--drivers/gpu/drm/i915/gvt/execlist.h2
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.c202
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.h40
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.c29
-rw-r--r--drivers/gpu/drm/i915/gvt/gvt.h63
-rw-r--r--drivers/gpu/drm/i915/gvt/handlers.c73
-rw-r--r--drivers/gpu/drm/i915/gvt/hypercall.h14
-rw-r--r--drivers/gpu/drm/i915/gvt/kvmgt.c597
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio.c6
-rw-r--r--drivers/gpu/drm/i915/gvt/mmio.h9
-rw-r--r--drivers/gpu/drm/i915/gvt/mpt.h55
-rw-r--r--drivers/gpu/drm/i915/gvt/opregion.c34
-rw-r--r--drivers/gpu/drm/i915/gvt/render.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/sched_policy.c2
-rw-r--r--drivers/gpu/drm/i915/gvt/scheduler.c51
-rw-r--r--drivers/gpu/drm/i915/gvt/vgpu.c180
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c18
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c32
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h543
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c608
-rw-r--r--drivers/gpu/drm/i915/i915_gem.h2
-rw-r--r--drivers/gpu/drm/i915/i915_gem_context.c1
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c13
-rw-r--r--drivers/gpu/drm/i915/i915_gem_fence_reg.c (renamed from drivers/gpu/drm/i915/i915_gem_fence.c)12
-rw-r--r--drivers/gpu/drm/i915/i915_gem_fence_reg.h51
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c579
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.h235
-rw-r--r--drivers/gpu/drm/i915/i915_gem_object.h338
-rw-r--r--drivers/gpu/drm/i915/i915_gem_request.c206
-rw-r--r--drivers/gpu/drm/i915/i915_gem_request.h74
-rw-r--r--drivers/gpu/drm/i915/i915_gem_stolen.c39
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c12
-rw-r--r--drivers/gpu/drm/i915/i915_gem_timeline.c33
-rw-r--r--drivers/gpu/drm/i915/i915_gem_timeline.h1
-rw-r--r--drivers/gpu/drm/i915/i915_gem_userptr.c3
-rw-r--r--drivers/gpu/drm/i915/i915_gpu_error.c39
-rw-r--r--drivers/gpu/drm/i915/i915_guc_submission.c15
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c32
-rw-r--r--drivers/gpu/drm/i915/i915_params.c9
-rw-r--r--drivers/gpu/drm/i915/i915_params.h2
-rw-r--r--drivers/gpu/drm/i915/i915_pci.c8
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h7
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c24
-rw-r--r--drivers/gpu/drm/i915/i915_sw_fence.c7
-rw-r--r--drivers/gpu/drm/i915/i915_sw_fence.h17
-rw-r--r--drivers/gpu/drm/i915/i915_vma.c638
-rw-r--r--drivers/gpu/drm/i915/i915_vma.h341
-rw-r--r--drivers/gpu/drm/i915/intel_atomic_plane.c9
-rw-r--r--drivers/gpu/drm/i915/intel_audio.c17
-rw-r--r--drivers/gpu/drm/i915/intel_bios.c30
-rw-r--r--drivers/gpu/drm/i915/intel_breadcrumbs.c31
-rw-r--r--drivers/gpu/drm/i915/intel_color.c36
-rw-r--r--drivers/gpu/drm/i915/intel_crt.c23
-rw-r--r--drivers/gpu/drm/i915/intel_ddi.c12
-rw-r--r--drivers/gpu/drm/i915/intel_display.c598
-rw-r--r--drivers/gpu/drm/i915/intel_dp.c52
-rw-r--r--drivers/gpu/drm/i915/intel_dp_mst.c1
-rw-r--r--drivers/gpu/drm/i915/intel_dpll_mgr.c5
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h28
-rw-r--r--drivers/gpu/drm/i915/intel_dsi_panel_vbt.c8
-rw-r--r--drivers/gpu/drm/i915/intel_engine_cs.c7
-rw-r--r--drivers/gpu/drm/i915/intel_fbc.c83
-rw-r--r--drivers/gpu/drm/i915/intel_fbdev.c10
-rw-r--r--drivers/gpu/drm/i915/intel_frontbuffer.h5
-rw-r--r--drivers/gpu/drm/i915/intel_guc_loader.c10
-rw-r--r--drivers/gpu/drm/i915/intel_hdmi.c50
-rw-r--r--drivers/gpu/drm/i915/intel_hotplug.c2
-rw-r--r--drivers/gpu/drm/i915/intel_lrc.c187
-rw-r--r--drivers/gpu/drm/i915/intel_lvds.c7
-rw-r--r--drivers/gpu/drm/i915/intel_opregion.c139
-rw-r--r--drivers/gpu/drm/i915/intel_panel.c12
-rw-r--r--drivers/gpu/drm/i915/intel_pm.c170
-rw-r--r--drivers/gpu/drm/i915/intel_psr.c6
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c2
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.h16
-rw-r--r--drivers/gpu/drm/i915/intel_runtime_pm.c4
-rw-r--r--drivers/gpu/drm/i915/intel_sdvo.c11
-rw-r--r--drivers/gpu/drm/i915/intel_sprite.c59
-rw-r--r--drivers/gpu/drm/i915/intel_tv.c5
-rw-r--r--drivers/gpu/drm/i915/intel_uncore.c121
-rw-r--r--drivers/gpu/drm/i915/intel_vbt_defs.h3
-rw-r--r--include/drm/drm_edid.h1
-rw-r--r--include/uapi/drm/i915_drm.h5
96 files changed, 4442 insertions, 2789 deletions
diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h
index c2b8d24a235c..d74747b031ec 100644
--- a/arch/x86/include/asm/kvm_page_track.h
+++ b/arch/x86/include/asm/kvm_page_track.h
@@ -29,9 +29,20 @@ struct kvm_page_track_notifier_node {
29 * @gpa: the physical address written by guest. 29 * @gpa: the physical address written by guest.
30 * @new: the data was written to the address. 30 * @new: the data was written to the address.
31 * @bytes: the written length. 31 * @bytes: the written length.
32 * @node: this node
32 */ 33 */
33 void (*track_write)(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 34 void (*track_write)(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
34 int bytes); 35 int bytes, struct kvm_page_track_notifier_node *node);
36 /*
37 * It is called when memory slot is being moved or removed
38 * users can drop write-protection for the pages in that memory slot
39 *
40 * @kvm: the kvm where memory slot being moved or removed
41 * @slot: the memory slot being moved or removed
42 * @node: this node
43 */
44 void (*track_flush_slot)(struct kvm *kvm, struct kvm_memory_slot *slot,
45 struct kvm_page_track_notifier_node *node);
35}; 46};
36 47
37void kvm_page_track_init(struct kvm *kvm); 48void kvm_page_track_init(struct kvm *kvm);
@@ -58,4 +69,5 @@ kvm_page_track_unregister_notifier(struct kvm *kvm,
58 struct kvm_page_track_notifier_node *n); 69 struct kvm_page_track_notifier_node *n);
59void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, 70void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
60 int bytes); 71 int bytes);
72void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot);
61#endif 73#endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index d9c7e986b4e4..87c5880ba3b7 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4405,7 +4405,8 @@ static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
4405} 4405}
4406 4406
4407static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 4407static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
4408 const u8 *new, int bytes) 4408 const u8 *new, int bytes,
4409 struct kvm_page_track_notifier_node *node)
4409{ 4410{
4410 gfn_t gfn = gpa >> PAGE_SHIFT; 4411 gfn_t gfn = gpa >> PAGE_SHIFT;
4411 struct kvm_mmu_page *sp; 4412 struct kvm_mmu_page *sp;
@@ -4617,11 +4618,19 @@ void kvm_mmu_setup(struct kvm_vcpu *vcpu)
4617 init_kvm_mmu(vcpu); 4618 init_kvm_mmu(vcpu);
4618} 4619}
4619 4620
4621static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
4622 struct kvm_memory_slot *slot,
4623 struct kvm_page_track_notifier_node *node)
4624{
4625 kvm_mmu_invalidate_zap_all_pages(kvm);
4626}
4627
4620void kvm_mmu_init_vm(struct kvm *kvm) 4628void kvm_mmu_init_vm(struct kvm *kvm)
4621{ 4629{
4622 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; 4630 struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
4623 4631
4624 node->track_write = kvm_mmu_pte_write; 4632 node->track_write = kvm_mmu_pte_write;
4633 node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
4625 kvm_page_track_register_notifier(kvm, node); 4634 kvm_page_track_register_notifier(kvm, node);
4626} 4635}
4627 4636
diff --git a/arch/x86/kvm/page_track.c b/arch/x86/kvm/page_track.c
index b431539c3714..4a1c13eaa518 100644
--- a/arch/x86/kvm/page_track.c
+++ b/arch/x86/kvm/page_track.c
@@ -106,6 +106,7 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
106 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) 106 if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn))
107 kvm_flush_remote_tlbs(kvm); 107 kvm_flush_remote_tlbs(kvm);
108} 108}
109EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
109 110
110/* 111/*
111 * remove the guest page from the tracking pool which stops the interception 112 * remove the guest page from the tracking pool which stops the interception
@@ -135,6 +136,7 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm,
135 */ 136 */
136 kvm_mmu_gfn_allow_lpage(slot, gfn); 137 kvm_mmu_gfn_allow_lpage(slot, gfn);
137} 138}
139EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page);
138 140
139/* 141/*
140 * check if the corresponding access on the specified guest page is tracked. 142 * check if the corresponding access on the specified guest page is tracked.
@@ -181,6 +183,7 @@ kvm_page_track_register_notifier(struct kvm *kvm,
181 hlist_add_head_rcu(&n->node, &head->track_notifier_list); 183 hlist_add_head_rcu(&n->node, &head->track_notifier_list);
182 spin_unlock(&kvm->mmu_lock); 184 spin_unlock(&kvm->mmu_lock);
183} 185}
186EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
184 187
185/* 188/*
186 * stop receiving the event interception. It is the opposed operation of 189 * stop receiving the event interception. It is the opposed operation of
@@ -199,6 +202,7 @@ kvm_page_track_unregister_notifier(struct kvm *kvm,
199 spin_unlock(&kvm->mmu_lock); 202 spin_unlock(&kvm->mmu_lock);
200 synchronize_srcu(&head->track_srcu); 203 synchronize_srcu(&head->track_srcu);
201} 204}
205EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
202 206
203/* 207/*
204 * Notify the node that write access is intercepted and write emulation is 208 * Notify the node that write access is intercepted and write emulation is
@@ -222,6 +226,31 @@ void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
222 idx = srcu_read_lock(&head->track_srcu); 226 idx = srcu_read_lock(&head->track_srcu);
223 hlist_for_each_entry_rcu(n, &head->track_notifier_list, node) 227 hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
224 if (n->track_write) 228 if (n->track_write)
225 n->track_write(vcpu, gpa, new, bytes); 229 n->track_write(vcpu, gpa, new, bytes, n);
230 srcu_read_unlock(&head->track_srcu, idx);
231}
232
233/*
234 * Notify the node that memory slot is being removed or moved so that it can
235 * drop write-protection for the pages in the memory slot.
236 *
237 * The node should figure out it has any write-protected pages in this slot
238 * by itself.
239 */
240void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
241{
242 struct kvm_page_track_notifier_head *head;
243 struct kvm_page_track_notifier_node *n;
244 int idx;
245
246 head = &kvm->arch.track_notifier_head;
247
248 if (hlist_empty(&head->track_notifier_list))
249 return;
250
251 idx = srcu_read_lock(&head->track_srcu);
252 hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
253 if (n->track_flush_slot)
254 n->track_flush_slot(kvm, slot, n);
226 srcu_read_unlock(&head->track_srcu, idx); 255 srcu_read_unlock(&head->track_srcu, idx);
227} 256}
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 3017de0431bd..7e30c720d0c5 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -8155,7 +8155,7 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm)
8155void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 8155void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
8156 struct kvm_memory_slot *slot) 8156 struct kvm_memory_slot *slot)
8157{ 8157{
8158 kvm_mmu_invalidate_zap_all_pages(kvm); 8158 kvm_page_track_flush_slot(kvm, slot);
8159} 8159}
8160 8160
8161static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 8161static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 7eec18925b70..6798c3ad9d53 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -3611,32 +3611,6 @@ int drm_av_sync_delay(struct drm_connector *connector,
3611EXPORT_SYMBOL(drm_av_sync_delay); 3611EXPORT_SYMBOL(drm_av_sync_delay);
3612 3612
3613/** 3613/**
3614 * drm_select_eld - select one ELD from multiple HDMI/DP sinks
3615 * @encoder: the encoder just changed display mode
3616 *
3617 * It's possible for one encoder to be associated with multiple HDMI/DP sinks.
3618 * The policy is now hard coded to simply use the first HDMI/DP sink's ELD.
3619 *
3620 * Return: The connector associated with the first HDMI/DP sink that has ELD
3621 * attached to it.
3622 */
3623struct drm_connector *drm_select_eld(struct drm_encoder *encoder)
3624{
3625 struct drm_connector *connector;
3626 struct drm_device *dev = encoder->dev;
3627
3628 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
3629 WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
3630
3631 drm_for_each_connector(connector, dev)
3632 if (connector->encoder == encoder && connector->eld[0])
3633 return connector;
3634
3635 return NULL;
3636}
3637EXPORT_SYMBOL(drm_select_eld);
3638
3639/**
3640 * drm_detect_hdmi_monitor - detect whether monitor is HDMI 3614 * drm_detect_hdmi_monitor - detect whether monitor is HDMI
3641 * @edid: monitor EDID information 3615 * @edid: monitor EDID information
3642 * 3616 *
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index df96aed6975a..5ddde7349fbd 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -36,15 +36,20 @@ config DRM_I915
36 36
37 If "M" is selected, the module will be called i915. 37 If "M" is selected, the module will be called i915.
38 38
39config DRM_I915_PRELIMINARY_HW_SUPPORT 39config DRM_I915_ALPHA_SUPPORT
40 bool "Enable preliminary support for prerelease Intel hardware by default" 40 bool "Enable alpha quality support for new Intel hardware by default"
41 depends on DRM_I915 41 depends on DRM_I915
42 default n 42 default n
43 help 43 help
44 Choose this option if you have prerelease Intel hardware and want the 44 Choose this option if you have new Intel hardware and want to enable
45 i915 driver to support it by default. You can enable such support at 45 the alpha quality i915 driver support for the hardware in this kernel
46 runtime with the module option i915.preliminary_hw_support=1; this 46 version. You can also enable the support at runtime using the module
47 option changes the default for that module option. 47 parameter i915.alpha_support=1; this option changes the default for
48 that module parameter.
49
50 It is recommended to upgrade to a kernel version with proper support
51 as soon as it is available. Generally fixes for platforms with alpha
52 support are not backported to older kernels.
48 53
49 If in doubt, say "N". 54 If in doubt, say "N".
50 55
@@ -107,6 +112,15 @@ config DRM_I915_GVT
107 112
108 If in doubt, say "N". 113 If in doubt, say "N".
109 114
115config DRM_I915_GVT_KVMGT
116 tristate "Enable KVM/VFIO support for Intel GVT-g"
117 depends on DRM_I915_GVT
118 depends on KVM
119 default n
120 help
121 Choose this option if you want to enable KVMGT support for
122 Intel GVT-g.
123
110menu "drm/i915 Debugging" 124menu "drm/i915 Debugging"
111depends on DRM_I915 125depends on DRM_I915
112depends on EXPERT 126depends on EXPERT
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 0857e5035f4d..3dea46af9fe6 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -33,7 +33,7 @@ i915-y += i915_cmd_parser.o \
33 i915_gem_dmabuf.o \ 33 i915_gem_dmabuf.o \
34 i915_gem_evict.o \ 34 i915_gem_evict.o \
35 i915_gem_execbuffer.o \ 35 i915_gem_execbuffer.o \
36 i915_gem_fence.o \ 36 i915_gem_fence_reg.o \
37 i915_gem_gtt.o \ 37 i915_gem_gtt.o \
38 i915_gem_internal.o \ 38 i915_gem_internal.o \
39 i915_gem.o \ 39 i915_gem.o \
@@ -45,6 +45,7 @@ i915-y += i915_cmd_parser.o \
45 i915_gem_timeline.o \ 45 i915_gem_timeline.o \
46 i915_gem_userptr.o \ 46 i915_gem_userptr.o \
47 i915_trace_points.o \ 47 i915_trace_points.o \
48 i915_vma.o \
48 intel_breadcrumbs.o \ 49 intel_breadcrumbs.o \
49 intel_engine_cs.o \ 50 intel_engine_cs.o \
50 intel_hangcheck.o \ 51 intel_hangcheck.o \
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 34ea4776af70..8a46a7f31d53 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -3,5 +3,8 @@ GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
3 interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \ 3 interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
4 execlist.o scheduler.o sched_policy.o render.o cmd_parser.o 4 execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
5 5
6ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) -Wall 6ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) -Wall
7i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) 7i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
8
9CFLAGS_kvmgt.o := -Wno-unused-function
10obj-$(CONFIG_DRM_I915_GVT_KVMGT) += $(GVT_DIR)/kvmgt.o
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index 4c687740f5f1..db516382a4d4 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -47,11 +47,9 @@ enum {
47 * Returns: 47 * Returns:
48 * Zero on success, negative error code if failed. 48 * Zero on success, negative error code if failed.
49 */ 49 */
50int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset, 50int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
51 void *p_data, unsigned int bytes) 51 void *p_data, unsigned int bytes)
52{ 52{
53 struct intel_vgpu *vgpu = __vgpu;
54
55 if (WARN_ON(bytes > 4)) 53 if (WARN_ON(bytes > 4))
56 return -EINVAL; 54 return -EINVAL;
57 55
@@ -82,9 +80,8 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map)
82 80
83 ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn, 81 ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn,
84 first_mfn, 82 first_mfn,
85 vgpu_aperture_sz(vgpu) 83 vgpu_aperture_sz(vgpu) >>
86 >> PAGE_SHIFT, map, 84 PAGE_SHIFT, map);
87 GVT_MAP_APERTURE);
88 if (ret) 85 if (ret)
89 return ret; 86 return ret;
90 87
@@ -235,10 +232,9 @@ static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset,
235 * Returns: 232 * Returns:
236 * Zero on success, negative error code if failed. 233 * Zero on success, negative error code if failed.
237 */ 234 */
238int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset, 235int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
239 void *p_data, unsigned int bytes) 236 void *p_data, unsigned int bytes)
240{ 237{
241 struct intel_vgpu *vgpu = __vgpu;
242 int ret; 238 int ret;
243 239
244 if (WARN_ON(bytes > 4)) 240 if (WARN_ON(bytes > 4))
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 0084ece8d8ff..d26a092c70e8 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -1418,8 +1418,8 @@ static int cmd_handler_mi_op_2e(struct parser_exec_state *s)
1418static int cmd_handler_mi_op_2f(struct parser_exec_state *s) 1418static int cmd_handler_mi_op_2f(struct parser_exec_state *s)
1419{ 1419{
1420 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1420 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
1421 int op_size = ((1 << (cmd_val(s, 0) & GENMASK(20, 19) >> 19)) * 1421 int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) *
1422 sizeof(u32)); 1422 sizeof(u32);
1423 unsigned long gma, gma_high; 1423 unsigned long gma, gma_high;
1424 int ret = 0; 1424 int ret = 0;
1425 1425
@@ -2537,7 +2537,8 @@ static int scan_workload(struct intel_vgpu_workload *workload)
2537 s.rb_va = workload->shadow_ring_buffer_va; 2537 s.rb_va = workload->shadow_ring_buffer_va;
2538 s.workload = workload; 2538 s.workload = workload;
2539 2539
2540 if (bypass_scan_mask & (1 << workload->ring_id)) 2540 if ((bypass_scan_mask & (1 << workload->ring_id)) ||
2541 gma_head == gma_tail)
2541 return 0; 2542 return 0;
2542 2543
2543 ret = ip_gma_set(&s, gma_head); 2544 ret = ip_gma_set(&s, gma_head);
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index 7e1da1c563ca..bda85dff7b2a 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -502,8 +502,7 @@ void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
502 * ACK of I2C_WRITE 502 * ACK of I2C_WRITE
503 * returned byte if it is READ 503 * returned byte if it is READ
504 */ 504 */
505 505 aux_data_for_write |= GVT_AUX_I2C_REPLY_ACK << 24;
506 aux_data_for_write |= (GVT_AUX_I2C_REPLY_ACK & 0xff) << 24;
507 vgpu_vreg(vgpu, offset + 4) = aux_data_for_write; 506 vgpu_vreg(vgpu, offset + 4) = aux_data_for_write;
508} 507}
509 508
diff --git a/drivers/gpu/drm/i915/gvt/edid.h b/drivers/gpu/drm/i915/gvt/edid.h
index de366b1d5196..f6dfc8b795ec 100644
--- a/drivers/gpu/drm/i915/gvt/edid.h
+++ b/drivers/gpu/drm/i915/gvt/edid.h
@@ -44,7 +44,7 @@
44#define GVT_AUX_I2C_READ 0x1 44#define GVT_AUX_I2C_READ 0x1
45#define GVT_AUX_I2C_STATUS 0x2 45#define GVT_AUX_I2C_STATUS 0x2
46#define GVT_AUX_I2C_MOT 0x4 46#define GVT_AUX_I2C_MOT 0x4
47#define GVT_AUX_I2C_REPLY_ACK (0x0 << 6) 47#define GVT_AUX_I2C_REPLY_ACK 0x0
48 48
49struct intel_vgpu_edid_data { 49struct intel_vgpu_edid_data {
50 bool data_valid; 50 bool data_valid;
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index c1f6019d8895..f32bb6f6495c 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -838,23 +838,21 @@ int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
838} 838}
839 839
840void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, 840void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
841 unsigned long ring_bitmap) 841 unsigned long engine_mask)
842{ 842{
843 int bit; 843 struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
844 struct list_head *pos, *n; 844 struct intel_engine_cs *engine;
845 struct intel_vgpu_workload *workload = NULL; 845 struct intel_vgpu_workload *pos, *n;
846 unsigned int tmp;
846 847
847 for_each_set_bit(bit, &ring_bitmap, sizeof(ring_bitmap) * 8) { 848 for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
848 if (bit >= I915_NUM_ENGINES)
849 break;
850 /* free the unsubmited workload in the queue */ 849 /* free the unsubmited workload in the queue */
851 list_for_each_safe(pos, n, &vgpu->workload_q_head[bit]) { 850 list_for_each_entry_safe(pos, n,
852 workload = container_of(pos, 851 &vgpu->workload_q_head[engine->id], list) {
853 struct intel_vgpu_workload, list); 852 list_del_init(&pos->list);
854 list_del_init(&workload->list); 853 free_workload(pos);
855 free_workload(workload);
856 } 854 }
857 855
858 init_vgpu_execlist(vgpu, bit); 856 init_vgpu_execlist(vgpu, engine->id);
859 } 857 }
860} 858}
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
index 635f31c6dcc1..7eced40a1e30 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.h
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -183,6 +183,6 @@ int intel_vgpu_init_execlist(struct intel_vgpu *vgpu);
183int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id); 183int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id);
184 184
185void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, 185void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
186 unsigned long ring_bitmap); 186 unsigned long engine_mask);
187 187
188#endif /*_GVT_EXECLIST_H_*/ 188#endif /*_GVT_EXECLIST_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 6554da9f9f5b..7eaaf1c9ed2b 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -138,36 +138,6 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
138 memcpy(&(e)->val64, &v, sizeof(v)); \ 138 memcpy(&(e)->val64, &v, sizeof(v)); \
139} while (0) 139} while (0)
140 140
141enum {
142 GTT_TYPE_INVALID = -1,
143
144 GTT_TYPE_GGTT_PTE,
145
146 GTT_TYPE_PPGTT_PTE_4K_ENTRY,
147 GTT_TYPE_PPGTT_PTE_2M_ENTRY,
148 GTT_TYPE_PPGTT_PTE_1G_ENTRY,
149
150 GTT_TYPE_PPGTT_PTE_ENTRY,
151
152 GTT_TYPE_PPGTT_PDE_ENTRY,
153 GTT_TYPE_PPGTT_PDP_ENTRY,
154 GTT_TYPE_PPGTT_PML4_ENTRY,
155
156 GTT_TYPE_PPGTT_ROOT_ENTRY,
157
158 GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
159 GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
160
161 GTT_TYPE_PPGTT_ENTRY,
162
163 GTT_TYPE_PPGTT_PTE_PT,
164 GTT_TYPE_PPGTT_PDE_PT,
165 GTT_TYPE_PPGTT_PDP_PT,
166 GTT_TYPE_PPGTT_PML4_PT,
167
168 GTT_TYPE_MAX,
169};
170
171/* 141/*
172 * Mappings between GTT_TYPE* enumerations. 142 * Mappings between GTT_TYPE* enumerations.
173 * Following information can be found according to the given type: 143 * Following information can be found according to the given type:
@@ -842,13 +812,18 @@ static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu,
842{ 812{
843 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 813 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
844 struct intel_vgpu_ppgtt_spt *s; 814 struct intel_vgpu_ppgtt_spt *s;
815 intel_gvt_gtt_type_t cur_pt_type;
845 816
846 if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type)))) 817 if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type))))
847 return -EINVAL; 818 return -EINVAL;
848 819
849 if (ops->get_pfn(e) == vgpu->gtt.scratch_page_mfn) 820 if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY
850 return 0; 821 && e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
851 822 cur_pt_type = get_next_pt_type(e->type) + 1;
823 if (ops->get_pfn(e) ==
824 vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)
825 return 0;
826 }
852 s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e)); 827 s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
853 if (!s) { 828 if (!s) {
854 gvt_err("vgpu%d: fail to find shadow page: mfn: 0x%lx\n", 829 gvt_err("vgpu%d: fail to find shadow page: mfn: 0x%lx\n",
@@ -999,7 +974,7 @@ fail:
999} 974}
1000 975
1001static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt, 976static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
1002 struct intel_gvt_gtt_entry *we, unsigned long index) 977 unsigned long index)
1003{ 978{
1004 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt); 979 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1005 struct intel_vgpu_shadow_page *sp = &spt->shadow_page; 980 struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
@@ -1008,34 +983,35 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
1008 struct intel_gvt_gtt_entry e; 983 struct intel_gvt_gtt_entry e;
1009 int ret; 984 int ret;
1010 985
1011 trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type,
1012 we->val64, index);
1013
1014 ppgtt_get_shadow_entry(spt, &e, index); 986 ppgtt_get_shadow_entry(spt, &e, index);
987
988 trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, e.val64,
989 index);
990
1015 if (!ops->test_present(&e)) 991 if (!ops->test_present(&e))
1016 return 0; 992 return 0;
1017 993
1018 if (ops->get_pfn(&e) == vgpu->gtt.scratch_page_mfn) 994 if (ops->get_pfn(&e) == vgpu->gtt.scratch_pt[sp->type].page_mfn)
1019 return 0; 995 return 0;
1020 996
1021 if (gtt_type_is_pt(get_next_pt_type(we->type))) { 997 if (gtt_type_is_pt(get_next_pt_type(e.type))) {
1022 struct intel_vgpu_guest_page *g = 998 struct intel_vgpu_ppgtt_spt *s =
1023 intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we)); 999 ppgtt_find_shadow_page(vgpu, ops->get_pfn(&e));
1024 if (!g) { 1000 if (!s) {
1025 gvt_err("fail to find guest page\n"); 1001 gvt_err("fail to find guest page\n");
1026 ret = -ENXIO; 1002 ret = -ENXIO;
1027 goto fail; 1003 goto fail;
1028 } 1004 }
1029 ret = ppgtt_invalidate_shadow_page(guest_page_to_ppgtt_spt(g)); 1005 ret = ppgtt_invalidate_shadow_page(s);
1030 if (ret) 1006 if (ret)
1031 goto fail; 1007 goto fail;
1032 } 1008 }
1033 ops->set_pfn(&e, vgpu->gtt.scratch_page_mfn); 1009 ops->set_pfn(&e, vgpu->gtt.scratch_pt[sp->type].page_mfn);
1034 ppgtt_set_shadow_entry(spt, &e, index); 1010 ppgtt_set_shadow_entry(spt, &e, index);
1035 return 0; 1011 return 0;
1036fail: 1012fail:
1037 gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n", 1013 gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
1038 vgpu->id, spt, we->val64, we->type); 1014 vgpu->id, spt, e.val64, e.type);
1039 return ret; 1015 return ret;
1040} 1016}
1041 1017
@@ -1256,23 +1232,16 @@ static int ppgtt_handle_guest_write_page_table(
1256 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt); 1232 struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1257 struct intel_vgpu *vgpu = spt->vgpu; 1233 struct intel_vgpu *vgpu = spt->vgpu;
1258 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops; 1234 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1259 struct intel_gvt_gtt_entry ge;
1260 1235
1261 int old_present, new_present;
1262 int ret; 1236 int ret;
1237 int new_present;
1263 1238
1264 ppgtt_get_guest_entry(spt, &ge, index);
1265
1266 old_present = ops->test_present(&ge);
1267 new_present = ops->test_present(we); 1239 new_present = ops->test_present(we);
1268 1240
1269 ppgtt_set_guest_entry(spt, we, index); 1241 ret = ppgtt_handle_guest_entry_removal(gpt, index);
1242 if (ret)
1243 goto fail;
1270 1244
1271 if (old_present) {
1272 ret = ppgtt_handle_guest_entry_removal(gpt, &ge, index);
1273 if (ret)
1274 goto fail;
1275 }
1276 if (new_present) { 1245 if (new_present) {
1277 ret = ppgtt_handle_guest_entry_add(gpt, we, index); 1246 ret = ppgtt_handle_guest_entry_add(gpt, we, index);
1278 if (ret) 1247 if (ret)
@@ -1318,7 +1287,7 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
1318{ 1287{
1319 struct list_head *pos, *n; 1288 struct list_head *pos, *n;
1320 struct intel_vgpu_ppgtt_spt *spt; 1289 struct intel_vgpu_ppgtt_spt *spt;
1321 struct intel_gvt_gtt_entry ge, e; 1290 struct intel_gvt_gtt_entry ge;
1322 unsigned long index; 1291 unsigned long index;
1323 int ret; 1292 int ret;
1324 1293
@@ -1329,9 +1298,6 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
1329 for_each_set_bit(index, spt->post_shadow_bitmap, 1298 for_each_set_bit(index, spt->post_shadow_bitmap,
1330 GTT_ENTRY_NUM_IN_ONE_PAGE) { 1299 GTT_ENTRY_NUM_IN_ONE_PAGE) {
1331 ppgtt_get_guest_entry(spt, &ge, index); 1300 ppgtt_get_guest_entry(spt, &ge, index);
1332 e = ge;
1333 e.val64 = 0;
1334 ppgtt_set_guest_entry(spt, &e, index);
1335 1301
1336 ret = ppgtt_handle_guest_write_page_table( 1302 ret = ppgtt_handle_guest_write_page_table(
1337 &spt->guest_page, &ge, index); 1303 &spt->guest_page, &ge, index);
@@ -1359,8 +1325,6 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
1359 index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift; 1325 index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift;
1360 1326
1361 ppgtt_get_guest_entry(spt, &we, index); 1327 ppgtt_get_guest_entry(spt, &we, index);
1362 memcpy((void *)&we.val64 + (pa & (info->gtt_entry_size - 1)),
1363 p_data, bytes);
1364 1328
1365 ops->test_pse(&we); 1329 ops->test_pse(&we);
1366 1330
@@ -1369,19 +1333,13 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
1369 if (ret) 1333 if (ret)
1370 return ret; 1334 return ret;
1371 } else { 1335 } else {
1372 struct intel_gvt_gtt_entry ge;
1373
1374 ppgtt_get_guest_entry(spt, &ge, index);
1375
1376 if (!test_bit(index, spt->post_shadow_bitmap)) { 1336 if (!test_bit(index, spt->post_shadow_bitmap)) {
1377 ret = ppgtt_handle_guest_entry_removal(gpt, 1337 ret = ppgtt_handle_guest_entry_removal(gpt, index);
1378 &ge, index);
1379 if (ret) 1338 if (ret)
1380 return ret; 1339 return ret;
1381 } 1340 }
1382 1341
1383 ppgtt_set_post_shadow(spt, index); 1342 ppgtt_set_post_shadow(spt, index);
1384 ppgtt_set_guest_entry(spt, &we, index);
1385 } 1343 }
1386 1344
1387 if (!enable_out_of_sync) 1345 if (!enable_out_of_sync)
@@ -1921,47 +1879,101 @@ int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1921 return ret; 1879 return ret;
1922} 1880}
1923 1881
1924static int create_scratch_page(struct intel_vgpu *vgpu) 1882static int alloc_scratch_pages(struct intel_vgpu *vgpu,
1883 intel_gvt_gtt_type_t type)
1925{ 1884{
1926 struct intel_vgpu_gtt *gtt = &vgpu->gtt; 1885 struct intel_vgpu_gtt *gtt = &vgpu->gtt;
1927 void *p; 1886 struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1928 void *vaddr; 1887 int page_entry_num = GTT_PAGE_SIZE >>
1888 vgpu->gvt->device_info.gtt_entry_size_shift;
1889 struct page *scratch_pt;
1929 unsigned long mfn; 1890 unsigned long mfn;
1891 int i;
1892 void *p;
1893
1894 if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX))
1895 return -EINVAL;
1930 1896
1931 gtt->scratch_page = alloc_page(GFP_KERNEL); 1897 scratch_pt = alloc_page(GFP_KERNEL | GFP_ATOMIC | __GFP_ZERO);
1932 if (!gtt->scratch_page) { 1898 if (!scratch_pt) {
1933 gvt_err("Failed to allocate scratch page.\n"); 1899 gvt_err("fail to allocate scratch page\n");
1934 return -ENOMEM; 1900 return -ENOMEM;
1935 } 1901 }
1936 1902
1937 /* set to zero */ 1903 p = kmap_atomic(scratch_pt);
1938 p = kmap_atomic(gtt->scratch_page); 1904 mfn = intel_gvt_hypervisor_virt_to_mfn(p);
1939 memset(p, 0, PAGE_SIZE); 1905 if (mfn == INTEL_GVT_INVALID_ADDR) {
1906 gvt_err("fail to translate vaddr:0x%llx\n", (u64)p);
1907 kunmap_atomic(p);
1908 __free_page(scratch_pt);
1909 return -EFAULT;
1910 }
1911 gtt->scratch_pt[type].page_mfn = mfn;
1912 gtt->scratch_pt[type].page = scratch_pt;
1913 gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n",
1914 vgpu->id, type, mfn);
1915
1916 /* Build the tree by full filled the scratch pt with the entries which
1917 * point to the next level scratch pt or scratch page. The
1918 * scratch_pt[type] indicate the scratch pt/scratch page used by the
1919 * 'type' pt.
1920 * e.g. scratch_pt[GTT_TYPE_PPGTT_PDE_PT] is used by
1921 * GTT_TYPE_PPGTT_PDE_PT level pt, that means this scatch_pt it self
1922 * is GTT_TYPE_PPGTT_PTE_PT, and full filled by scratch page mfn.
1923 */
1924 if (type > GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX) {
1925 struct intel_gvt_gtt_entry se;
1926
1927 memset(&se, 0, sizeof(struct intel_gvt_gtt_entry));
1928 se.type = get_entry_type(type - 1);
1929 ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);
1930
1931 /* The entry parameters like present/writeable/cache type
1932 * set to the same as i915's scratch page tree.
1933 */
1934 se.val64 |= _PAGE_PRESENT | _PAGE_RW;
1935 if (type == GTT_TYPE_PPGTT_PDE_PT)
1936 se.val64 |= PPAT_CACHED_INDEX;
1937
1938 for (i = 0; i < page_entry_num; i++)
1939 ops->set_entry(p, &se, i, false, 0, vgpu);
1940 }
1941
1940 kunmap_atomic(p); 1942 kunmap_atomic(p);
1941 1943
1942 /* translate page to mfn */ 1944 return 0;
1943 vaddr = page_address(gtt->scratch_page); 1945}
1944 mfn = intel_gvt_hypervisor_virt_to_mfn(vaddr); 1946
1947static int release_scratch_page_tree(struct intel_vgpu *vgpu)
1948{
1949 int i;
1945 1950
1946 if (mfn == INTEL_GVT_INVALID_ADDR) { 1951 for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
1947 gvt_err("fail to translate vaddr: 0x%p\n", vaddr); 1952 if (vgpu->gtt.scratch_pt[i].page != NULL) {
1948 __free_page(gtt->scratch_page); 1953 __free_page(vgpu->gtt.scratch_pt[i].page);
1949 gtt->scratch_page = NULL; 1954 vgpu->gtt.scratch_pt[i].page = NULL;
1950 return -ENXIO; 1955 vgpu->gtt.scratch_pt[i].page_mfn = 0;
1956 }
1951 } 1957 }
1952 1958
1953 gtt->scratch_page_mfn = mfn;
1954 gvt_dbg_core("vgpu%d create scratch page: mfn=0x%lx\n", vgpu->id, mfn);
1955 return 0; 1959 return 0;
1956} 1960}
1957 1961
1958static void release_scratch_page(struct intel_vgpu *vgpu) 1962static int create_scratch_page_tree(struct intel_vgpu *vgpu)
1959{ 1963{
1960 if (vgpu->gtt.scratch_page != NULL) { 1964 int i, ret;
1961 __free_page(vgpu->gtt.scratch_page); 1965
1962 vgpu->gtt.scratch_page = NULL; 1966 for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
1963 vgpu->gtt.scratch_page_mfn = 0; 1967 ret = alloc_scratch_pages(vgpu, i);
1968 if (ret)
1969 goto err;
1964 } 1970 }
1971
1972 return 0;
1973
1974err:
1975 release_scratch_page_tree(vgpu);
1976 return ret;
1965} 1977}
1966 1978
1967/** 1979/**
@@ -1995,7 +2007,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
1995 2007
1996 gtt->ggtt_mm = ggtt_mm; 2008 gtt->ggtt_mm = ggtt_mm;
1997 2009
1998 return create_scratch_page(vgpu); 2010 return create_scratch_page_tree(vgpu);
1999} 2011}
2000 2012
2001/** 2013/**
@@ -2014,7 +2026,7 @@ void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
2014 struct intel_vgpu_mm *mm; 2026 struct intel_vgpu_mm *mm;
2015 2027
2016 ppgtt_free_all_shadow_page(vgpu); 2028 ppgtt_free_all_shadow_page(vgpu);
2017 release_scratch_page(vgpu); 2029 release_scratch_page_tree(vgpu);
2018 2030
2019 list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) { 2031 list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
2020 mm = container_of(pos, struct intel_vgpu_mm, list); 2032 mm = container_of(pos, struct intel_vgpu_mm, list);
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index e4dcde78f3f9..d250013bc37b 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -88,6 +88,36 @@ enum {
88 INTEL_GVT_MM_PPGTT, 88 INTEL_GVT_MM_PPGTT,
89}; 89};
90 90
91typedef enum {
92 GTT_TYPE_INVALID = -1,
93
94 GTT_TYPE_GGTT_PTE,
95
96 GTT_TYPE_PPGTT_PTE_4K_ENTRY,
97 GTT_TYPE_PPGTT_PTE_2M_ENTRY,
98 GTT_TYPE_PPGTT_PTE_1G_ENTRY,
99
100 GTT_TYPE_PPGTT_PTE_ENTRY,
101
102 GTT_TYPE_PPGTT_PDE_ENTRY,
103 GTT_TYPE_PPGTT_PDP_ENTRY,
104 GTT_TYPE_PPGTT_PML4_ENTRY,
105
106 GTT_TYPE_PPGTT_ROOT_ENTRY,
107
108 GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
109 GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
110
111 GTT_TYPE_PPGTT_ENTRY,
112
113 GTT_TYPE_PPGTT_PTE_PT,
114 GTT_TYPE_PPGTT_PDE_PT,
115 GTT_TYPE_PPGTT_PDP_PT,
116 GTT_TYPE_PPGTT_PML4_PT,
117
118 GTT_TYPE_MAX,
119} intel_gvt_gtt_type_t;
120
91struct intel_vgpu_mm { 121struct intel_vgpu_mm {
92 int type; 122 int type;
93 bool initialized; 123 bool initialized;
@@ -151,6 +181,12 @@ extern void intel_vgpu_destroy_mm(struct kref *mm_ref);
151 181
152struct intel_vgpu_guest_page; 182struct intel_vgpu_guest_page;
153 183
184struct intel_vgpu_scratch_pt {
185 struct page *page;
186 unsigned long page_mfn;
187};
188
189
154struct intel_vgpu_gtt { 190struct intel_vgpu_gtt {
155 struct intel_vgpu_mm *ggtt_mm; 191 struct intel_vgpu_mm *ggtt_mm;
156 unsigned long active_ppgtt_mm_bitmap; 192 unsigned long active_ppgtt_mm_bitmap;
@@ -160,8 +196,8 @@ struct intel_vgpu_gtt {
160 atomic_t n_write_protected_guest_page; 196 atomic_t n_write_protected_guest_page;
161 struct list_head oos_page_list_head; 197 struct list_head oos_page_list_head;
162 struct list_head post_shadow_list_head; 198 struct list_head post_shadow_list_head;
163 struct page *scratch_page; 199 struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX];
164 unsigned long scratch_page_mfn; 200
165}; 201};
166 202
167extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu); 203extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu);
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index 385969a89216..398877c3d2fd 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -44,11 +44,14 @@ static const char * const supported_hypervisors[] = {
44 [INTEL_GVT_HYPERVISOR_KVM] = "KVM", 44 [INTEL_GVT_HYPERVISOR_KVM] = "KVM",
45}; 45};
46 46
47struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops = { 47static const struct intel_gvt_ops intel_gvt_ops = {
48 .emulate_cfg_read = intel_vgpu_emulate_cfg_read, 48 .emulate_cfg_read = intel_vgpu_emulate_cfg_read,
49 .emulate_cfg_write = intel_vgpu_emulate_cfg_write, 49 .emulate_cfg_write = intel_vgpu_emulate_cfg_write,
50 .emulate_mmio_read = intel_vgpu_emulate_mmio_read, 50 .emulate_mmio_read = intel_vgpu_emulate_mmio_read,
51 .emulate_mmio_write = intel_vgpu_emulate_mmio_write, 51 .emulate_mmio_write = intel_vgpu_emulate_mmio_write,
52 .vgpu_create = intel_gvt_create_vgpu,
53 .vgpu_destroy = intel_gvt_destroy_vgpu,
54 .vgpu_reset = intel_gvt_reset_vgpu,
52}; 55};
53 56
54/** 57/**
@@ -81,10 +84,12 @@ int intel_gvt_init_host(void)
81 symbol_get(xengt_mpt), "xengt"); 84 symbol_get(xengt_mpt), "xengt");
82 intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN; 85 intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN;
83 } else { 86 } else {
87#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
84 /* not in Xen. Try KVMGT */ 88 /* not in Xen. Try KVMGT */
85 intel_gvt_host.mpt = try_then_request_module( 89 intel_gvt_host.mpt = try_then_request_module(
86 symbol_get(kvmgt_mpt), "kvm"); 90 symbol_get(kvmgt_mpt), "kvmgt");
87 intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM; 91 intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM;
92#endif
88 } 93 }
89 94
90 /* Fail to load MPT modules - bail out */ 95 /* Fail to load MPT modules - bail out */
@@ -193,6 +198,9 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
193 intel_gvt_clean_mmio_info(gvt); 198 intel_gvt_clean_mmio_info(gvt);
194 intel_gvt_free_firmware(gvt); 199 intel_gvt_free_firmware(gvt);
195 200
201 intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
202 intel_gvt_clean_vgpu_types(gvt);
203
196 kfree(dev_priv->gvt); 204 kfree(dev_priv->gvt);
197 dev_priv->gvt = NULL; 205 dev_priv->gvt = NULL;
198} 206}
@@ -270,10 +278,25 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
270 if (ret) 278 if (ret)
271 goto out_clean_cmd_parser; 279 goto out_clean_cmd_parser;
272 280
273 gvt_dbg_core("gvt device creation is done\n"); 281 ret = intel_gvt_init_vgpu_types(gvt);
282 if (ret)
283 goto out_clean_thread;
284
285 ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt,
286 &intel_gvt_ops);
287 if (ret) {
288 gvt_err("failed to register gvt-g host device: %d\n", ret);
289 goto out_clean_types;
290 }
291
292 gvt_dbg_core("gvt device initialization is done\n");
274 dev_priv->gvt = gvt; 293 dev_priv->gvt = gvt;
275 return 0; 294 return 0;
276 295
296out_clean_types:
297 intel_gvt_clean_vgpu_types(gvt);
298out_clean_thread:
299 clean_service_thread(gvt);
277out_clean_cmd_parser: 300out_clean_cmd_parser:
278 intel_gvt_clean_cmd_parser(gvt); 301 intel_gvt_clean_cmd_parser(gvt);
279out_clean_sched_policy: 302out_clean_sched_policy:
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 62fc9e3ac5c6..3d4223e8ebe3 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -161,6 +161,20 @@ struct intel_vgpu {
161 DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES); 161 DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
162 struct i915_gem_context *shadow_ctx; 162 struct i915_gem_context *shadow_ctx;
163 struct notifier_block shadow_ctx_notifier_block; 163 struct notifier_block shadow_ctx_notifier_block;
164
165#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
166 struct {
167 struct device *mdev;
168 struct vfio_region *region;
169 int num_regions;
170 struct eventfd_ctx *intx_trigger;
171 struct eventfd_ctx *msi_trigger;
172 struct rb_root cache;
173 struct mutex cache_lock;
174 void *vfio_group;
175 struct notifier_block iommu_notifier;
176 } vdev;
177#endif
164}; 178};
165 179
166struct intel_gvt_gm { 180struct intel_gvt_gm {
@@ -190,6 +204,16 @@ struct intel_gvt_opregion {
190 u32 opregion_pa; 204 u32 opregion_pa;
191}; 205};
192 206
207#define NR_MAX_INTEL_VGPU_TYPES 20
208struct intel_vgpu_type {
209 char name[16];
210 unsigned int max_instance;
211 unsigned int avail_instance;
212 unsigned int low_gm_size;
213 unsigned int high_gm_size;
214 unsigned int fence;
215};
216
193struct intel_gvt { 217struct intel_gvt {
194 struct mutex lock; 218 struct mutex lock;
195 struct drm_i915_private *dev_priv; 219 struct drm_i915_private *dev_priv;
@@ -205,6 +229,8 @@ struct intel_gvt {
205 struct intel_gvt_opregion opregion; 229 struct intel_gvt_opregion opregion;
206 struct intel_gvt_workload_scheduler scheduler; 230 struct intel_gvt_workload_scheduler scheduler;
207 DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS); 231 DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
232 struct intel_vgpu_type *types;
233 unsigned int num_types;
208 234
209 struct task_struct *service_thread; 235 struct task_struct *service_thread;
210 wait_queue_head_t service_thread_wq; 236 wait_queue_head_t service_thread_wq;
@@ -231,6 +257,14 @@ void intel_gvt_free_firmware(struct intel_gvt *gvt);
231int intel_gvt_load_firmware(struct intel_gvt *gvt); 257int intel_gvt_load_firmware(struct intel_gvt *gvt);
232 258
233/* Aperture/GM space definitions for GVT device */ 259/* Aperture/GM space definitions for GVT device */
260#define MB_TO_BYTES(mb) ((mb) << 20ULL)
261#define BYTES_TO_MB(b) ((b) >> 20ULL)
262
263#define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
264#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
265#define HOST_FENCE 4
266
267/* Aperture/GM space definitions for GVT device */
234#define gvt_aperture_sz(gvt) (gvt->dev_priv->ggtt.mappable_end) 268#define gvt_aperture_sz(gvt) (gvt->dev_priv->ggtt.mappable_end)
235#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base) 269#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base)
236 270
@@ -330,11 +364,14 @@ static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
330 } 364 }
331} 365}
332 366
333struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, 367int intel_gvt_init_vgpu_types(struct intel_gvt *gvt);
334 struct intel_vgpu_creation_params * 368void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt);
335 param);
336 369
370struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
371 struct intel_vgpu_type *type);
337void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu); 372void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
373void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
374
338 375
339/* validating GM functions */ 376/* validating GM functions */
340#define vgpu_gmadr_is_aperture(vgpu, gmadr) \ 377#define vgpu_gmadr_is_aperture(vgpu, gmadr) \
@@ -369,10 +406,10 @@ int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
369int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index, 406int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
370 unsigned long *g_index); 407 unsigned long *g_index);
371 408
372int intel_vgpu_emulate_cfg_read(void *__vgpu, unsigned int offset, 409int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
373 void *p_data, unsigned int bytes); 410 void *p_data, unsigned int bytes);
374 411
375int intel_vgpu_emulate_cfg_write(void *__vgpu, unsigned int offset, 412int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
376 void *p_data, unsigned int bytes); 413 void *p_data, unsigned int bytes);
377 414
378void intel_gvt_clean_opregion(struct intel_gvt *gvt); 415void intel_gvt_clean_opregion(struct intel_gvt *gvt);
@@ -385,6 +422,22 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
385int setup_vgpu_mmio(struct intel_vgpu *vgpu); 422int setup_vgpu_mmio(struct intel_vgpu *vgpu);
386void populate_pvinfo_page(struct intel_vgpu *vgpu); 423void populate_pvinfo_page(struct intel_vgpu *vgpu);
387 424
425struct intel_gvt_ops {
426 int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
427 unsigned int);
428 int (*emulate_cfg_write)(struct intel_vgpu *, unsigned int, void *,
429 unsigned int);
430 int (*emulate_mmio_read)(struct intel_vgpu *, u64, void *,
431 unsigned int);
432 int (*emulate_mmio_write)(struct intel_vgpu *, u64, void *,
433 unsigned int);
434 struct intel_vgpu *(*vgpu_create)(struct intel_gvt *,
435 struct intel_vgpu_type *);
436 void (*vgpu_destroy)(struct intel_vgpu *);
437 void (*vgpu_reset)(struct intel_vgpu *);
438};
439
440
388#include "mpt.h" 441#include "mpt.h"
389 442
390#endif 443#endif
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 9ab1f95dddc5..522809710312 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -1158,7 +1158,10 @@ static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu,
1158static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset, 1158static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset,
1159 void *p_data, unsigned int bytes) 1159 void *p_data, unsigned int bytes)
1160{ 1160{
1161 u32 mode = *(u32 *)p_data; 1161 u32 mode;
1162
1163 write_vreg(vgpu, offset, p_data, bytes);
1164 mode = vgpu_vreg(vgpu, offset);
1162 1165
1163 if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) { 1166 if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) {
1164 WARN_ONCE(1, "VM(%d): iGVT-g doesn't supporte GuC\n", 1167 WARN_ONCE(1, "VM(%d): iGVT-g doesn't supporte GuC\n",
@@ -1275,19 +1278,18 @@ static int skl_misc_ctl_write(struct intel_vgpu *vgpu, unsigned int offset,
1275 switch (offset) { 1278 switch (offset) {
1276 case 0x4ddc: 1279 case 0x4ddc:
1277 vgpu_vreg(vgpu, offset) = 0x8000003c; 1280 vgpu_vreg(vgpu, offset) = 0x8000003c;
1281 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl */
1282 I915_WRITE(reg, vgpu_vreg(vgpu, offset));
1278 break; 1283 break;
1279 case 0x42080: 1284 case 0x42080:
1280 vgpu_vreg(vgpu, offset) = 0x8000; 1285 vgpu_vreg(vgpu, offset) = 0x8000;
1286 /* WaCompressedResourceDisplayNewHashMode:skl */
1287 I915_WRITE(reg, vgpu_vreg(vgpu, offset));
1281 break; 1288 break;
1282 default: 1289 default:
1283 return -EINVAL; 1290 return -EINVAL;
1284 } 1291 }
1285 1292
1286 /**
1287 * TODO: need detect stepping info after gvt contain such information
1288 * 0x4ddc enabled after C0, 0x42080 enabled after E0.
1289 */
1290 I915_WRITE(reg, vgpu_vreg(vgpu, offset));
1291 return 0; 1293 return 0;
1292} 1294}
1293 1295
@@ -1367,6 +1369,9 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
1367 int rc = 0; 1369 int rc = 0;
1368 unsigned int id = 0; 1370 unsigned int id = 0;
1369 1371
1372 write_vreg(vgpu, offset, p_data, bytes);
1373 vgpu_vreg(vgpu, offset) = 0;
1374
1370 switch (offset) { 1375 switch (offset) {
1371 case 0x4260: 1376 case 0x4260:
1372 id = RCS; 1377 id = RCS;
@@ -1392,6 +1397,23 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
1392 return rc; 1397 return rc;
1393} 1398}
1394 1399
1400static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
1401 unsigned int offset, void *p_data, unsigned int bytes)
1402{
1403 u32 data;
1404
1405 write_vreg(vgpu, offset, p_data, bytes);
1406 data = vgpu_vreg(vgpu, offset);
1407
1408 if (data & _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET))
1409 data |= RESET_CTL_READY_TO_RESET;
1410 else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET))
1411 data &= ~RESET_CTL_READY_TO_RESET;
1412
1413 vgpu_vreg(vgpu, offset) = data;
1414 return 0;
1415}
1416
1395#define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ 1417#define MMIO_F(reg, s, f, am, rm, d, r, w) do { \
1396 ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \ 1418 ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \
1397 f, s, am, rm, d, r, w); \ 1419 f, s, am, rm, d, r, w); \
@@ -1485,7 +1507,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
1485 1507
1486 MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL); 1508 MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
1487 MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL); 1509 MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL);
1488 MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK, NULL, NULL); 1510 MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
1489 1511
1490 MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL); 1512 MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL);
1491 MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL); 1513 MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL);
@@ -1494,7 +1516,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
1494 MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL); 1516 MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL);
1495 MMIO_D(GAM_ECOCHK, D_ALL); 1517 MMIO_D(GAM_ECOCHK, D_ALL);
1496 MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL); 1518 MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL);
1497 MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK, NULL, NULL); 1519 MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
1498 MMIO_D(0x9030, D_ALL); 1520 MMIO_D(0x9030, D_ALL);
1499 MMIO_D(0x20a0, D_ALL); 1521 MMIO_D(0x20a0, D_ALL);
1500 MMIO_D(0x2420, D_ALL); 1522 MMIO_D(0x2420, D_ALL);
@@ -1503,7 +1525,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
1503 MMIO_D(0x2438, D_ALL); 1525 MMIO_D(0x2438, D_ALL);
1504 MMIO_D(0x243c, D_ALL); 1526 MMIO_D(0x243c, D_ALL);
1505 MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL); 1527 MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL);
1506 MMIO_DFH(0xe184, D_ALL, F_MODE_MASK, NULL, NULL); 1528 MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
1507 MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL); 1529 MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL);
1508 1530
1509 /* display */ 1531 /* display */
@@ -2116,6 +2138,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
2116 MMIO_D(GEN6_MBCTL, D_ALL); 2138 MMIO_D(GEN6_MBCTL, D_ALL);
2117 MMIO_D(0x911c, D_ALL); 2139 MMIO_D(0x911c, D_ALL);
2118 MMIO_D(0x9120, D_ALL); 2140 MMIO_D(0x9120, D_ALL);
2141 MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL);
2119 2142
2120 MMIO_D(GAB_CTL, D_ALL); 2143 MMIO_D(GAB_CTL, D_ALL);
2121 MMIO_D(0x48800, D_ALL); 2144 MMIO_D(0x48800, D_ALL);
@@ -2298,6 +2321,15 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
2298 2321
2299 MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS); 2322 MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS);
2300 2323
2324#define RING_REG(base) (base + 0xd0)
2325 MMIO_RING_F(RING_REG, 4, F_RO, 0,
2326 ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
2327 ring_reset_ctl_write);
2328 MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0,
2329 ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
2330 ring_reset_ctl_write);
2331#undef RING_REG
2332
2301#define RING_REG(base) (base + 0x230) 2333#define RING_REG(base) (base + 0x230)
2302 MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write); 2334 MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write);
2303 MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, elsp_mmio_write); 2335 MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, elsp_mmio_write);
@@ -2345,7 +2377,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
2345 MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL); 2377 MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
2346 MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL); 2378 MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL);
2347 2379
2348 MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); 2380 MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
2349 2381
2350 MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW); 2382 MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW);
2351 MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW); 2383 MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW);
@@ -2364,7 +2396,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
2364 MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS); 2396 MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS);
2365 2397
2366 MMIO_D(0xfdc, D_BDW); 2398 MMIO_D(0xfdc, D_BDW);
2367 MMIO_D(GEN8_ROW_CHICKEN, D_BDW_PLUS); 2399 MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
2368 MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS); 2400 MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS);
2369 MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS); 2401 MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS);
2370 2402
@@ -2375,10 +2407,10 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
2375 MMIO_D(0xb10c, D_BDW); 2407 MMIO_D(0xb10c, D_BDW);
2376 MMIO_D(0xb110, D_BDW); 2408 MMIO_D(0xb110, D_BDW);
2377 2409
2378 MMIO_DH(0x24d0, D_BDW_PLUS, NULL, NULL); 2410 MMIO_DFH(0x24d0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
2379 MMIO_DH(0x24d4, D_BDW_PLUS, NULL, NULL); 2411 MMIO_DFH(0x24d4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
2380 MMIO_DH(0x24d8, D_BDW_PLUS, NULL, NULL); 2412 MMIO_DFH(0x24d8, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
2381 MMIO_DH(0x24dc, D_BDW_PLUS, NULL, NULL); 2413 MMIO_DFH(0x24dc, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
2382 2414
2383 MMIO_D(0x83a4, D_BDW); 2415 MMIO_D(0x83a4, D_BDW);
2384 MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS); 2416 MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS);
@@ -2392,9 +2424,9 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
2392 MMIO_D(0x6e570, D_BDW_PLUS); 2424 MMIO_D(0x6e570, D_BDW_PLUS);
2393 MMIO_D(0x65f10, D_BDW_PLUS); 2425 MMIO_D(0x65f10, D_BDW_PLUS);
2394 2426
2395 MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); 2427 MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
2396 MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); 2428 MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
2397 MMIO_DFH(0xe180, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); 2429 MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
2398 MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); 2430 MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
2399 2431
2400 MMIO_D(0x2248, D_BDW); 2432 MMIO_D(0x2248, D_BDW);
@@ -2425,6 +2457,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
2425 MMIO_D(0xa210, D_SKL_PLUS); 2457 MMIO_D(0xa210, D_SKL_PLUS);
2426 MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS); 2458 MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
2427 MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS); 2459 MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
2460 MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
2428 MMIO_DH(0x4ddc, D_SKL, NULL, skl_misc_ctl_write); 2461 MMIO_DH(0x4ddc, D_SKL, NULL, skl_misc_ctl_write);
2429 MMIO_DH(0x42080, D_SKL, NULL, skl_misc_ctl_write); 2462 MMIO_DH(0x42080, D_SKL, NULL, skl_misc_ctl_write);
2430 MMIO_D(0x45504, D_SKL); 2463 MMIO_D(0x45504, D_SKL);
@@ -2574,8 +2607,8 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
2574 MMIO_D(0x51000, D_SKL); 2607 MMIO_D(0x51000, D_SKL);
2575 MMIO_D(0x6c00c, D_SKL); 2608 MMIO_D(0x6c00c, D_SKL);
2576 2609
2577 MMIO_F(0xc800, 0x7f8, 0, 0, 0, D_SKL, NULL, NULL); 2610 MMIO_F(0xc800, 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL);
2578 MMIO_F(0xb020, 0x80, 0, 0, 0, D_SKL, NULL, NULL); 2611 MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL);
2579 2612
2580 MMIO_D(0xd08, D_SKL); 2613 MMIO_D(0xd08, D_SKL);
2581 MMIO_D(0x20e0, D_SKL); 2614 MMIO_D(0x20e0, D_SKL);
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index 027ef558d91c..30e543f5a703 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -33,21 +33,14 @@
33#ifndef _GVT_HYPERCALL_H_ 33#ifndef _GVT_HYPERCALL_H_
34#define _GVT_HYPERCALL_H_ 34#define _GVT_HYPERCALL_H_
35 35
36struct intel_gvt_io_emulation_ops {
37 int (*emulate_cfg_read)(void *, unsigned int, void *, unsigned int);
38 int (*emulate_cfg_write)(void *, unsigned int, void *, unsigned int);
39 int (*emulate_mmio_read)(void *, u64, void *, unsigned int);
40 int (*emulate_mmio_write)(void *, u64, void *, unsigned int);
41};
42
43extern struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops;
44
45/* 36/*
46 * Specific GVT-g MPT modules function collections. Currently GVT-g supports 37 * Specific GVT-g MPT modules function collections. Currently GVT-g supports
47 * both Xen and KVM by providing dedicated hypervisor-related MPT modules. 38 * both Xen and KVM by providing dedicated hypervisor-related MPT modules.
48 */ 39 */
49struct intel_gvt_mpt { 40struct intel_gvt_mpt {
50 int (*detect_host)(void); 41 int (*detect_host)(void);
42 int (*host_init)(struct device *dev, void *gvt, const void *ops);
43 void (*host_exit)(struct device *dev, void *gvt);
51 int (*attach_vgpu)(void *vgpu, unsigned long *handle); 44 int (*attach_vgpu)(void *vgpu, unsigned long *handle);
52 void (*detach_vgpu)(unsigned long handle); 45 void (*detach_vgpu)(unsigned long handle);
53 int (*inject_msi)(unsigned long handle, u32 addr, u16 data); 46 int (*inject_msi)(unsigned long handle, u32 addr, u16 data);
@@ -60,8 +53,7 @@ struct intel_gvt_mpt {
60 unsigned long len); 53 unsigned long len);
61 unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn); 54 unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn);
62 int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn, 55 int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn,
63 unsigned long mfn, unsigned int nr, bool map, 56 unsigned long mfn, unsigned int nr, bool map);
64 int type);
65 int (*set_trap_area)(unsigned long handle, u64 start, u64 end, 57 int (*set_trap_area)(unsigned long handle, u64 start, u64 end,
66 bool map); 58 bool map);
67}; 59};
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
new file mode 100644
index 000000000000..dc0365033157
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -0,0 +1,597 @@
1/*
2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM
3 *
4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Kevin Tian <kevin.tian@intel.com>
27 * Jike Song <jike.song@intel.com>
28 * Xiaoguang Chen <xiaoguang.chen@intel.com>
29 */
30
31#include <linux/init.h>
32#include <linux/device.h>
33#include <linux/mm.h>
34#include <linux/types.h>
35#include <linux/list.h>
36#include <linux/rbtree.h>
37#include <linux/spinlock.h>
38#include <linux/eventfd.h>
39#include <linux/uuid.h>
40#include <linux/kvm_host.h>
41#include <linux/vfio.h>
42
43#include "i915_drv.h"
44#include "gvt.h"
45
46static inline long kvmgt_pin_pages(struct device *dev, unsigned long *user_pfn,
47 long npage, int prot, unsigned long *phys_pfn)
48{
49 return 0;
50}
51static inline long kvmgt_unpin_pages(struct device *dev, unsigned long *pfn,
52 long npage)
53{
54 return 0;
55}
56
57static const struct intel_gvt_ops *intel_gvt_ops;
58
59
60/* helper macros copied from vfio-pci */
61#define VFIO_PCI_OFFSET_SHIFT 40
62#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
63#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
64#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
65
66struct vfio_region {
67 u32 type;
68 u32 subtype;
69 size_t size;
70 u32 flags;
71};
72
73struct kvmgt_pgfn {
74 gfn_t gfn;
75 struct hlist_node hnode;
76};
77
78struct kvmgt_guest_info {
79 struct kvm *kvm;
80 struct intel_vgpu *vgpu;
81 struct kvm_page_track_notifier_node track_node;
82#define NR_BKT (1 << 18)
83 struct hlist_head ptable[NR_BKT];
84#undef NR_BKT
85};
86
87struct gvt_dma {
88 struct rb_node node;
89 gfn_t gfn;
90 kvm_pfn_t pfn;
91};
92
93static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
94{
95 struct rb_node *node = vgpu->vdev.cache.rb_node;
96 struct gvt_dma *ret = NULL;
97
98 while (node) {
99 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node);
100
101 if (gfn < itr->gfn)
102 node = node->rb_left;
103 else if (gfn > itr->gfn)
104 node = node->rb_right;
105 else {
106 ret = itr;
107 goto out;
108 }
109 }
110
111out:
112 return ret;
113}
114
115static kvm_pfn_t gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
116{
117 struct gvt_dma *entry;
118
119 mutex_lock(&vgpu->vdev.cache_lock);
120 entry = __gvt_cache_find(vgpu, gfn);
121 mutex_unlock(&vgpu->vdev.cache_lock);
122
123 return entry == NULL ? 0 : entry->pfn;
124}
125
126static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, kvm_pfn_t pfn)
127{
128 struct gvt_dma *new, *itr;
129 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL;
130
131 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
132 if (!new)
133 return;
134
135 new->gfn = gfn;
136 new->pfn = pfn;
137
138 mutex_lock(&vgpu->vdev.cache_lock);
139 while (*link) {
140 parent = *link;
141 itr = rb_entry(parent, struct gvt_dma, node);
142
143 if (gfn == itr->gfn)
144 goto out;
145 else if (gfn < itr->gfn)
146 link = &parent->rb_left;
147 else
148 link = &parent->rb_right;
149 }
150
151 rb_link_node(&new->node, parent, link);
152 rb_insert_color(&new->node, &vgpu->vdev.cache);
153 mutex_unlock(&vgpu->vdev.cache_lock);
154 return;
155
156out:
157 mutex_unlock(&vgpu->vdev.cache_lock);
158 kfree(new);
159}
160
161static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
162 struct gvt_dma *entry)
163{
164 rb_erase(&entry->node, &vgpu->vdev.cache);
165 kfree(entry);
166}
167
168static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn)
169{
170 struct device *dev = vgpu->vdev.mdev;
171 struct gvt_dma *this;
172 unsigned long pfn;
173
174 mutex_lock(&vgpu->vdev.cache_lock);
175 this = __gvt_cache_find(vgpu, gfn);
176 if (!this) {
177 mutex_unlock(&vgpu->vdev.cache_lock);
178 return;
179 }
180
181 pfn = this->pfn;
182 WARN_ON((kvmgt_unpin_pages(dev, &pfn, 1) != 1));
183 __gvt_cache_remove_entry(vgpu, this);
184 mutex_unlock(&vgpu->vdev.cache_lock);
185}
186
187static void gvt_cache_init(struct intel_vgpu *vgpu)
188{
189 vgpu->vdev.cache = RB_ROOT;
190 mutex_init(&vgpu->vdev.cache_lock);
191}
192
193static void gvt_cache_destroy(struct intel_vgpu *vgpu)
194{
195 struct gvt_dma *dma;
196 struct rb_node *node = NULL;
197 struct device *dev = vgpu->vdev.mdev;
198 unsigned long pfn;
199
200 mutex_lock(&vgpu->vdev.cache_lock);
201 while ((node = rb_first(&vgpu->vdev.cache))) {
202 dma = rb_entry(node, struct gvt_dma, node);
203 pfn = dma->pfn;
204
205 kvmgt_unpin_pages(dev, &pfn, 1);
206 __gvt_cache_remove_entry(vgpu, dma);
207 }
208 mutex_unlock(&vgpu->vdev.cache_lock);
209}
210
211static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
212 const char *name)
213{
214 int i;
215 struct intel_vgpu_type *t;
216 const char *driver_name = dev_driver_string(
217 &gvt->dev_priv->drm.pdev->dev);
218
219 for (i = 0; i < gvt->num_types; i++) {
220 t = &gvt->types[i];
221 if (!strncmp(t->name, name + strlen(driver_name) + 1,
222 sizeof(t->name)))
223 return t;
224 }
225
226 return NULL;
227}
228
229static struct attribute *type_attrs[] = {
230 NULL,
231};
232
233static struct attribute_group *intel_vgpu_type_groups[] = {
234 [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
235};
236
237static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
238{
239 int i, j;
240 struct intel_vgpu_type *type;
241 struct attribute_group *group;
242
243 for (i = 0; i < gvt->num_types; i++) {
244 type = &gvt->types[i];
245
246 group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
247 if (WARN_ON(!group))
248 goto unwind;
249
250 group->name = type->name;
251 group->attrs = type_attrs;
252 intel_vgpu_type_groups[i] = group;
253 }
254
255 return true;
256
257unwind:
258 for (j = 0; j < i; j++) {
259 group = intel_vgpu_type_groups[j];
260 kfree(group);
261 }
262
263 return false;
264}
265
266static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
267{
268 int i;
269 struct attribute_group *group;
270
271 for (i = 0; i < gvt->num_types; i++) {
272 group = intel_vgpu_type_groups[i];
273 kfree(group);
274 }
275}
276
277static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
278{
279 hash_init(info->ptable);
280}
281
282static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
283{
284 struct kvmgt_pgfn *p;
285 struct hlist_node *tmp;
286 int i;
287
288 hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
289 hash_del(&p->hnode);
290 kfree(p);
291 }
292}
293
294static struct kvmgt_pgfn *
295__kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
296{
297 struct kvmgt_pgfn *p, *res = NULL;
298
299 hash_for_each_possible(info->ptable, p, hnode, gfn) {
300 if (gfn == p->gfn) {
301 res = p;
302 break;
303 }
304 }
305
306 return res;
307}
308
309static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
310 gfn_t gfn)
311{
312 struct kvmgt_pgfn *p;
313
314 p = __kvmgt_protect_table_find(info, gfn);
315 return !!p;
316}
317
318static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
319{
320 struct kvmgt_pgfn *p;
321
322 if (kvmgt_gfn_is_write_protected(info, gfn))
323 return;
324
325 p = kmalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
326 if (WARN(!p, "gfn: 0x%llx\n", gfn))
327 return;
328
329 p->gfn = gfn;
330 hash_add(info->ptable, &p->hnode, gfn);
331}
332
333static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
334 gfn_t gfn)
335{
336 struct kvmgt_pgfn *p;
337
338 p = __kvmgt_protect_table_find(info, gfn);
339 if (p) {
340 hash_del(&p->hnode);
341 kfree(p);
342 }
343}
344
345static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
346{
347 if (!intel_gvt_init_vgpu_type_groups(gvt))
348 return -EFAULT;
349
350 intel_gvt_ops = ops;
351
352 /* MDEV is not yet available */
353 return -ENODEV;
354}
355
356static void kvmgt_host_exit(struct device *dev, void *gvt)
357{
358 intel_gvt_cleanup_vgpu_type_groups(gvt);
359}
360
361static int kvmgt_write_protect_add(unsigned long handle, u64 gfn)
362{
363 struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
364 struct kvm *kvm = info->kvm;
365 struct kvm_memory_slot *slot;
366 int idx;
367
368 idx = srcu_read_lock(&kvm->srcu);
369 slot = gfn_to_memslot(kvm, gfn);
370
371 spin_lock(&kvm->mmu_lock);
372
373 if (kvmgt_gfn_is_write_protected(info, gfn))
374 goto out;
375
376 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
377 kvmgt_protect_table_add(info, gfn);
378
379out:
380 spin_unlock(&kvm->mmu_lock);
381 srcu_read_unlock(&kvm->srcu, idx);
382 return 0;
383}
384
385static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn)
386{
387 struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
388 struct kvm *kvm = info->kvm;
389 struct kvm_memory_slot *slot;
390 int idx;
391
392 idx = srcu_read_lock(&kvm->srcu);
393 slot = gfn_to_memslot(kvm, gfn);
394
395 spin_lock(&kvm->mmu_lock);
396
397 if (!kvmgt_gfn_is_write_protected(info, gfn))
398 goto out;
399
400 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
401 kvmgt_protect_table_del(info, gfn);
402
403out:
404 spin_unlock(&kvm->mmu_lock);
405 srcu_read_unlock(&kvm->srcu, idx);
406 return 0;
407}
408
409static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
410 const u8 *val, int len,
411 struct kvm_page_track_notifier_node *node)
412{
413 struct kvmgt_guest_info *info = container_of(node,
414 struct kvmgt_guest_info, track_node);
415
416 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
417 intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa,
418 (void *)val, len);
419}
420
421static void kvmgt_page_track_flush_slot(struct kvm *kvm,
422 struct kvm_memory_slot *slot,
423 struct kvm_page_track_notifier_node *node)
424{
425 int i;
426 gfn_t gfn;
427 struct kvmgt_guest_info *info = container_of(node,
428 struct kvmgt_guest_info, track_node);
429
430 spin_lock(&kvm->mmu_lock);
431 for (i = 0; i < slot->npages; i++) {
432 gfn = slot->base_gfn + i;
433 if (kvmgt_gfn_is_write_protected(info, gfn)) {
434 kvm_slot_page_track_remove_page(kvm, slot, gfn,
435 KVM_PAGE_TRACK_WRITE);
436 kvmgt_protect_table_del(info, gfn);
437 }
438 }
439 spin_unlock(&kvm->mmu_lock);
440}
441
442static bool kvmgt_check_guest(void)
443{
444 unsigned int eax, ebx, ecx, edx;
445 char s[12];
446 unsigned int *i;
447
448 eax = KVM_CPUID_SIGNATURE;
449 ebx = ecx = edx = 0;
450
451 asm volatile ("cpuid"
452 : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
453 :
454 : "cc", "memory");
455 i = (unsigned int *)s;
456 i[0] = ebx;
457 i[1] = ecx;
458 i[2] = edx;
459
460 return !strncmp(s, "KVMKVMKVM", strlen("KVMKVMKVM"));
461}
462
463/**
464 * NOTE:
465 * It's actually impossible to check if we are running in KVM host,
466 * since the "KVM host" is simply native. So we only dectect guest here.
467 */
468static int kvmgt_detect_host(void)
469{
470#ifdef CONFIG_INTEL_IOMMU
471 if (intel_iommu_gfx_mapped) {
472 gvt_err("Hardware IOMMU compatibility not yet supported, try to boot with intel_iommu=igfx_off\n");
473 return -ENODEV;
474 }
475#endif
476 return kvmgt_check_guest() ? -ENODEV : 0;
477}
478
479static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
480{
481 /* nothing to do here */
482 return 0;
483}
484
485static void kvmgt_detach_vgpu(unsigned long handle)
486{
487 /* nothing to do here */
488}
489
490static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
491{
492 struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
493 struct intel_vgpu *vgpu = info->vgpu;
494
495 if (vgpu->vdev.msi_trigger)
496 return eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1;
497
498 return false;
499}
500
501static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
502{
503 unsigned long pfn;
504 struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
505 int rc;
506
507 pfn = gvt_cache_find(info->vgpu, gfn);
508 if (pfn != 0)
509 return pfn;
510
511 rc = kvmgt_pin_pages(info->vgpu->vdev.mdev, &gfn, 1,
512 IOMMU_READ | IOMMU_WRITE, &pfn);
513 if (rc != 1) {
514 gvt_err("vfio_pin_pages failed for gfn: 0x%lx\n", gfn);
515 return 0;
516 }
517
518 gvt_cache_add(info->vgpu, gfn, pfn);
519 return pfn;
520}
521
522static void *kvmgt_gpa_to_hva(unsigned long handle, unsigned long gpa)
523{
524 unsigned long pfn;
525 gfn_t gfn = gpa_to_gfn(gpa);
526
527 pfn = kvmgt_gfn_to_pfn(handle, gfn);
528 if (!pfn)
529 return NULL;
530
531 return (char *)pfn_to_kaddr(pfn) + offset_in_page(gpa);
532}
533
534static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
535 void *buf, unsigned long len, bool write)
536{
537 void *hva = NULL;
538
539 hva = kvmgt_gpa_to_hva(handle, gpa);
540 if (!hva)
541 return -EFAULT;
542
543 if (write)
544 memcpy(hva, buf, len);
545 else
546 memcpy(buf, hva, len);
547
548 return 0;
549}
550
551static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
552 void *buf, unsigned long len)
553{
554 return kvmgt_rw_gpa(handle, gpa, buf, len, false);
555}
556
557static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
558 void *buf, unsigned long len)
559{
560 return kvmgt_rw_gpa(handle, gpa, buf, len, true);
561}
562
563static unsigned long kvmgt_virt_to_pfn(void *addr)
564{
565 return PFN_DOWN(__pa(addr));
566}
567
568struct intel_gvt_mpt kvmgt_mpt = {
569 .detect_host = kvmgt_detect_host,
570 .host_init = kvmgt_host_init,
571 .host_exit = kvmgt_host_exit,
572 .attach_vgpu = kvmgt_attach_vgpu,
573 .detach_vgpu = kvmgt_detach_vgpu,
574 .inject_msi = kvmgt_inject_msi,
575 .from_virt_to_mfn = kvmgt_virt_to_pfn,
576 .set_wp_page = kvmgt_write_protect_add,
577 .unset_wp_page = kvmgt_write_protect_remove,
578 .read_gpa = kvmgt_read_gpa,
579 .write_gpa = kvmgt_write_gpa,
580 .gfn_to_mfn = kvmgt_gfn_to_pfn,
581};
582EXPORT_SYMBOL_GPL(kvmgt_mpt);
583
584static int __init kvmgt_init(void)
585{
586 return 0;
587}
588
589static void __exit kvmgt_exit(void)
590{
591}
592
593module_init(kvmgt_init);
594module_exit(kvmgt_exit);
595
596MODULE_LICENSE("GPL and additional rights");
597MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 585b01f63254..09c9450a1946 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -67,10 +67,9 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
67 * Returns: 67 * Returns:
68 * Zero on success, negative error code if failed 68 * Zero on success, negative error code if failed
69 */ 69 */
70int intel_vgpu_emulate_mmio_read(void *__vgpu, uint64_t pa, 70int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
71 void *p_data, unsigned int bytes) 71 void *p_data, unsigned int bytes)
72{ 72{
73 struct intel_vgpu *vgpu = __vgpu;
74 struct intel_gvt *gvt = vgpu->gvt; 73 struct intel_gvt *gvt = vgpu->gvt;
75 struct intel_gvt_mmio_info *mmio; 74 struct intel_gvt_mmio_info *mmio;
76 unsigned int offset = 0; 75 unsigned int offset = 0;
@@ -179,10 +178,9 @@ err:
179 * Returns: 178 * Returns:
180 * Zero on success, negative error code if failed 179 * Zero on success, negative error code if failed
181 */ 180 */
182int intel_vgpu_emulate_mmio_write(void *__vgpu, uint64_t pa, 181int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
183 void *p_data, unsigned int bytes) 182 void *p_data, unsigned int bytes)
184{ 183{
185 struct intel_vgpu *vgpu = __vgpu;
186 struct intel_gvt *gvt = vgpu->gvt; 184 struct intel_gvt *gvt = vgpu->gvt;
187 struct intel_gvt_mmio_info *mmio; 185 struct intel_gvt_mmio_info *mmio;
188 unsigned int offset = 0; 186 unsigned int offset = 0;
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
index 9dc739a01892..87d5b5e366a3 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.h
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -87,10 +87,11 @@ struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
87}) 87})
88 88
89int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa); 89int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa);
90int intel_vgpu_emulate_mmio_read(void *__vgpu, u64 pa, void *p_data, 90
91 unsigned int bytes); 91int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa,
92int intel_vgpu_emulate_mmio_write(void *__vgpu, u64 pa, void *p_data, 92 void *p_data, unsigned int bytes);
93 unsigned int bytes); 93int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa,
94 void *p_data, unsigned int bytes);
94bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt, 95bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
95 unsigned int offset); 96 unsigned int offset);
96bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset); 97bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset);
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 67858782d327..1af5830c0a56 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -56,6 +56,35 @@ static inline int intel_gvt_hypervisor_detect_host(void)
56} 56}
57 57
58/** 58/**
59 * intel_gvt_hypervisor_host_init - init GVT-g host side
60 *
61 * Returns:
62 * Zero on success, negative error code if failed
63 */
64static inline int intel_gvt_hypervisor_host_init(struct device *dev,
65 void *gvt, const void *ops)
66{
67 /* optional to provide */
68 if (!intel_gvt_host.mpt->host_init)
69 return 0;
70
71 return intel_gvt_host.mpt->host_init(dev, gvt, ops);
72}
73
74/**
75 * intel_gvt_hypervisor_host_exit - exit GVT-g host side
76 */
77static inline void intel_gvt_hypervisor_host_exit(struct device *dev,
78 void *gvt)
79{
80 /* optional to provide */
81 if (!intel_gvt_host.mpt->host_exit)
82 return;
83
84 intel_gvt_host.mpt->host_exit(dev, gvt);
85}
86
87/**
59 * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU 88 * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU
60 * related stuffs inside hypervisor. 89 * related stuffs inside hypervisor.
61 * 90 *
@@ -64,6 +93,10 @@ static inline int intel_gvt_hypervisor_detect_host(void)
64 */ 93 */
65static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu) 94static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu)
66{ 95{
96 /* optional to provide */
97 if (!intel_gvt_host.mpt->attach_vgpu)
98 return 0;
99
67 return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle); 100 return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle);
68} 101}
69 102
@@ -76,6 +109,10 @@ static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu)
76 */ 109 */
77static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu) 110static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu)
78{ 111{
112 /* optional to provide */
113 if (!intel_gvt_host.mpt->detach_vgpu)
114 return;
115
79 intel_gvt_host.mpt->detach_vgpu(vgpu->handle); 116 intel_gvt_host.mpt->detach_vgpu(vgpu->handle);
80} 117}
81 118
@@ -224,11 +261,6 @@ static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn(
224 return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn); 261 return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn);
225} 262}
226 263
227enum {
228 GVT_MAP_APERTURE = 0,
229 GVT_MAP_OPREGION,
230};
231
232/** 264/**
233 * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN 265 * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN
234 * @vgpu: a vGPU 266 * @vgpu: a vGPU
@@ -236,7 +268,6 @@ enum {
236 * @mfn: host PFN 268 * @mfn: host PFN
237 * @nr: amount of PFNs 269 * @nr: amount of PFNs
238 * @map: map or unmap 270 * @map: map or unmap
239 * @type: map type
240 * 271 *
241 * Returns: 272 * Returns:
242 * Zero on success, negative error code if failed. 273 * Zero on success, negative error code if failed.
@@ -244,10 +275,14 @@ enum {
244static inline int intel_gvt_hypervisor_map_gfn_to_mfn( 275static inline int intel_gvt_hypervisor_map_gfn_to_mfn(
245 struct intel_vgpu *vgpu, unsigned long gfn, 276 struct intel_vgpu *vgpu, unsigned long gfn,
246 unsigned long mfn, unsigned int nr, 277 unsigned long mfn, unsigned int nr,
247 bool map, int type) 278 bool map)
248{ 279{
280 /* a MPT implementation could have MMIO mapped elsewhere */
281 if (!intel_gvt_host.mpt->map_gfn_to_mfn)
282 return 0;
283
249 return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr, 284 return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr,
250 map, type); 285 map);
251} 286}
252 287
253/** 288/**
@@ -263,6 +298,10 @@ static inline int intel_gvt_hypervisor_map_gfn_to_mfn(
263static inline int intel_gvt_hypervisor_set_trap_area( 298static inline int intel_gvt_hypervisor_set_trap_area(
264 struct intel_vgpu *vgpu, u64 start, u64 end, bool map) 299 struct intel_vgpu *vgpu, u64 start, u64 end, bool map)
265{ 300{
301 /* a MPT implementation could have MMIO trapped elsewhere */
302 if (!intel_gvt_host.mpt->set_trap_area)
303 return 0;
304
266 return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map); 305 return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map);
267} 306}
268 307
diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c
index 95218913b0bc..d2a0fbc896c3 100644
--- a/drivers/gpu/drm/i915/gvt/opregion.c
+++ b/drivers/gpu/drm/i915/gvt/opregion.c
@@ -73,7 +73,7 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map)
73 } 73 }
74 ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, 74 ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu,
75 vgpu_opregion(vgpu)->gfn[i], 75 vgpu_opregion(vgpu)->gfn[i],
76 mfn, 1, map, GVT_MAP_OPREGION); 76 mfn, 1, map);
77 if (ret) { 77 if (ret) {
78 gvt_err("fail to map GFN to MFN, errno: %d\n", ret); 78 gvt_err("fail to map GFN to MFN, errno: %d\n", ret);
79 return ret; 79 return ret;
@@ -89,28 +89,18 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map)
89 */ 89 */
90void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu) 90void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu)
91{ 91{
92 int i;
93
94 gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id); 92 gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id);
95 93
96 if (!vgpu_opregion(vgpu)->va) 94 if (!vgpu_opregion(vgpu)->va)
97 return; 95 return;
98 96
99 if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) { 97 if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
100 vunmap(vgpu_opregion(vgpu)->va);
101 for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) {
102 if (vgpu_opregion(vgpu)->pages[i]) {
103 put_page(vgpu_opregion(vgpu)->pages[i]);
104 vgpu_opregion(vgpu)->pages[i] = NULL;
105 }
106 }
107 } else {
108 map_vgpu_opregion(vgpu, false); 98 map_vgpu_opregion(vgpu, false);
109 free_pages((unsigned long)vgpu_opregion(vgpu)->va, 99 free_pages((unsigned long)vgpu_opregion(vgpu)->va,
110 INTEL_GVT_OPREGION_PORDER); 100 INTEL_GVT_OPREGION_PORDER);
111 }
112 101
113 vgpu_opregion(vgpu)->va = NULL; 102 vgpu_opregion(vgpu)->va = NULL;
103 }
114} 104}
115 105
116/** 106/**
@@ -137,22 +127,8 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa)
137 ret = map_vgpu_opregion(vgpu, true); 127 ret = map_vgpu_opregion(vgpu, true);
138 if (ret) 128 if (ret)
139 return ret; 129 return ret;
140 } else {
141 gvt_dbg_core("emulate opregion from userspace\n");
142
143 /*
144 * If opregion pages are not allocated from host kenrel,
145 * most of the params are meaningless
146 */
147 ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu,
148 0, /* not used */
149 0, /* not used */
150 2, /* not used */
151 1,
152 GVT_MAP_OPREGION);
153 if (ret)
154 return ret;
155 } 130 }
131
156 return 0; 132 return 0;
157} 133}
158 134
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
index 3af894b3d257..44136b1f3aab 100644
--- a/drivers/gpu/drm/i915/gvt/render.c
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -152,6 +152,8 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
152 152
153 if (wait_for_atomic((I915_READ_FW(reg) == 0), 50)) 153 if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
154 gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id); 154 gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id);
155 else
156 vgpu_vreg(vgpu, regs[ring_id]) = 0;
155 157
156 intel_uncore_forcewake_put(dev_priv, fw); 158 intel_uncore_forcewake_put(dev_priv, fw);
157 159
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
index 1df6a5460f3e..678b0be85376 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.c
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -36,12 +36,10 @@
36 36
37static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu) 37static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu)
38{ 38{
39 struct intel_vgpu_execlist *execlist;
40 enum intel_engine_id i; 39 enum intel_engine_id i;
41 struct intel_engine_cs *engine; 40 struct intel_engine_cs *engine;
42 41
43 for_each_engine(engine, vgpu->gvt->dev_priv, i) { 42 for_each_engine(engine, vgpu->gvt->dev_priv, i) {
44 execlist = &vgpu->execlist[i];
45 if (!list_empty(workload_q_head(vgpu, i))) 43 if (!list_empty(workload_q_head(vgpu, i)))
46 return true; 44 return true;
47 } 45 }
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 18acb45dd14d..f898df38dd9a 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -89,15 +89,15 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
89 } 89 }
90 90
91 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i); 91 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
92 dst = kmap_atomic(page); 92 dst = kmap(page);
93 intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, 93 intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
94 GTT_PAGE_SIZE); 94 GTT_PAGE_SIZE);
95 kunmap_atomic(dst); 95 kunmap(page);
96 i++; 96 i++;
97 } 97 }
98 98
99 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); 99 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
100 shadow_ring_context = kmap_atomic(page); 100 shadow_ring_context = kmap(page);
101 101
102#define COPY_REG(name) \ 102#define COPY_REG(name) \
103 intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ 103 intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
@@ -123,7 +123,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
123 sizeof(*shadow_ring_context), 123 sizeof(*shadow_ring_context),
124 GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); 124 GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
125 125
126 kunmap_atomic(shadow_ring_context); 126 kunmap(page);
127 return 0; 127 return 0;
128} 128}
129 129
@@ -160,8 +160,6 @@ static int shadow_context_status_change(struct notifier_block *nb,
160 160
161static int dispatch_workload(struct intel_vgpu_workload *workload) 161static int dispatch_workload(struct intel_vgpu_workload *workload)
162{ 162{
163 struct intel_vgpu *vgpu = workload->vgpu;
164 struct intel_gvt *gvt = vgpu->gvt;
165 int ring_id = workload->ring_id; 163 int ring_id = workload->ring_id;
166 struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; 164 struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
167 struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; 165 struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
@@ -174,6 +172,8 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
174 shadow_ctx->desc_template = workload->ctx_desc.addressing_mode << 172 shadow_ctx->desc_template = workload->ctx_desc.addressing_mode <<
175 GEN8_CTX_ADDRESSING_MODE_SHIFT; 173 GEN8_CTX_ADDRESSING_MODE_SHIFT;
176 174
175 mutex_lock(&dev_priv->drm.struct_mutex);
176
177 rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx); 177 rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
178 if (IS_ERR(rq)) { 178 if (IS_ERR(rq)) {
179 gvt_err("fail to allocate gem request\n"); 179 gvt_err("fail to allocate gem request\n");
@@ -185,40 +185,35 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
185 185
186 workload->req = i915_gem_request_get(rq); 186 workload->req = i915_gem_request_get(rq);
187 187
188 mutex_lock(&gvt->lock);
189
190 ret = intel_gvt_scan_and_shadow_workload(workload); 188 ret = intel_gvt_scan_and_shadow_workload(workload);
191 if (ret) 189 if (ret)
192 goto err; 190 goto out;
193 191
194 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx); 192 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
195 if (ret) 193 if (ret)
196 goto err; 194 goto out;
197 195
198 ret = populate_shadow_context(workload); 196 ret = populate_shadow_context(workload);
199 if (ret) 197 if (ret)
200 goto err; 198 goto out;
201 199
202 if (workload->prepare) { 200 if (workload->prepare) {
203 ret = workload->prepare(workload); 201 ret = workload->prepare(workload);
204 if (ret) 202 if (ret)
205 goto err; 203 goto out;
206 } 204 }
207 205
208 mutex_unlock(&gvt->lock);
209
210 gvt_dbg_sched("ring id %d submit workload to i915 %p\n", 206 gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
211 ring_id, workload->req); 207 ring_id, workload->req);
212 208
213 i915_add_request_no_flush(rq); 209 ret = 0;
214 workload->dispatched = true; 210 workload->dispatched = true;
215 return 0; 211out:
216err: 212 if (ret)
217 workload->status = ret; 213 workload->status = ret;
218
219 mutex_unlock(&gvt->lock);
220 214
221 i915_add_request_no_flush(rq); 215 i915_add_request_no_flush(rq);
216 mutex_unlock(&dev_priv->drm.struct_mutex);
222 return ret; 217 return ret;
223} 218}
224 219
@@ -318,10 +313,10 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
318 } 313 }
319 314
320 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i); 315 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
321 src = kmap_atomic(page); 316 src = kmap(page);
322 intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, 317 intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
323 GTT_PAGE_SIZE); 318 GTT_PAGE_SIZE);
324 kunmap_atomic(src); 319 kunmap(page);
325 i++; 320 i++;
326 } 321 }
327 322
@@ -329,7 +324,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
329 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4); 324 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
330 325
331 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); 326 page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
332 shadow_ring_context = kmap_atomic(page); 327 shadow_ring_context = kmap(page);
333 328
334#define COPY_REG(name) \ 329#define COPY_REG(name) \
335 intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \ 330 intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
@@ -347,7 +342,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
347 sizeof(*shadow_ring_context), 342 sizeof(*shadow_ring_context),
348 GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); 343 GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
349 344
350 kunmap_atomic(shadow_ring_context); 345 kunmap(page);
351} 346}
352 347
353static void complete_current_workload(struct intel_gvt *gvt, int ring_id) 348static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
@@ -438,9 +433,9 @@ static int workload_thread(void *priv)
438 intel_uncore_forcewake_get(gvt->dev_priv, 433 intel_uncore_forcewake_get(gvt->dev_priv,
439 FORCEWAKE_ALL); 434 FORCEWAKE_ALL);
440 435
441 mutex_lock(&gvt->dev_priv->drm.struct_mutex); 436 mutex_lock(&gvt->lock);
442 ret = dispatch_workload(workload); 437 ret = dispatch_workload(workload);
443 mutex_unlock(&gvt->dev_priv->drm.struct_mutex); 438 mutex_unlock(&gvt->lock);
444 439
445 if (ret) { 440 if (ret) {
446 gvt_err("fail to dispatch workload, skip\n"); 441 gvt_err("fail to dispatch workload, skip\n");
@@ -455,15 +450,15 @@ static int workload_thread(void *priv)
455 if (lret < 0) { 450 if (lret < 0) {
456 workload->status = lret; 451 workload->status = lret;
457 gvt_err("fail to wait workload, skip\n"); 452 gvt_err("fail to wait workload, skip\n");
453 } else {
454 workload->status = 0;
458 } 455 }
459 456
460complete: 457complete:
461 gvt_dbg_sched("will complete workload %p\n, status: %d\n", 458 gvt_dbg_sched("will complete workload %p\n, status: %d\n",
462 workload, workload->status); 459 workload, workload->status);
463 460
464 mutex_lock(&gvt->dev_priv->drm.struct_mutex);
465 complete_current_workload(gvt, ring_id); 461 complete_current_workload(gvt, ring_id);
466 mutex_unlock(&gvt->dev_priv->drm.struct_mutex);
467 462
468 i915_gem_request_put(fetch_and_zero(&workload->req)); 463 i915_gem_request_put(fetch_and_zero(&workload->req));
469 464
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 4f54005b976d..4f64845d8a4c 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -46,9 +46,13 @@ int setup_vgpu_mmio(struct intel_vgpu *vgpu)
46 struct intel_gvt *gvt = vgpu->gvt; 46 struct intel_gvt *gvt = vgpu->gvt;
47 const struct intel_gvt_device_info *info = &gvt->device_info; 47 const struct intel_gvt_device_info *info = &gvt->device_info;
48 48
49 vgpu->mmio.vreg = vzalloc(info->mmio_size * 2); 49 if (vgpu->mmio.vreg)
50 if (!vgpu->mmio.vreg) 50 memset(vgpu->mmio.vreg, 0, info->mmio_size * 2);
51 return -ENOMEM; 51 else {
52 vgpu->mmio.vreg = vzalloc(info->mmio_size * 2);
53 if (!vgpu->mmio.vreg)
54 return -ENOMEM;
55 }
52 56
53 vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size; 57 vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
54 58
@@ -95,6 +99,7 @@ static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu,
95 */ 99 */
96 memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4); 100 memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4);
97 memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4); 101 memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4);
102 memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4);
98 103
99 for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) { 104 for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) {
100 vgpu->cfg_space.bar[i].size = pci_resource_len( 105 vgpu->cfg_space.bar[i].size = pci_resource_len(
@@ -133,6 +138,106 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
133} 138}
134 139
135/** 140/**
141 * intel_gvt_init_vgpu_types - initialize vGPU type list
142 * @gvt : GVT device
143 *
144 * Initialize vGPU type list based on available resource.
145 *
146 */
147int intel_gvt_init_vgpu_types(struct intel_gvt *gvt)
148{
149 unsigned int num_types;
150 unsigned int i, low_avail;
151 unsigned int min_low;
152
153 /* vGPU type name is defined as GVTg_Vx_y which contains
154 * physical GPU generation type and 'y' means maximum vGPU
155 * instances user can create on one physical GPU for this
156 * type.
157 *
158 * Depend on physical SKU resource, might see vGPU types like
159 * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create
160 * different types of vGPU on same physical GPU depending on
161 * available resource. Each vGPU type will have "avail_instance"
162 * to indicate how many vGPU instance can be created for this
163 * type.
164 *
165 * Currently use static size here as we init type earlier..
166 */
167 low_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE;
168 num_types = 4;
169
170 gvt->types = kzalloc(num_types * sizeof(struct intel_vgpu_type),
171 GFP_KERNEL);
172 if (!gvt->types)
173 return -ENOMEM;
174
175 min_low = MB_TO_BYTES(32);
176 for (i = 0; i < num_types; ++i) {
177 if (low_avail / min_low == 0)
178 break;
179 gvt->types[i].low_gm_size = min_low;
180 gvt->types[i].high_gm_size = 3 * gvt->types[i].low_gm_size;
181 gvt->types[i].fence = 4;
182 gvt->types[i].max_instance = low_avail / min_low;
183 gvt->types[i].avail_instance = gvt->types[i].max_instance;
184
185 if (IS_GEN8(gvt->dev_priv))
186 sprintf(gvt->types[i].name, "GVTg_V4_%u",
187 gvt->types[i].max_instance);
188 else if (IS_GEN9(gvt->dev_priv))
189 sprintf(gvt->types[i].name, "GVTg_V5_%u",
190 gvt->types[i].max_instance);
191
192 min_low <<= 1;
193 gvt_dbg_core("type[%d]: %s max %u avail %u low %u high %u fence %u\n",
194 i, gvt->types[i].name, gvt->types[i].max_instance,
195 gvt->types[i].avail_instance,
196 gvt->types[i].low_gm_size,
197 gvt->types[i].high_gm_size, gvt->types[i].fence);
198 }
199
200 gvt->num_types = i;
201 return 0;
202}
203
204void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt)
205{
206 kfree(gvt->types);
207}
208
209static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt)
210{
211 int i;
212 unsigned int low_gm_avail, high_gm_avail, fence_avail;
213 unsigned int low_gm_min, high_gm_min, fence_min, total_min;
214
215 /* Need to depend on maxium hw resource size but keep on
216 * static config for now.
217 */
218 low_gm_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE -
219 gvt->gm.vgpu_allocated_low_gm_size;
220 high_gm_avail = MB_TO_BYTES(256) * 3 - HOST_HIGH_GM_SIZE -
221 gvt->gm.vgpu_allocated_high_gm_size;
222 fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
223 gvt->fence.vgpu_allocated_fence_num;
224
225 for (i = 0; i < gvt->num_types; i++) {
226 low_gm_min = low_gm_avail / gvt->types[i].low_gm_size;
227 high_gm_min = high_gm_avail / gvt->types[i].high_gm_size;
228 fence_min = fence_avail / gvt->types[i].fence;
229 total_min = min(min(low_gm_min, high_gm_min), fence_min);
230 gvt->types[i].avail_instance = min(gvt->types[i].max_instance,
231 total_min);
232
233 gvt_dbg_core("update type[%d]: %s max %u avail %u low %u high %u fence %u\n",
234 i, gvt->types[i].name, gvt->types[i].max_instance,
235 gvt->types[i].avail_instance, gvt->types[i].low_gm_size,
236 gvt->types[i].high_gm_size, gvt->types[i].fence);
237 }
238}
239
240/**
136 * intel_gvt_destroy_vgpu - destroy a virtual GPU 241 * intel_gvt_destroy_vgpu - destroy a virtual GPU
137 * @vgpu: virtual GPU 242 * @vgpu: virtual GPU
138 * 243 *
@@ -166,20 +271,11 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
166 clean_vgpu_mmio(vgpu); 271 clean_vgpu_mmio(vgpu);
167 vfree(vgpu); 272 vfree(vgpu);
168 273
274 intel_gvt_update_vgpu_types(gvt);
169 mutex_unlock(&gvt->lock); 275 mutex_unlock(&gvt->lock);
170} 276}
171 277
172/** 278static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
173 * intel_gvt_create_vgpu - create a virtual GPU
174 * @gvt: GVT device
175 * @param: vGPU creation parameters
176 *
177 * This function is called when user wants to create a virtual GPU.
178 *
179 * Returns:
180 * pointer to intel_vgpu, error pointer if failed.
181 */
182struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
183 struct intel_vgpu_creation_params *param) 279 struct intel_vgpu_creation_params *param)
184{ 280{
185 struct intel_vgpu *vgpu; 281 struct intel_vgpu *vgpu;
@@ -224,15 +320,9 @@ struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
224 if (ret) 320 if (ret)
225 goto out_detach_hypervisor_vgpu; 321 goto out_detach_hypervisor_vgpu;
226 322
227 if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) {
228 ret = intel_vgpu_init_opregion(vgpu, 0);
229 if (ret)
230 goto out_clean_gtt;
231 }
232
233 ret = intel_vgpu_init_display(vgpu); 323 ret = intel_vgpu_init_display(vgpu);
234 if (ret) 324 if (ret)
235 goto out_clean_opregion; 325 goto out_clean_gtt;
236 326
237 ret = intel_vgpu_init_execlist(vgpu); 327 ret = intel_vgpu_init_execlist(vgpu);
238 if (ret) 328 if (ret)
@@ -257,8 +347,6 @@ out_clean_execlist:
257 intel_vgpu_clean_execlist(vgpu); 347 intel_vgpu_clean_execlist(vgpu);
258out_clean_display: 348out_clean_display:
259 intel_vgpu_clean_display(vgpu); 349 intel_vgpu_clean_display(vgpu);
260out_clean_opregion:
261 intel_vgpu_clean_opregion(vgpu);
262out_clean_gtt: 350out_clean_gtt:
263 intel_vgpu_clean_gtt(vgpu); 351 intel_vgpu_clean_gtt(vgpu);
264out_detach_hypervisor_vgpu: 352out_detach_hypervisor_vgpu:
@@ -272,3 +360,49 @@ out_free_vgpu:
272 mutex_unlock(&gvt->lock); 360 mutex_unlock(&gvt->lock);
273 return ERR_PTR(ret); 361 return ERR_PTR(ret);
274} 362}
363
364/**
365 * intel_gvt_create_vgpu - create a virtual GPU
366 * @gvt: GVT device
367 * @type: type of the vGPU to create
368 *
369 * This function is called when user wants to create a virtual GPU.
370 *
371 * Returns:
372 * pointer to intel_vgpu, error pointer if failed.
373 */
374struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
375 struct intel_vgpu_type *type)
376{
377 struct intel_vgpu_creation_params param;
378 struct intel_vgpu *vgpu;
379
380 param.handle = 0;
381 param.low_gm_sz = type->low_gm_size;
382 param.high_gm_sz = type->high_gm_size;
383 param.fence_sz = type->fence;
384
385 /* XXX current param based on MB */
386 param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz);
387 param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz);
388
389 vgpu = __intel_gvt_create_vgpu(gvt, &param);
390 if (IS_ERR(vgpu))
391 return vgpu;
392
393 /* calculate left instance change for types */
394 intel_gvt_update_vgpu_types(gvt);
395
396 return vgpu;
397}
398
399/**
400 * intel_gvt_reset_vgpu - reset a virtual GPU
401 * @vgpu: virtual GPU
402 *
403 * This function is called when user wants to reset a virtual GPU.
404 *
405 */
406void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu)
407{
408}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 7166a1a6265d..96407f684f7f 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -547,11 +547,11 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
547 pipe, plane); 547 pipe, plane);
548 } 548 }
549 if (work->flip_queued_req) { 549 if (work->flip_queued_req) {
550 struct intel_engine_cs *engine = i915_gem_request_get_engine(work->flip_queued_req); 550 struct intel_engine_cs *engine = work->flip_queued_req->engine;
551 551
552 seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n", 552 seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n",
553 engine->name, 553 engine->name,
554 i915_gem_request_get_seqno(work->flip_queued_req), 554 work->flip_queued_req->global_seqno,
555 atomic_read(&dev_priv->gt.global_timeline.next_seqno), 555 atomic_read(&dev_priv->gt.global_timeline.next_seqno),
556 intel_engine_get_seqno(engine), 556 intel_engine_get_seqno(engine),
557 i915_gem_request_completed(work->flip_queued_req)); 557 i915_gem_request_completed(work->flip_queued_req));
@@ -631,8 +631,9 @@ static void print_request(struct seq_file *m,
631 struct drm_i915_gem_request *rq, 631 struct drm_i915_gem_request *rq,
632 const char *prefix) 632 const char *prefix)
633{ 633{
634 seq_printf(m, "%s%x [%x:%x] @ %d: %s\n", prefix, 634 seq_printf(m, "%s%x [%x:%x] prio=%d @ %dms: %s\n", prefix,
635 rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, 635 rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno,
636 rq->priotree.priority,
636 jiffies_to_msecs(jiffies - rq->emitted_jiffies), 637 jiffies_to_msecs(jiffies - rq->emitted_jiffies),
637 rq->timeline->common->name); 638 rq->timeline->common->name);
638} 639}
@@ -1761,8 +1762,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
1761 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); 1762 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
1762 intel_runtime_pm_put(dev_priv); 1763 intel_runtime_pm_put(dev_priv);
1763 1764
1764 seq_printf(m, "self-refresh: %s\n", 1765 seq_printf(m, "self-refresh: %s\n", enableddisabled(sr_enabled));
1765 sr_enabled ? "enabled" : "disabled");
1766 1766
1767 return 0; 1767 return 0;
1768} 1768}
@@ -3216,6 +3216,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
3216 3216
3217 if (i915.enable_execlists) { 3217 if (i915.enable_execlists) {
3218 u32 ptr, read, write; 3218 u32 ptr, read, write;
3219 struct rb_node *rb;
3219 3220
3220 seq_printf(m, "\tExeclist status: 0x%08x %08x\n", 3221 seq_printf(m, "\tExeclist status: 0x%08x %08x\n",
3221 I915_READ(RING_EXECLIST_STATUS_LO(engine)), 3222 I915_READ(RING_EXECLIST_STATUS_LO(engine)),
@@ -3254,11 +3255,12 @@ static int i915_engine_info(struct seq_file *m, void *unused)
3254 seq_printf(m, "\t\tELSP[1] idle\n"); 3255 seq_printf(m, "\t\tELSP[1] idle\n");
3255 rcu_read_unlock(); 3256 rcu_read_unlock();
3256 3257
3257 spin_lock_irq(&engine->execlist_lock); 3258 spin_lock_irq(&engine->timeline->lock);
3258 list_for_each_entry(rq, &engine->execlist_queue, execlist_link) { 3259 for (rb = engine->execlist_first; rb; rb = rb_next(rb)) {
3260 rq = rb_entry(rb, typeof(*rq), priotree.node);
3259 print_request(m, rq, "\t\tQ "); 3261 print_request(m, rq, "\t\tQ ");
3260 } 3262 }
3261 spin_unlock_irq(&engine->execlist_lock); 3263 spin_unlock_irq(&engine->timeline->lock);
3262 } else if (INTEL_GEN(dev_priv) > 6) { 3264 } else if (INTEL_GEN(dev_priv) > 6) {
3263 seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n", 3265 seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
3264 I915_READ(RING_PP_DIR_BASE(engine))); 3266 I915_READ(RING_PP_DIR_BASE(engine)));
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 0213a3090ab3..445fec9c2841 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -150,7 +150,7 @@ static void intel_detect_pch(struct drm_device *dev)
150 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting 150 /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
151 * (which really amounts to a PCH but no South Display). 151 * (which really amounts to a PCH but no South Display).
152 */ 152 */
153 if (INTEL_INFO(dev)->num_pipes == 0) { 153 if (INTEL_INFO(dev_priv)->num_pipes == 0) {
154 dev_priv->pch_type = PCH_NOP; 154 dev_priv->pch_type = PCH_NOP;
155 return; 155 return;
156 } 156 }
@@ -323,6 +323,10 @@ static int i915_getparam(struct drm_device *dev, void *data,
323 */ 323 */
324 value = i915_gem_mmap_gtt_version(); 324 value = i915_gem_mmap_gtt_version();
325 break; 325 break;
326 case I915_PARAM_HAS_SCHEDULER:
327 value = dev_priv->engine[RCS] &&
328 dev_priv->engine[RCS]->schedule;
329 break;
326 case I915_PARAM_MMAP_VERSION: 330 case I915_PARAM_MMAP_VERSION:
327 /* Remember to bump this if the version changes! */ 331 /* Remember to bump this if the version changes! */
328 case I915_PARAM_HAS_GEM: 332 case I915_PARAM_HAS_GEM:
@@ -374,12 +378,12 @@ static int
374intel_alloc_mchbar_resource(struct drm_device *dev) 378intel_alloc_mchbar_resource(struct drm_device *dev)
375{ 379{
376 struct drm_i915_private *dev_priv = to_i915(dev); 380 struct drm_i915_private *dev_priv = to_i915(dev);
377 int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 381 int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
378 u32 temp_lo, temp_hi = 0; 382 u32 temp_lo, temp_hi = 0;
379 u64 mchbar_addr; 383 u64 mchbar_addr;
380 int ret; 384 int ret;
381 385
382 if (INTEL_INFO(dev)->gen >= 4) 386 if (INTEL_GEN(dev_priv) >= 4)
383 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); 387 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
384 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); 388 pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
385 mchbar_addr = ((u64)temp_hi << 32) | temp_lo; 389 mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
@@ -406,7 +410,7 @@ intel_alloc_mchbar_resource(struct drm_device *dev)
406 return ret; 410 return ret;
407 } 411 }
408 412
409 if (INTEL_INFO(dev)->gen >= 4) 413 if (INTEL_GEN(dev_priv) >= 4)
410 pci_write_config_dword(dev_priv->bridge_dev, reg + 4, 414 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
411 upper_32_bits(dev_priv->mch_res.start)); 415 upper_32_bits(dev_priv->mch_res.start));
412 416
@@ -420,7 +424,7 @@ static void
420intel_setup_mchbar(struct drm_device *dev) 424intel_setup_mchbar(struct drm_device *dev)
421{ 425{
422 struct drm_i915_private *dev_priv = to_i915(dev); 426 struct drm_i915_private *dev_priv = to_i915(dev);
423 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 427 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
424 u32 temp; 428 u32 temp;
425 bool enabled; 429 bool enabled;
426 430
@@ -460,7 +464,7 @@ static void
460intel_teardown_mchbar(struct drm_device *dev) 464intel_teardown_mchbar(struct drm_device *dev)
461{ 465{
462 struct drm_i915_private *dev_priv = to_i915(dev); 466 struct drm_i915_private *dev_priv = to_i915(dev);
463 int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; 467 int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
464 468
465 if (dev_priv->mchbar_need_disable) { 469 if (dev_priv->mchbar_need_disable) {
466 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) { 470 if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
@@ -491,7 +495,7 @@ static unsigned int i915_vga_set_decode(void *cookie, bool state)
491{ 495{
492 struct drm_device *dev = cookie; 496 struct drm_device *dev = cookie;
493 497
494 intel_modeset_vga_set_state(dev, state); 498 intel_modeset_vga_set_state(to_i915(dev), state);
495 if (state) 499 if (state)
496 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 500 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
497 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 501 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
@@ -607,7 +611,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
607 611
608 intel_modeset_gem_init(dev); 612 intel_modeset_gem_init(dev);
609 613
610 if (INTEL_INFO(dev)->num_pipes == 0) 614 if (INTEL_INFO(dev_priv)->num_pipes == 0)
611 return 0; 615 return 0;
612 616
613 ret = intel_fbdev_init(dev); 617 ret = intel_fbdev_init(dev);
@@ -879,7 +883,7 @@ static int i915_mmio_setup(struct drm_device *dev)
879 * the register BAR remains the same size for all the earlier 883 * the register BAR remains the same size for all the earlier
880 * generations up to Ironlake. 884 * generations up to Ironlake.
881 */ 885 */
882 if (INTEL_INFO(dev)->gen < 5) 886 if (INTEL_GEN(dev_priv) < 5)
883 mmio_size = 512 * 1024; 887 mmio_size = 512 * 1024;
884 else 888 else
885 mmio_size = 2 * 1024 * 1024; 889 mmio_size = 2 * 1024 * 1024;
@@ -1168,8 +1172,8 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
1168 1172
1169/** 1173/**
1170 * i915_driver_load - setup chip and create an initial config 1174 * i915_driver_load - setup chip and create an initial config
1171 * @dev: DRM device 1175 * @pdev: PCI device
1172 * @flags: startup flags 1176 * @ent: matching PCI ID entry
1173 * 1177 *
1174 * The driver load routine has to do several things: 1178 * The driver load routine has to do several things:
1175 * - drive output discovery via intel_modeset_init() 1179 * - drive output discovery via intel_modeset_init()
@@ -1438,7 +1442,7 @@ static int i915_drm_suspend(struct drm_device *dev)
1438 1442
1439 intel_suspend_hw(dev_priv); 1443 intel_suspend_hw(dev_priv);
1440 1444
1441 i915_gem_suspend_gtt_mappings(dev); 1445 i915_gem_suspend_gtt_mappings(dev_priv);
1442 1446
1443 i915_save_state(dev); 1447 i915_save_state(dev);
1444 1448
@@ -1512,7 +1516,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
1512 * Fujitsu FSC S7110 1516 * Fujitsu FSC S7110
1513 * Acer Aspire 1830T 1517 * Acer Aspire 1830T
1514 */ 1518 */
1515 if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6)) 1519 if (!(hibernation && INTEL_GEN(dev_priv) < 6))
1516 pci_set_power_state(pdev, PCI_D3hot); 1520 pci_set_power_state(pdev, PCI_D3hot);
1517 1521
1518 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv); 1522 dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
@@ -2422,7 +2426,7 @@ static int intel_runtime_resume(struct device *kdev)
2422 * No point of rolling back things in case of an error, as the best 2426 * No point of rolling back things in case of an error, as the best
2423 * we can do is to hope that things will still work (and disable RPM). 2427 * we can do is to hope that things will still work (and disable RPM).
2424 */ 2428 */
2425 i915_gem_init_swizzling(dev); 2429 i915_gem_init_swizzling(dev_priv);
2426 2430
2427 intel_runtime_pm_enable_interrupts(dev_priv); 2431 intel_runtime_pm_enable_interrupts(dev_priv);
2428 2432
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 6f4a6bcf6ed4..56002a52936d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -60,11 +60,15 @@
60#include "intel_ringbuffer.h" 60#include "intel_ringbuffer.h"
61 61
62#include "i915_gem.h" 62#include "i915_gem.h"
63#include "i915_gem_fence_reg.h"
64#include "i915_gem_object.h"
63#include "i915_gem_gtt.h" 65#include "i915_gem_gtt.h"
64#include "i915_gem_render_state.h" 66#include "i915_gem_render_state.h"
65#include "i915_gem_request.h" 67#include "i915_gem_request.h"
66#include "i915_gem_timeline.h" 68#include "i915_gem_timeline.h"
67 69
70#include "i915_vma.h"
71
68#include "intel_gvt.h" 72#include "intel_gvt.h"
69 73
70/* General customization: 74/* General customization:
@@ -72,8 +76,8 @@
72 76
73#define DRIVER_NAME "i915" 77#define DRIVER_NAME "i915"
74#define DRIVER_DESC "Intel Graphics" 78#define DRIVER_DESC "Intel Graphics"
75#define DRIVER_DATE "20161108" 79#define DRIVER_DATE "20161121"
76#define DRIVER_TIMESTAMP 1478587895 80#define DRIVER_TIMESTAMP 1479717903
77 81
78#undef WARN_ON 82#undef WARN_ON
79/* Many gcc seem to no see through this and fall over :( */ 83/* Many gcc seem to no see through this and fall over :( */
@@ -125,6 +129,11 @@ static inline const char *onoff(bool v)
125 return v ? "on" : "off"; 129 return v ? "on" : "off";
126} 130}
127 131
132static inline const char *enableddisabled(bool v)
133{
134 return v ? "enabled" : "disabled";
135}
136
128enum pipe { 137enum pipe {
129 INVALID_PIPE = -1, 138 INVALID_PIPE = -1,
130 PIPE_A = 0, 139 PIPE_A = 0,
@@ -459,23 +468,6 @@ struct intel_opregion {
459struct intel_overlay; 468struct intel_overlay;
460struct intel_overlay_error_state; 469struct intel_overlay_error_state;
461 470
462struct drm_i915_fence_reg {
463 struct list_head link;
464 struct drm_i915_private *i915;
465 struct i915_vma *vma;
466 int pin_count;
467 int id;
468 /**
469 * Whether the tiling parameters for the currently
470 * associated fence register have changed. Note that
471 * for the purposes of tracking tiling changes we also
472 * treat the unfenced register, the register slot that
473 * the object occupies whilst it executes a fenced
474 * command (such as BLT on gen2/3), as a "fence".
475 */
476 bool dirty;
477};
478
479struct sdvo_device_mapping { 471struct sdvo_device_mapping {
480 u8 initialized; 472 u8 initialized;
481 u8 dvo_port; 473 u8 dvo_port;
@@ -487,6 +479,7 @@ struct sdvo_device_mapping {
487 479
488struct intel_connector; 480struct intel_connector;
489struct intel_encoder; 481struct intel_encoder;
482struct intel_atomic_state;
490struct intel_crtc_state; 483struct intel_crtc_state;
491struct intel_initial_plane_config; 484struct intel_initial_plane_config;
492struct intel_crtc; 485struct intel_crtc;
@@ -500,8 +493,12 @@ struct drm_i915_display_funcs {
500 int (*compute_intermediate_wm)(struct drm_device *dev, 493 int (*compute_intermediate_wm)(struct drm_device *dev,
501 struct intel_crtc *intel_crtc, 494 struct intel_crtc *intel_crtc,
502 struct intel_crtc_state *newstate); 495 struct intel_crtc_state *newstate);
503 void (*initial_watermarks)(struct intel_crtc_state *cstate); 496 void (*initial_watermarks)(struct intel_atomic_state *state,
504 void (*optimize_watermarks)(struct intel_crtc_state *cstate); 497 struct intel_crtc_state *cstate);
498 void (*atomic_update_watermarks)(struct intel_atomic_state *state,
499 struct intel_crtc_state *cstate);
500 void (*optimize_watermarks)(struct intel_atomic_state *state,
501 struct intel_crtc_state *cstate);
505 int (*compute_global_watermarks)(struct drm_atomic_state *state); 502 int (*compute_global_watermarks)(struct drm_atomic_state *state);
506 void (*update_wm)(struct intel_crtc *crtc); 503 void (*update_wm)(struct intel_crtc *crtc);
507 int (*modeset_calc_cdclk)(struct drm_atomic_state *state); 504 int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
@@ -562,6 +559,18 @@ enum forcewake_domains {
562#define FW_REG_READ (1) 559#define FW_REG_READ (1)
563#define FW_REG_WRITE (2) 560#define FW_REG_WRITE (2)
564 561
562enum decoupled_power_domain {
563 GEN9_DECOUPLED_PD_BLITTER = 0,
564 GEN9_DECOUPLED_PD_RENDER,
565 GEN9_DECOUPLED_PD_MEDIA,
566 GEN9_DECOUPLED_PD_ALL
567};
568
569enum decoupled_ops {
570 GEN9_DECOUPLED_OP_WRITE = 0,
571 GEN9_DECOUPLED_OP_READ
572};
573
565enum forcewake_domains 574enum forcewake_domains
566intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, 575intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
567 i915_reg_t reg, unsigned int op); 576 i915_reg_t reg, unsigned int op);
@@ -668,7 +677,7 @@ struct intel_csr {
668 func(is_skylake); \ 677 func(is_skylake); \
669 func(is_broxton); \ 678 func(is_broxton); \
670 func(is_kabylake); \ 679 func(is_kabylake); \
671 func(is_preliminary); \ 680 func(is_alpha_support); \
672 /* Keep has_* in alphabetical order */ \ 681 /* Keep has_* in alphabetical order */ \
673 func(has_64bit_reloc); \ 682 func(has_64bit_reloc); \
674 func(has_csr); \ 683 func(has_csr); \
@@ -696,7 +705,8 @@ struct intel_csr {
696 func(cursor_needs_physical); \ 705 func(cursor_needs_physical); \
697 func(hws_needs_physical); \ 706 func(hws_needs_physical); \
698 func(overlay_needs_physical); \ 707 func(overlay_needs_physical); \
699 func(supports_tv) 708 func(supports_tv); \
709 func(has_decoupled_mmio)
700 710
701struct sseu_dev_info { 711struct sseu_dev_info {
702 u8 slice_mask; 712 u8 slice_mask;
@@ -949,6 +959,7 @@ struct i915_gem_context {
949 /* Unique identifier for this context, used by the hw for tracking */ 959 /* Unique identifier for this context, used by the hw for tracking */
950 unsigned int hw_id; 960 unsigned int hw_id;
951 u32 user_handle; 961 u32 user_handle;
962 int priority; /* greater priorities are serviced first */
952 963
953 u32 ggtt_alignment; 964 u32 ggtt_alignment;
954 965
@@ -1791,6 +1802,7 @@ struct drm_i915_private {
1791 struct kmem_cache *objects; 1802 struct kmem_cache *objects;
1792 struct kmem_cache *vmas; 1803 struct kmem_cache *vmas;
1793 struct kmem_cache *requests; 1804 struct kmem_cache *requests;
1805 struct kmem_cache *dependencies;
1794 1806
1795 const struct intel_device_info info; 1807 const struct intel_device_info info;
1796 1808
@@ -2053,13 +2065,6 @@ struct drm_i915_private {
2053 */ 2065 */
2054 uint16_t skl_latency[8]; 2066 uint16_t skl_latency[8];
2055 2067
2056 /*
2057 * The skl_wm_values structure is a bit too big for stack
2058 * allocation, so we keep the staging struct where we store
2059 * intermediate results here instead.
2060 */
2061 struct skl_wm_values skl_results;
2062
2063 /* current hardware state */ 2068 /* current hardware state */
2064 union { 2069 union {
2065 struct ilk_wm_values hw; 2070 struct ilk_wm_values hw;
@@ -2179,31 +2184,6 @@ enum hdmi_force_audio {
2179 2184
2180#define I915_GTT_OFFSET_NONE ((u32)-1) 2185#define I915_GTT_OFFSET_NONE ((u32)-1)
2181 2186
2182struct drm_i915_gem_object_ops {
2183 unsigned int flags;
2184#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1
2185#define I915_GEM_OBJECT_IS_SHRINKABLE 0x2
2186
2187 /* Interface between the GEM object and its backing storage.
2188 * get_pages() is called once prior to the use of the associated set
2189 * of pages before to binding them into the GTT, and put_pages() is
2190 * called after we no longer need them. As we expect there to be
2191 * associated cost with migrating pages between the backing storage
2192 * and making them available for the GPU (e.g. clflush), we may hold
2193 * onto the pages after they are no longer referenced by the GPU
2194 * in case they may be used again shortly (for example migrating the
2195 * pages to a different memory domain within the GTT). put_pages()
2196 * will therefore most likely be called when the object itself is
2197 * being released or under memory pressure (where we attempt to
2198 * reap pages for the shrinker).
2199 */
2200 struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
2201 void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
2202
2203 int (*dmabuf_export)(struct drm_i915_gem_object *);
2204 void (*release)(struct drm_i915_gem_object *);
2205};
2206
2207/* 2187/*
2208 * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is 2188 * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
2209 * considered to be the frontbuffer for the given plane interface-wise. This 2189 * considered to be the frontbuffer for the given plane interface-wise. This
@@ -2225,292 +2205,6 @@ struct drm_i915_gem_object_ops {
2225#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ 2205#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
2226 (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) 2206 (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
2227 2207
2228struct drm_i915_gem_object {
2229 struct drm_gem_object base;
2230
2231 const struct drm_i915_gem_object_ops *ops;
2232
2233 /** List of VMAs backed by this object */
2234 struct list_head vma_list;
2235 struct rb_root vma_tree;
2236
2237 /** Stolen memory for this object, instead of being backed by shmem. */
2238 struct drm_mm_node *stolen;
2239 struct list_head global_link;
2240 union {
2241 struct rcu_head rcu;
2242 struct llist_node freed;
2243 };
2244
2245 /**
2246 * Whether the object is currently in the GGTT mmap.
2247 */
2248 struct list_head userfault_link;
2249
2250 /** Used in execbuf to temporarily hold a ref */
2251 struct list_head obj_exec_link;
2252
2253 struct list_head batch_pool_link;
2254
2255 unsigned long flags;
2256
2257 /**
2258 * Have we taken a reference for the object for incomplete GPU
2259 * activity?
2260 */
2261#define I915_BO_ACTIVE_REF 0
2262
2263 /*
2264 * Is the object to be mapped as read-only to the GPU
2265 * Only honoured if hardware has relevant pte bit
2266 */
2267 unsigned long gt_ro:1;
2268 unsigned int cache_level:3;
2269 unsigned int cache_dirty:1;
2270
2271 atomic_t frontbuffer_bits;
2272 unsigned int frontbuffer_ggtt_origin; /* write once */
2273
2274 /** Current tiling stride for the object, if it's tiled. */
2275 unsigned int tiling_and_stride;
2276#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
2277#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
2278#define STRIDE_MASK (~TILING_MASK)
2279
2280 /** Count of VMA actually bound by this object */
2281 unsigned int bind_count;
2282 unsigned int active_count;
2283 unsigned int pin_display;
2284
2285 struct {
2286 struct mutex lock; /* protects the pages and their use */
2287 atomic_t pages_pin_count;
2288
2289 struct sg_table *pages;
2290 void *mapping;
2291
2292 struct i915_gem_object_page_iter {
2293 struct scatterlist *sg_pos;
2294 unsigned int sg_idx; /* in pages, but 32bit eek! */
2295
2296 struct radix_tree_root radix;
2297 struct mutex lock; /* protects this cache */
2298 } get_page;
2299
2300 /**
2301 * Advice: are the backing pages purgeable?
2302 */
2303 unsigned int madv:2;
2304
2305 /**
2306 * This is set if the object has been written to since the
2307 * pages were last acquired.
2308 */
2309 bool dirty:1;
2310
2311 /**
2312 * This is set if the object has been pinned due to unknown
2313 * swizzling.
2314 */
2315 bool quirked:1;
2316 } mm;
2317
2318 /** Breadcrumb of last rendering to the buffer.
2319 * There can only be one writer, but we allow for multiple readers.
2320 * If there is a writer that necessarily implies that all other
2321 * read requests are complete - but we may only be lazily clearing
2322 * the read requests. A read request is naturally the most recent
2323 * request on a ring, so we may have two different write and read
2324 * requests on one ring where the write request is older than the
2325 * read request. This allows for the CPU to read from an active
2326 * buffer by only waiting for the write to complete.
2327 */
2328 struct reservation_object *resv;
2329
2330 /** References from framebuffers, locks out tiling changes. */
2331 unsigned long framebuffer_references;
2332
2333 /** Record of address bit 17 of each page at last unbind. */
2334 unsigned long *bit_17;
2335
2336 struct i915_gem_userptr {
2337 uintptr_t ptr;
2338 unsigned read_only :1;
2339
2340 struct i915_mm_struct *mm;
2341 struct i915_mmu_object *mmu_object;
2342 struct work_struct *work;
2343 } userptr;
2344
2345 /** for phys allocated objects */
2346 struct drm_dma_handle *phys_handle;
2347
2348 struct reservation_object __builtin_resv;
2349};
2350
2351static inline struct drm_i915_gem_object *
2352to_intel_bo(struct drm_gem_object *gem)
2353{
2354 /* Assert that to_intel_bo(NULL) == NULL */
2355 BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
2356
2357 return container_of(gem, struct drm_i915_gem_object, base);
2358}
2359
2360/**
2361 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
2362 * @filp: DRM file private date
2363 * @handle: userspace handle
2364 *
2365 * Returns:
2366 *
2367 * A pointer to the object named by the handle if such exists on @filp, NULL
2368 * otherwise. This object is only valid whilst under the RCU read lock, and
2369 * note carefully the object may be in the process of being destroyed.
2370 */
2371static inline struct drm_i915_gem_object *
2372i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
2373{
2374#ifdef CONFIG_LOCKDEP
2375 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map));
2376#endif
2377 return idr_find(&file->object_idr, handle);
2378}
2379
2380static inline struct drm_i915_gem_object *
2381i915_gem_object_lookup(struct drm_file *file, u32 handle)
2382{
2383 struct drm_i915_gem_object *obj;
2384
2385 rcu_read_lock();
2386 obj = i915_gem_object_lookup_rcu(file, handle);
2387 if (obj && !kref_get_unless_zero(&obj->base.refcount))
2388 obj = NULL;
2389 rcu_read_unlock();
2390
2391 return obj;
2392}
2393
2394__deprecated
2395extern struct drm_gem_object *
2396drm_gem_object_lookup(struct drm_file *file, u32 handle);
2397
2398__attribute__((nonnull))
2399static inline struct drm_i915_gem_object *
2400i915_gem_object_get(struct drm_i915_gem_object *obj)
2401{
2402 drm_gem_object_reference(&obj->base);
2403 return obj;
2404}
2405
2406__deprecated
2407extern void drm_gem_object_reference(struct drm_gem_object *);
2408
2409__attribute__((nonnull))
2410static inline void
2411i915_gem_object_put(struct drm_i915_gem_object *obj)
2412{
2413 __drm_gem_object_unreference(&obj->base);
2414}
2415
2416__deprecated
2417extern void drm_gem_object_unreference(struct drm_gem_object *);
2418
2419__deprecated
2420extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
2421
2422static inline bool
2423i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
2424{
2425 return atomic_read(&obj->base.refcount.refcount) == 0;
2426}
2427
2428static inline bool
2429i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
2430{
2431 return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
2432}
2433
2434static inline bool
2435i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
2436{
2437 return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;
2438}
2439
2440static inline bool
2441i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
2442{
2443 return obj->active_count;
2444}
2445
2446static inline bool
2447i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
2448{
2449 return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
2450}
2451
2452static inline void
2453i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
2454{
2455 lockdep_assert_held(&obj->base.dev->struct_mutex);
2456 __set_bit(I915_BO_ACTIVE_REF, &obj->flags);
2457}
2458
2459static inline void
2460i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
2461{
2462 lockdep_assert_held(&obj->base.dev->struct_mutex);
2463 __clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
2464}
2465
2466void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
2467
2468static inline unsigned int
2469i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
2470{
2471 return obj->tiling_and_stride & TILING_MASK;
2472}
2473
2474static inline bool
2475i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
2476{
2477 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
2478}
2479
2480static inline unsigned int
2481i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
2482{
2483 return obj->tiling_and_stride & STRIDE_MASK;
2484}
2485
2486static inline struct intel_engine_cs *
2487i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
2488{
2489 struct intel_engine_cs *engine = NULL;
2490 struct dma_fence *fence;
2491
2492 rcu_read_lock();
2493 fence = reservation_object_get_excl_rcu(obj->resv);
2494 rcu_read_unlock();
2495
2496 if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))
2497 engine = to_request(fence)->engine;
2498 dma_fence_put(fence);
2499
2500 return engine;
2501}
2502
2503static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
2504{
2505 i915_gem_object_get(vma->obj);
2506 return vma;
2507}
2508
2509static inline void i915_vma_put(struct i915_vma *vma)
2510{
2511 i915_gem_object_put(vma->obj);
2512}
2513
2514/* 2208/*
2515 * Optimised SGL iterator for GEM objects 2209 * Optimised SGL iterator for GEM objects
2516 */ 2210 */
@@ -2683,24 +2377,19 @@ struct drm_i915_cmd_table {
2683 int count; 2377 int count;
2684}; 2378};
2685 2379
2686/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */ 2380static inline const struct intel_device_info *
2687#define __I915__(p) ({ \ 2381intel_info(const struct drm_i915_private *dev_priv)
2688 struct drm_i915_private *__p; \ 2382{
2689 if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \ 2383 return &dev_priv->info;
2690 __p = (struct drm_i915_private *)p; \ 2384}
2691 else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \ 2385
2692 __p = to_i915((struct drm_device *)p); \ 2386#define INTEL_INFO(dev_priv) intel_info((dev_priv))
2693 else \
2694 BUILD_BUG(); \
2695 __p; \
2696})
2697#define INTEL_INFO(p) (&__I915__(p)->info)
2698 2387
2699#define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) 2388#define INTEL_GEN(dev_priv) ((dev_priv)->info.gen)
2700#define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) 2389#define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id)
2701 2390
2702#define REVID_FOREVER 0xff 2391#define REVID_FOREVER 0xff
2703#define INTEL_REVID(p) (__I915__(p)->drm.pdev->revision) 2392#define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision)
2704 2393
2705#define GEN_FOREVER (0) 2394#define GEN_FOREVER (0)
2706/* 2395/*
@@ -2797,7 +2486,7 @@ struct drm_i915_cmd_table {
2797#define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ 2486#define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \
2798 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030) 2487 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030)
2799 2488
2800#define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) 2489#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
2801 2490
2802#define SKL_REVID_A0 0x0 2491#define SKL_REVID_A0 0x0
2803#define SKL_REVID_B0 0x1 2492#define SKL_REVID_B0 0x1
@@ -2851,28 +2540,31 @@ struct drm_i915_cmd_table {
2851#define ALL_ENGINES (~0) 2540#define ALL_ENGINES (~0)
2852 2541
2853#define HAS_ENGINE(dev_priv, id) \ 2542#define HAS_ENGINE(dev_priv, id) \
2854 (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id))) 2543 (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id)))
2855 2544
2856#define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) 2545#define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS)
2857#define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) 2546#define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2)
2858#define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) 2547#define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS)
2859#define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) 2548#define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS)
2860 2549
2861#define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) 2550#define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc)
2862#define HAS_SNOOP(dev) (INTEL_INFO(dev)->has_snoop) 2551#define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop)
2863#define HAS_EDRAM(dev) (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED)) 2552#define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED))
2864#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ 2553#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \
2865 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) 2554 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
2866#define HWS_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->hws_needs_physical)
2867 2555
2868#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->has_hw_contexts) 2556#define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical)
2869#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->has_logical_ring_contexts)
2870#define USES_PPGTT(dev) (i915.enable_ppgtt)
2871#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2)
2872#define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3)
2873 2557
2874#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) 2558#define HAS_HW_CONTEXTS(dev_priv) ((dev_priv)->info.has_hw_contexts)
2875#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) 2559#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
2560 ((dev_priv)->info.has_logical_ring_contexts)
2561#define USES_PPGTT(dev_priv) (i915.enable_ppgtt)
2562#define USES_FULL_PPGTT(dev_priv) (i915.enable_ppgtt >= 2)
2563#define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3)
2564
2565#define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay)
2566#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \
2567 ((dev_priv)->info.overlay_needs_physical)
2876 2568
2877/* Early gen2 have a totally busted CS tlb and require pinned batches. */ 2569/* Early gen2 have a totally busted CS tlb and require pinned batches. */
2878#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_845G(dev_priv)) 2570#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_845G(dev_priv))
@@ -2889,8 +2581,8 @@ struct drm_i915_cmd_table {
2889 * legacy irq no. is shared with another device. The kernel then disables that 2581 * legacy irq no. is shared with another device. The kernel then disables that
2890 * interrupt source and so prevents the other device from working properly. 2582 * interrupt source and so prevents the other device from working properly.
2891 */ 2583 */
2892#define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) 2584#define HAS_AUX_IRQ(dev_priv) ((dev_priv)->info.gen >= 5)
2893#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->has_gmbus_irq) 2585#define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq)
2894 2586
2895/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte 2587/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
2896 * rows, which changed the alignment requirements and fence programming. 2588 * rows, which changed the alignment requirements and fence programming.
@@ -2898,24 +2590,24 @@ struct drm_i915_cmd_table {
2898#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ 2590#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \
2899 !(IS_I915G(dev_priv) || \ 2591 !(IS_I915G(dev_priv) || \
2900 IS_I915GM(dev_priv))) 2592 IS_I915GM(dev_priv)))
2901#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) 2593#define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv)
2902#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) 2594#define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug)
2903 2595
2904#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) 2596#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2)
2905#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) 2597#define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr)
2906#define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) 2598#define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc)
2907 2599
2908#define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) 2600#define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
2909 2601
2910#define HAS_DP_MST(dev) (INTEL_INFO(dev)->has_dp_mst) 2602#define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst)
2911 2603
2912#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) 2604#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi)
2913#define HAS_FPGA_DBG_UNCLAIMED(dev) (INTEL_INFO(dev)->has_fpga_dbg) 2605#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg)
2914#define HAS_PSR(dev) (INTEL_INFO(dev)->has_psr) 2606#define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr)
2915#define HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6) 2607#define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6)
2916#define HAS_RC6p(dev) (INTEL_INFO(dev)->has_rc6p) 2608#define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p)
2917 2609
2918#define HAS_CSR(dev) (INTEL_INFO(dev)->has_csr) 2610#define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr)
2919 2611
2920#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) 2612#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm)
2921#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) 2613#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc)
@@ -2925,13 +2617,13 @@ struct drm_i915_cmd_table {
2925 * command submission once loaded. But these are logically independent 2617 * command submission once loaded. But these are logically independent
2926 * properties, so we have separate macros to test them. 2618 * properties, so we have separate macros to test them.
2927 */ 2619 */
2928#define HAS_GUC(dev) (INTEL_INFO(dev)->has_guc) 2620#define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc)
2929#define HAS_GUC_UCODE(dev) (HAS_GUC(dev)) 2621#define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv))
2930#define HAS_GUC_SCHED(dev) (HAS_GUC(dev)) 2622#define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv))
2931 2623
2932#define HAS_RESOURCE_STREAMER(dev) (INTEL_INFO(dev)->has_resource_streamer) 2624#define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer)
2933 2625
2934#define HAS_POOLED_EU(dev) (INTEL_INFO(dev)->has_pooled_eu) 2626#define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu)
2935 2627
2936#define INTEL_PCH_DEVICE_ID_MASK 0xff00 2628#define INTEL_PCH_DEVICE_ID_MASK 0xff00
2937#define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 2629#define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00
@@ -2971,6 +2663,8 @@ struct drm_i915_cmd_table {
2971#define GT_FREQUENCY_MULTIPLIER 50 2663#define GT_FREQUENCY_MULTIPLIER 50
2972#define GEN9_FREQ_SCALER 3 2664#define GEN9_FREQ_SCALER 3
2973 2665
2666#define HAS_DECOUPLED_MMIO(dev_priv) (INTEL_INFO(dev_priv)->has_decoupled_mmio)
2667
2974#include "i915_trace.h" 2668#include "i915_trace.h"
2975 2669
2976static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) 2670static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
@@ -3222,13 +2916,6 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3222 u64 alignment, 2916 u64 alignment,
3223 u64 flags); 2917 u64 flags);
3224 2918
3225int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
3226 u32 flags);
3227void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
3228int __must_check i915_vma_unbind(struct i915_vma *vma);
3229void i915_vma_close(struct i915_vma *vma);
3230void i915_vma_destroy(struct i915_vma *vma);
3231
3232int i915_gem_object_unbind(struct drm_i915_gem_object *obj); 2919int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
3233void i915_gem_release_mmap(struct drm_i915_gem_object *obj); 2920void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
3234 2921
@@ -3405,10 +3092,10 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error)
3405 3092
3406void i915_gem_reset(struct drm_i915_private *dev_priv); 3093void i915_gem_reset(struct drm_i915_private *dev_priv);
3407void i915_gem_set_wedged(struct drm_i915_private *dev_priv); 3094void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
3408bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); 3095void i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
3409int __must_check i915_gem_init(struct drm_device *dev); 3096int __must_check i915_gem_init(struct drm_device *dev);
3410int __must_check i915_gem_init_hw(struct drm_device *dev); 3097int __must_check i915_gem_init_hw(struct drm_device *dev);
3411void i915_gem_init_swizzling(struct drm_device *dev); 3098void i915_gem_init_swizzling(struct drm_i915_private *dev_priv);
3412void i915_gem_cleanup_engines(struct drm_device *dev); 3099void i915_gem_cleanup_engines(struct drm_device *dev);
3413int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, 3100int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
3414 unsigned int flags); 3101 unsigned int flags);
@@ -3419,6 +3106,11 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj,
3419 unsigned int flags, 3106 unsigned int flags,
3420 long timeout, 3107 long timeout,
3421 struct intel_rps_client *rps); 3108 struct intel_rps_client *rps);
3109int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
3110 unsigned int flags,
3111 int priority);
3112#define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX
3113
3422int __must_check 3114int __must_check
3423i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, 3115i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
3424 bool write); 3116 bool write);
@@ -3478,57 +3170,13 @@ i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o,
3478 return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view)); 3170 return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view));
3479} 3171}
3480 3172
3481/* i915_gem_fence.c */ 3173/* i915_gem_fence_reg.c */
3482int __must_check i915_vma_get_fence(struct i915_vma *vma); 3174int __must_check i915_vma_get_fence(struct i915_vma *vma);
3483int __must_check i915_vma_put_fence(struct i915_vma *vma); 3175int __must_check i915_vma_put_fence(struct i915_vma *vma);
3484 3176
3485/** 3177void i915_gem_restore_fences(struct drm_i915_private *dev_priv);
3486 * i915_vma_pin_fence - pin fencing state
3487 * @vma: vma to pin fencing for
3488 *
3489 * This pins the fencing state (whether tiled or untiled) to make sure the
3490 * vma (and its object) is ready to be used as a scanout target. Fencing
3491 * status must be synchronize first by calling i915_vma_get_fence():
3492 *
3493 * The resulting fence pin reference must be released again with
3494 * i915_vma_unpin_fence().
3495 *
3496 * Returns:
3497 *
3498 * True if the vma has a fence, false otherwise.
3499 */
3500static inline bool
3501i915_vma_pin_fence(struct i915_vma *vma)
3502{
3503 lockdep_assert_held(&vma->vm->dev->struct_mutex);
3504 if (vma->fence) {
3505 vma->fence->pin_count++;
3506 return true;
3507 } else
3508 return false;
3509}
3510
3511/**
3512 * i915_vma_unpin_fence - unpin fencing state
3513 * @vma: vma to unpin fencing for
3514 *
3515 * This releases the fence pin reference acquired through
3516 * i915_vma_pin_fence. It will handle both objects with and without an
3517 * attached fence correctly, callers do not need to distinguish this.
3518 */
3519static inline void
3520i915_vma_unpin_fence(struct i915_vma *vma)
3521{
3522 lockdep_assert_held(&vma->vm->dev->struct_mutex);
3523 if (vma->fence) {
3524 GEM_BUG_ON(vma->fence->pin_count <= 0);
3525 vma->fence->pin_count--;
3526 }
3527}
3528
3529void i915_gem_restore_fences(struct drm_device *dev);
3530 3178
3531void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); 3179void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv);
3532void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, 3180void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
3533 struct sg_table *pages); 3181 struct sg_table *pages);
3534void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, 3182void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
@@ -3631,7 +3279,7 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
3631 u64 end); 3279 u64 end);
3632void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, 3280void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
3633 struct drm_mm_node *node); 3281 struct drm_mm_node *node);
3634int i915_gem_init_stolen(struct drm_device *dev); 3282int i915_gem_init_stolen(struct drm_i915_private *dev_priv);
3635void i915_gem_cleanup_stolen(struct drm_device *dev); 3283void i915_gem_cleanup_stolen(struct drm_device *dev);
3636struct drm_i915_gem_object * 3284struct drm_i915_gem_object *
3637i915_gem_object_create_stolen(struct drm_device *dev, u32 size); 3285i915_gem_object_create_stolen(struct drm_device *dev, u32 size);
@@ -3833,10 +3481,11 @@ extern void intel_modeset_gem_init(struct drm_device *dev);
3833extern void intel_modeset_cleanup(struct drm_device *dev); 3481extern void intel_modeset_cleanup(struct drm_device *dev);
3834extern int intel_connector_register(struct drm_connector *); 3482extern int intel_connector_register(struct drm_connector *);
3835extern void intel_connector_unregister(struct drm_connector *); 3483extern void intel_connector_unregister(struct drm_connector *);
3836extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); 3484extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv,
3485 bool state);
3837extern void intel_display_resume(struct drm_device *dev); 3486extern void intel_display_resume(struct drm_device *dev);
3838extern void i915_redisable_vga(struct drm_device *dev); 3487extern void i915_redisable_vga(struct drm_i915_private *dev_priv);
3839extern void i915_redisable_vga_power_on(struct drm_device *dev); 3488extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv);
3840extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); 3489extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
3841extern void intel_init_pch_refclk(struct drm_device *dev); 3490extern void intel_init_pch_refclk(struct drm_device *dev);
3842extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val); 3491extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
@@ -3855,7 +3504,7 @@ extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e,
3855extern struct intel_display_error_state * 3504extern struct intel_display_error_state *
3856intel_display_capture_error_state(struct drm_i915_private *dev_priv); 3505intel_display_capture_error_state(struct drm_i915_private *dev_priv);
3857extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, 3506extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
3858 struct drm_device *dev, 3507 struct drm_i915_private *dev_priv,
3859 struct intel_display_error_state *error); 3508 struct intel_display_error_state *error);
3860 3509
3861int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); 3510int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 41e697e5dbcd..902fa427c196 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,6 +34,7 @@
34#include "intel_drv.h" 34#include "intel_drv.h"
35#include "intel_frontbuffer.h" 35#include "intel_frontbuffer.h"
36#include "intel_mocs.h" 36#include "intel_mocs.h"
37#include <linux/dma-fence-array.h>
37#include <linux/reservation.h> 38#include <linux/reservation.h>
38#include <linux/shmem_fs.h> 39#include <linux/shmem_fs.h>
39#include <linux/slab.h> 40#include <linux/slab.h>
@@ -48,7 +49,7 @@ static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *o
48static bool cpu_cache_is_coherent(struct drm_device *dev, 49static bool cpu_cache_is_coherent(struct drm_device *dev,
49 enum i915_cache_level level) 50 enum i915_cache_level level)
50{ 51{
51 return HAS_LLC(dev) || level != I915_CACHE_NONE; 52 return HAS_LLC(to_i915(dev)) || level != I915_CACHE_NONE;
52} 53}
53 54
54static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 55static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
@@ -220,15 +221,17 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
220} 221}
221 222
222static void 223static void
223__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj) 224__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
225 struct sg_table *pages)
224{ 226{
225 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); 227 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
226 228
227 if (obj->mm.madv == I915_MADV_DONTNEED) 229 if (obj->mm.madv == I915_MADV_DONTNEED)
228 obj->mm.dirty = false; 230 obj->mm.dirty = false;
229 231
230 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 232 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
231 i915_gem_clflush_object(obj, false); 233 !cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
234 drm_clflush_sg(pages);
232 235
233 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 236 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
234 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 237 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
@@ -238,7 +241,7 @@ static void
238i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 241i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
239 struct sg_table *pages) 242 struct sg_table *pages)
240{ 243{
241 __i915_gem_object_release_shmem(obj); 244 __i915_gem_object_release_shmem(obj, pages);
242 245
243 if (obj->mm.dirty) { 246 if (obj->mm.dirty) {
244 struct address_space *mapping = obj->base.filp->f_mapping; 247 struct address_space *mapping = obj->base.filp->f_mapping;
@@ -433,6 +436,70 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
433 return timeout; 436 return timeout;
434} 437}
435 438
439static void __fence_set_priority(struct dma_fence *fence, int prio)
440{
441 struct drm_i915_gem_request *rq;
442 struct intel_engine_cs *engine;
443
444 if (!dma_fence_is_i915(fence))
445 return;
446
447 rq = to_request(fence);
448 engine = rq->engine;
449 if (!engine->schedule)
450 return;
451
452 engine->schedule(rq, prio);
453}
454
455static void fence_set_priority(struct dma_fence *fence, int prio)
456{
457 /* Recurse once into a fence-array */
458 if (dma_fence_is_array(fence)) {
459 struct dma_fence_array *array = to_dma_fence_array(fence);
460 int i;
461
462 for (i = 0; i < array->num_fences; i++)
463 __fence_set_priority(array->fences[i], prio);
464 } else {
465 __fence_set_priority(fence, prio);
466 }
467}
468
469int
470i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
471 unsigned int flags,
472 int prio)
473{
474 struct dma_fence *excl;
475
476 if (flags & I915_WAIT_ALL) {
477 struct dma_fence **shared;
478 unsigned int count, i;
479 int ret;
480
481 ret = reservation_object_get_fences_rcu(obj->resv,
482 &excl, &count, &shared);
483 if (ret)
484 return ret;
485
486 for (i = 0; i < count; i++) {
487 fence_set_priority(shared[i], prio);
488 dma_fence_put(shared[i]);
489 }
490
491 kfree(shared);
492 } else {
493 excl = reservation_object_get_excl_rcu(obj->resv);
494 }
495
496 if (excl) {
497 fence_set_priority(excl, prio);
498 dma_fence_put(excl);
499 }
500 return 0;
501}
502
436/** 503/**
437 * Waits for rendering to the object to be completed 504 * Waits for rendering to the object to be completed
438 * @obj: i915 gem object 505 * @obj: i915 gem object
@@ -1757,7 +1824,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
1757 goto err_rpm; 1824 goto err_rpm;
1758 1825
1759 /* Access to snoopable pages through the GTT is incoherent. */ 1826 /* Access to snoopable pages through the GTT is incoherent. */
1760 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { 1827 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
1761 ret = -EFAULT; 1828 ret = -EFAULT;
1762 goto err_unlock; 1829 goto err_unlock;
1763 } 1830 }
@@ -2150,7 +2217,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2150 struct sgt_iter sgt_iter; 2217 struct sgt_iter sgt_iter;
2151 struct page *page; 2218 struct page *page;
2152 2219
2153 __i915_gem_object_release_shmem(obj); 2220 __i915_gem_object_release_shmem(obj, pages);
2154 2221
2155 i915_gem_gtt_finish_pages(obj, pages); 2222 i915_gem_gtt_finish_pages(obj, pages);
2156 2223
@@ -2232,6 +2299,30 @@ static unsigned int swiotlb_max_size(void)
2232#endif 2299#endif
2233} 2300}
2234 2301
2302static void i915_sg_trim(struct sg_table *orig_st)
2303{
2304 struct sg_table new_st;
2305 struct scatterlist *sg, *new_sg;
2306 unsigned int i;
2307
2308 if (orig_st->nents == orig_st->orig_nents)
2309 return;
2310
2311 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL))
2312 return;
2313
2314 new_sg = new_st.sgl;
2315 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2316 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2317 /* called before being DMA mapped, no need to copy sg->dma_* */
2318 new_sg = sg_next(new_sg);
2319 }
2320
2321 sg_free_table(orig_st);
2322
2323 *orig_st = new_st;
2324}
2325
2235static struct sg_table * 2326static struct sg_table *
2236i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2327i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2237{ 2328{
@@ -2296,7 +2387,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2296 page = shmem_read_mapping_page(mapping, i); 2387 page = shmem_read_mapping_page(mapping, i);
2297 if (IS_ERR(page)) { 2388 if (IS_ERR(page)) {
2298 ret = PTR_ERR(page); 2389 ret = PTR_ERR(page);
2299 goto err_pages; 2390 goto err_sg;
2300 } 2391 }
2301 } 2392 }
2302 if (!i || 2393 if (!i ||
@@ -2317,6 +2408,9 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2317 if (sg) /* loop terminated early; short sg table */ 2408 if (sg) /* loop terminated early; short sg table */
2318 sg_mark_end(sg); 2409 sg_mark_end(sg);
2319 2410
2411 /* Trim unused sg entries to avoid wasting memory. */
2412 i915_sg_trim(st);
2413
2320 ret = i915_gem_gtt_prepare_pages(obj, st); 2414 ret = i915_gem_gtt_prepare_pages(obj, st);
2321 if (ret) 2415 if (ret)
2322 goto err_pages; 2416 goto err_pages;
@@ -2326,8 +2420,9 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2326 2420
2327 return st; 2421 return st;
2328 2422
2329err_pages: 2423err_sg:
2330 sg_mark_end(sg); 2424 sg_mark_end(sg);
2425err_pages:
2331 for_each_sgt_page(page, sgt_iter, st) 2426 for_each_sgt_page(page, sgt_iter, st)
2332 put_page(page); 2427 put_page(page);
2333 sg_free_table(st); 2428 sg_free_table(st);
@@ -2657,7 +2752,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
2657 for_each_engine(engine, dev_priv, id) 2752 for_each_engine(engine, dev_priv, id)
2658 i915_gem_reset_engine(engine); 2753 i915_gem_reset_engine(engine);
2659 2754
2660 i915_gem_restore_fences(&dev_priv->drm); 2755 i915_gem_restore_fences(dev_priv);
2661 2756
2662 if (dev_priv->gt.awake) { 2757 if (dev_priv->gt.awake) {
2663 intel_sanitize_gt_powersave(dev_priv); 2758 intel_sanitize_gt_powersave(dev_priv);
@@ -2689,12 +2784,17 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
2689 */ 2784 */
2690 2785
2691 if (i915.enable_execlists) { 2786 if (i915.enable_execlists) {
2692 spin_lock(&engine->execlist_lock); 2787 unsigned long flags;
2693 INIT_LIST_HEAD(&engine->execlist_queue); 2788
2789 spin_lock_irqsave(&engine->timeline->lock, flags);
2790
2694 i915_gem_request_put(engine->execlist_port[0].request); 2791 i915_gem_request_put(engine->execlist_port[0].request);
2695 i915_gem_request_put(engine->execlist_port[1].request); 2792 i915_gem_request_put(engine->execlist_port[1].request);
2696 memset(engine->execlist_port, 0, sizeof(engine->execlist_port)); 2793 memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
2697 spin_unlock(&engine->execlist_lock); 2794 engine->execlist_queue = RB_ROOT;
2795 engine->execlist_first = NULL;
2796
2797 spin_unlock_irqrestore(&engine->timeline->lock, flags);
2698 } 2798 }
2699} 2799}
2700 2800
@@ -2892,117 +2992,6 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2892 return ret; 2992 return ret;
2893} 2993}
2894 2994
2895static void __i915_vma_iounmap(struct i915_vma *vma)
2896{
2897 GEM_BUG_ON(i915_vma_is_pinned(vma));
2898
2899 if (vma->iomap == NULL)
2900 return;
2901
2902 io_mapping_unmap(vma->iomap);
2903 vma->iomap = NULL;
2904}
2905
2906int i915_vma_unbind(struct i915_vma *vma)
2907{
2908 struct drm_i915_gem_object *obj = vma->obj;
2909 unsigned long active;
2910 int ret;
2911
2912 lockdep_assert_held(&obj->base.dev->struct_mutex);
2913
2914 /* First wait upon any activity as retiring the request may
2915 * have side-effects such as unpinning or even unbinding this vma.
2916 */
2917 active = i915_vma_get_active(vma);
2918 if (active) {
2919 int idx;
2920
2921 /* When a closed VMA is retired, it is unbound - eek.
2922 * In order to prevent it from being recursively closed,
2923 * take a pin on the vma so that the second unbind is
2924 * aborted.
2925 *
2926 * Even more scary is that the retire callback may free
2927 * the object (last active vma). To prevent the explosion
2928 * we defer the actual object free to a worker that can
2929 * only proceed once it acquires the struct_mutex (which
2930 * we currently hold, therefore it cannot free this object
2931 * before we are finished).
2932 */
2933 __i915_vma_pin(vma);
2934
2935 for_each_active(active, idx) {
2936 ret = i915_gem_active_retire(&vma->last_read[idx],
2937 &vma->vm->dev->struct_mutex);
2938 if (ret)
2939 break;
2940 }
2941
2942 __i915_vma_unpin(vma);
2943 if (ret)
2944 return ret;
2945
2946 GEM_BUG_ON(i915_vma_is_active(vma));
2947 }
2948
2949 if (i915_vma_is_pinned(vma))
2950 return -EBUSY;
2951
2952 if (!drm_mm_node_allocated(&vma->node))
2953 goto destroy;
2954
2955 GEM_BUG_ON(obj->bind_count == 0);
2956 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
2957
2958 if (i915_vma_is_map_and_fenceable(vma)) {
2959 /* release the fence reg _after_ flushing */
2960 ret = i915_vma_put_fence(vma);
2961 if (ret)
2962 return ret;
2963
2964 /* Force a pagefault for domain tracking on next user access */
2965 i915_gem_release_mmap(obj);
2966
2967 __i915_vma_iounmap(vma);
2968 vma->flags &= ~I915_VMA_CAN_FENCE;
2969 }
2970
2971 if (likely(!vma->vm->closed)) {
2972 trace_i915_vma_unbind(vma);
2973 vma->vm->unbind_vma(vma);
2974 }
2975 vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
2976
2977 drm_mm_remove_node(&vma->node);
2978 list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
2979
2980 if (vma->pages != obj->mm.pages) {
2981 GEM_BUG_ON(!vma->pages);
2982 sg_free_table(vma->pages);
2983 kfree(vma->pages);
2984 }
2985 vma->pages = NULL;
2986
2987 /* Since the unbound list is global, only move to that list if
2988 * no more VMAs exist. */
2989 if (--obj->bind_count == 0)
2990 list_move_tail(&obj->global_link,
2991 &to_i915(obj->base.dev)->mm.unbound_list);
2992
2993 /* And finally now the object is completely decoupled from this vma,
2994 * we can drop its hold on the backing storage and allow it to be
2995 * reaped by the shrinker.
2996 */
2997 i915_gem_object_unpin_pages(obj);
2998
2999destroy:
3000 if (unlikely(i915_vma_is_closed(vma)))
3001 i915_vma_destroy(vma);
3002
3003 return 0;
3004}
3005
3006static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags) 2995static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
3007{ 2996{
3008 int ret, i; 2997 int ret, i;
@@ -3018,201 +3007,43 @@ static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
3018 3007
3019int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) 3008int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
3020{ 3009{
3021 struct i915_gem_timeline *tl;
3022 int ret; 3010 int ret;
3023 3011
3024 list_for_each_entry(tl, &i915->gt.timelines, link) { 3012 if (flags & I915_WAIT_LOCKED) {
3025 ret = wait_for_timeline(tl, flags); 3013 struct i915_gem_timeline *tl;
3026 if (ret)
3027 return ret;
3028 }
3029
3030 return 0;
3031}
3032
3033static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3034 unsigned long cache_level)
3035{
3036 struct drm_mm_node *gtt_space = &vma->node;
3037 struct drm_mm_node *other;
3038
3039 /*
3040 * On some machines we have to be careful when putting differing types
3041 * of snoopable memory together to avoid the prefetcher crossing memory
3042 * domains and dying. During vm initialisation, we decide whether or not
3043 * these constraints apply and set the drm_mm.color_adjust
3044 * appropriately.
3045 */
3046 if (vma->vm->mm.color_adjust == NULL)
3047 return true;
3048
3049 if (!drm_mm_node_allocated(gtt_space))
3050 return true;
3051
3052 if (list_empty(&gtt_space->node_list))
3053 return true;
3054
3055 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3056 if (other->allocated && !other->hole_follows && other->color != cache_level)
3057 return false;
3058 3014
3059 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3015 lockdep_assert_held(&i915->drm.struct_mutex);
3060 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3061 return false;
3062 3016
3063 return true; 3017 list_for_each_entry(tl, &i915->gt.timelines, link) {
3064} 3018 ret = wait_for_timeline(tl, flags);
3065
3066/**
3067 * i915_vma_insert - finds a slot for the vma in its address space
3068 * @vma: the vma
3069 * @size: requested size in bytes (can be larger than the VMA)
3070 * @alignment: required alignment
3071 * @flags: mask of PIN_* flags to use
3072 *
3073 * First we try to allocate some free space that meets the requirements for
3074 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
3075 * preferrably the oldest idle entry to make room for the new VMA.
3076 *
3077 * Returns:
3078 * 0 on success, negative error code otherwise.
3079 */
3080static int
3081i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
3082{
3083 struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
3084 struct drm_i915_gem_object *obj = vma->obj;
3085 u64 start, end;
3086 int ret;
3087
3088 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
3089 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
3090
3091 size = max(size, vma->size);
3092 if (flags & PIN_MAPPABLE)
3093 size = i915_gem_get_ggtt_size(dev_priv, size,
3094 i915_gem_object_get_tiling(obj));
3095
3096 alignment = max(max(alignment, vma->display_alignment),
3097 i915_gem_get_ggtt_alignment(dev_priv, size,
3098 i915_gem_object_get_tiling(obj),
3099 flags & PIN_MAPPABLE));
3100
3101 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3102
3103 end = vma->vm->total;
3104 if (flags & PIN_MAPPABLE)
3105 end = min_t(u64, end, dev_priv->ggtt.mappable_end);
3106 if (flags & PIN_ZONE_4G)
3107 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3108
3109 /* If binding the object/GGTT view requires more space than the entire
3110 * aperture has, reject it early before evicting everything in a vain
3111 * attempt to find space.
3112 */
3113 if (size > end) {
3114 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
3115 size, obj->base.size,
3116 flags & PIN_MAPPABLE ? "mappable" : "total",
3117 end);
3118 return -E2BIG;
3119 }
3120
3121 ret = i915_gem_object_pin_pages(obj);
3122 if (ret)
3123 return ret;
3124
3125 if (flags & PIN_OFFSET_FIXED) {
3126 u64 offset = flags & PIN_OFFSET_MASK;
3127 if (offset & (alignment - 1) || offset > end - size) {
3128 ret = -EINVAL;
3129 goto err_unpin;
3130 }
3131
3132 vma->node.start = offset;
3133 vma->node.size = size;
3134 vma->node.color = obj->cache_level;
3135 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3136 if (ret) {
3137 ret = i915_gem_evict_for_vma(vma);
3138 if (ret == 0)
3139 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
3140 if (ret) 3019 if (ret)
3141 goto err_unpin; 3020 return ret;
3142 } 3021 }
3143 } else { 3022 } else {
3144 u32 search_flag, alloc_flag; 3023 ret = wait_for_timeline(&i915->gt.global_timeline, flags);
3145 3024 if (ret)
3146 if (flags & PIN_HIGH) { 3025 return ret;
3147 search_flag = DRM_MM_SEARCH_BELOW;
3148 alloc_flag = DRM_MM_CREATE_TOP;
3149 } else {
3150 search_flag = DRM_MM_SEARCH_DEFAULT;
3151 alloc_flag = DRM_MM_CREATE_DEFAULT;
3152 }
3153
3154 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3155 * so we know that we always have a minimum alignment of 4096.
3156 * The drm_mm range manager is optimised to return results
3157 * with zero alignment, so where possible use the optimal
3158 * path.
3159 */
3160 if (alignment <= 4096)
3161 alignment = 0;
3162
3163search_free:
3164 ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
3165 &vma->node,
3166 size, alignment,
3167 obj->cache_level,
3168 start, end,
3169 search_flag,
3170 alloc_flag);
3171 if (ret) {
3172 ret = i915_gem_evict_something(vma->vm, size, alignment,
3173 obj->cache_level,
3174 start, end,
3175 flags);
3176 if (ret == 0)
3177 goto search_free;
3178
3179 goto err_unpin;
3180 }
3181
3182 GEM_BUG_ON(vma->node.start < start);
3183 GEM_BUG_ON(vma->node.start + vma->node.size > end);
3184 } 3026 }
3185 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
3186
3187 list_move_tail(&obj->global_link, &dev_priv->mm.bound_list);
3188 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3189 obj->bind_count++;
3190 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
3191 3027
3192 return 0; 3028 return 0;
3193
3194err_unpin:
3195 i915_gem_object_unpin_pages(obj);
3196 return ret;
3197} 3029}
3198 3030
3199bool 3031void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3200i915_gem_clflush_object(struct drm_i915_gem_object *obj, 3032 bool force)
3201 bool force)
3202{ 3033{
3203 /* If we don't have a page list set up, then we're not pinned 3034 /* If we don't have a page list set up, then we're not pinned
3204 * to GPU, and we can ignore the cache flush because it'll happen 3035 * to GPU, and we can ignore the cache flush because it'll happen
3205 * again at bind time. 3036 * again at bind time.
3206 */ 3037 */
3207 if (!obj->mm.pages) 3038 if (!obj->mm.pages)
3208 return false; 3039 return;
3209 3040
3210 /* 3041 /*
3211 * Stolen memory is always coherent with the GPU as it is explicitly 3042 * Stolen memory is always coherent with the GPU as it is explicitly
3212 * marked as wc by the system, or the system is cache-coherent. 3043 * marked as wc by the system, or the system is cache-coherent.
3213 */ 3044 */
3214 if (obj->stolen || obj->phys_handle) 3045 if (obj->stolen || obj->phys_handle)
3215 return false; 3046 return;
3216 3047
3217 /* If the GPU is snooping the contents of the CPU cache, 3048 /* If the GPU is snooping the contents of the CPU cache,
3218 * we do not need to manually clear the CPU cache lines. However, 3049 * we do not need to manually clear the CPU cache lines. However,
@@ -3224,14 +3055,12 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3224 */ 3055 */
3225 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { 3056 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3226 obj->cache_dirty = true; 3057 obj->cache_dirty = true;
3227 return false; 3058 return;
3228 } 3059 }
3229 3060
3230 trace_i915_gem_object_clflush(obj); 3061 trace_i915_gem_object_clflush(obj);
3231 drm_clflush_sg(obj->mm.pages); 3062 drm_clflush_sg(obj->mm.pages);
3232 obj->cache_dirty = false; 3063 obj->cache_dirty = false;
3233
3234 return true;
3235} 3064}
3236 3065
3237/** Flushes the GTT write domain for the object if it's dirty. */ 3066/** Flushes the GTT write domain for the object if it's dirty. */
@@ -3277,9 +3106,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3277 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 3106 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3278 return; 3107 return;
3279 3108
3280 if (i915_gem_clflush_object(obj, obj->pin_display)) 3109 i915_gem_clflush_object(obj, obj->pin_display);
3281 i915_gem_chipset_flush(to_i915(obj->base.dev));
3282
3283 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 3110 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3284 3111
3285 obj->base.write_domain = 0; 3112 obj->base.write_domain = 0;
@@ -3378,12 +3205,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3378 enum i915_cache_level cache_level) 3205 enum i915_cache_level cache_level)
3379{ 3206{
3380 struct i915_vma *vma; 3207 struct i915_vma *vma;
3381 int ret = 0; 3208 int ret;
3382 3209
3383 lockdep_assert_held(&obj->base.dev->struct_mutex); 3210 lockdep_assert_held(&obj->base.dev->struct_mutex);
3384 3211
3385 if (obj->cache_level == cache_level) 3212 if (obj->cache_level == cache_level)
3386 goto out; 3213 return 0;
3387 3214
3388 /* Inspect the list of currently bound VMA and unbind any that would 3215 /* Inspect the list of currently bound VMA and unbind any that would
3389 * be invalid given the new cache-level. This is principally to 3216 * be invalid given the new cache-level. This is principally to
@@ -3435,7 +3262,8 @@ restart:
3435 if (ret) 3262 if (ret)
3436 return ret; 3263 return ret;
3437 3264
3438 if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) { 3265 if (!HAS_LLC(to_i915(obj->base.dev)) &&
3266 cache_level != I915_CACHE_NONE) {
3439 /* Access to snoopable pages through the GTT is 3267 /* Access to snoopable pages through the GTT is
3440 * incoherent and on some machines causes a hard 3268 * incoherent and on some machines causes a hard
3441 * lockup. Relinquish the CPU mmaping to force 3269 * lockup. Relinquish the CPU mmaping to force
@@ -3477,20 +3305,14 @@ restart:
3477 } 3305 }
3478 } 3306 }
3479 3307
3308 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU &&
3309 cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3310 obj->cache_dirty = true;
3311
3480 list_for_each_entry(vma, &obj->vma_list, obj_link) 3312 list_for_each_entry(vma, &obj->vma_list, obj_link)
3481 vma->node.color = cache_level; 3313 vma->node.color = cache_level;
3482 obj->cache_level = cache_level; 3314 obj->cache_level = cache_level;
3483 3315
3484out:
3485 /* Flush the dirty CPU caches to the backing storage so that the
3486 * object is now coherent at its new cache level (with respect
3487 * to the access domain).
3488 */
3489 if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3490 if (i915_gem_clflush_object(obj, true))
3491 i915_gem_chipset_flush(to_i915(obj->base.dev));
3492 }
3493
3494 return 0; 3316 return 0;
3495} 3317}
3496 3318
@@ -3646,7 +3468,11 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3646 3468
3647 vma->display_alignment = max_t(u64, vma->display_alignment, alignment); 3469 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
3648 3470
3649 i915_gem_object_flush_cpu_write_domain(obj); 3471 /* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
3472 if (obj->cache_dirty) {
3473 i915_gem_clflush_object(obj, true);
3474 intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
3475 }
3650 3476
3651 old_write_domain = obj->base.write_domain; 3477 old_write_domain = obj->base.write_domain;
3652 old_read_domains = obj->base.read_domains; 3478 old_read_domains = obj->base.read_domains;
@@ -3798,100 +3624,6 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3798 return ret < 0 ? ret : 0; 3624 return ret < 0 ? ret : 0;
3799} 3625}
3800 3626
3801static bool
3802i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
3803{
3804 if (!drm_mm_node_allocated(&vma->node))
3805 return false;
3806
3807 if (vma->node.size < size)
3808 return true;
3809
3810 if (alignment && vma->node.start & (alignment - 1))
3811 return true;
3812
3813 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
3814 return true;
3815
3816 if (flags & PIN_OFFSET_BIAS &&
3817 vma->node.start < (flags & PIN_OFFSET_MASK))
3818 return true;
3819
3820 if (flags & PIN_OFFSET_FIXED &&
3821 vma->node.start != (flags & PIN_OFFSET_MASK))
3822 return true;
3823
3824 return false;
3825}
3826
3827void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3828{
3829 struct drm_i915_gem_object *obj = vma->obj;
3830 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3831 bool mappable, fenceable;
3832 u32 fence_size, fence_alignment;
3833
3834 fence_size = i915_gem_get_ggtt_size(dev_priv,
3835 vma->size,
3836 i915_gem_object_get_tiling(obj));
3837 fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
3838 vma->size,
3839 i915_gem_object_get_tiling(obj),
3840 true);
3841
3842 fenceable = (vma->node.size == fence_size &&
3843 (vma->node.start & (fence_alignment - 1)) == 0);
3844
3845 mappable = (vma->node.start + fence_size <=
3846 dev_priv->ggtt.mappable_end);
3847
3848 /*
3849 * Explicitly disable for rotated VMA since the display does not
3850 * need the fence and the VMA is not accessible to other users.
3851 */
3852 if (mappable && fenceable &&
3853 vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED)
3854 vma->flags |= I915_VMA_CAN_FENCE;
3855 else
3856 vma->flags &= ~I915_VMA_CAN_FENCE;
3857}
3858
3859int __i915_vma_do_pin(struct i915_vma *vma,
3860 u64 size, u64 alignment, u64 flags)
3861{
3862 unsigned int bound = vma->flags;
3863 int ret;
3864
3865 lockdep_assert_held(&vma->vm->dev->struct_mutex);
3866 GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
3867 GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
3868
3869 if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
3870 ret = -EBUSY;
3871 goto err;
3872 }
3873
3874 if ((bound & I915_VMA_BIND_MASK) == 0) {
3875 ret = i915_vma_insert(vma, size, alignment, flags);
3876 if (ret)
3877 goto err;
3878 }
3879
3880 ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
3881 if (ret)
3882 goto err;
3883
3884 if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
3885 __i915_vma_set_map_and_fenceable(vma);
3886
3887 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
3888 return 0;
3889
3890err:
3891 __i915_vma_unpin(vma);
3892 return ret;
3893}
3894
3895struct i915_vma * 3627struct i915_vma *
3896i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 3628i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3897 const struct i915_ggtt_view *view, 3629 const struct i915_ggtt_view *view,
@@ -4156,6 +3888,16 @@ out:
4156 return err; 3888 return err;
4157} 3889}
4158 3890
3891static void
3892frontbuffer_retire(struct i915_gem_active *active,
3893 struct drm_i915_gem_request *request)
3894{
3895 struct drm_i915_gem_object *obj =
3896 container_of(active, typeof(*obj), frontbuffer_write);
3897
3898 intel_fb_obj_flush(obj, true, ORIGIN_CS);
3899}
3900
4159void i915_gem_object_init(struct drm_i915_gem_object *obj, 3901void i915_gem_object_init(struct drm_i915_gem_object *obj,
4160 const struct drm_i915_gem_object_ops *ops) 3902 const struct drm_i915_gem_object_ops *ops)
4161{ 3903{
@@ -4173,6 +3915,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
4173 obj->resv = &obj->__builtin_resv; 3915 obj->resv = &obj->__builtin_resv;
4174 3916
4175 obj->frontbuffer_ggtt_origin = ORIGIN_GTT; 3917 obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
3918 init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
4176 3919
4177 obj->mm.madv = I915_MADV_WILLNEED; 3920 obj->mm.madv = I915_MADV_WILLNEED;
4178 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); 3921 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
@@ -4235,7 +3978,7 @@ i915_gem_object_create(struct drm_device *dev, u64 size)
4235 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3978 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4236 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3979 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4237 3980
4238 if (HAS_LLC(dev)) { 3981 if (HAS_LLC(dev_priv)) {
4239 /* On some devices, we can have the GPU use the LLC (the CPU 3982 /* On some devices, we can have the GPU use the LLC (the CPU
4240 * cache) for about a 10% performance improvement 3983 * cache) for about a 10% performance improvement
4241 * compared to uncached. Graphics requests other than 3984 * compared to uncached. Graphics requests other than
@@ -4481,7 +4224,7 @@ int i915_gem_suspend(struct drm_device *dev)
4481 * machines is a good idea, we don't - just in case it leaves the 4224 * machines is a good idea, we don't - just in case it leaves the
4482 * machine in an unusable condition. 4225 * machine in an unusable condition.
4483 */ 4226 */
4484 if (HAS_HW_CONTEXTS(dev)) { 4227 if (HAS_HW_CONTEXTS(dev_priv)) {
4485 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES); 4228 int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
4486 WARN_ON(reset && reset != -ENODEV); 4229 WARN_ON(reset && reset != -ENODEV);
4487 } 4230 }
@@ -4500,7 +4243,7 @@ void i915_gem_resume(struct drm_device *dev)
4500 WARN_ON(dev_priv->gt.awake); 4243 WARN_ON(dev_priv->gt.awake);
4501 4244
4502 mutex_lock(&dev->struct_mutex); 4245 mutex_lock(&dev->struct_mutex);
4503 i915_gem_restore_gtt_mappings(dev); 4246 i915_gem_restore_gtt_mappings(dev_priv);
4504 4247
4505 /* As we didn't flush the kernel context before suspend, we cannot 4248 /* As we didn't flush the kernel context before suspend, we cannot
4506 * guarantee that the context image is complete. So let's just reset 4249 * guarantee that the context image is complete. So let's just reset
@@ -4511,11 +4254,9 @@ void i915_gem_resume(struct drm_device *dev)
4511 mutex_unlock(&dev->struct_mutex); 4254 mutex_unlock(&dev->struct_mutex);
4512} 4255}
4513 4256
4514void i915_gem_init_swizzling(struct drm_device *dev) 4257void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
4515{ 4258{
4516 struct drm_i915_private *dev_priv = to_i915(dev); 4259 if (INTEL_GEN(dev_priv) < 5 ||
4517
4518 if (INTEL_INFO(dev)->gen < 5 ||
4519 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 4260 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4520 return; 4261 return;
4521 4262
@@ -4574,7 +4315,7 @@ i915_gem_init_hw(struct drm_device *dev)
4574 /* Double layer security blanket, see i915_gem_init() */ 4315 /* Double layer security blanket, see i915_gem_init() */
4575 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 4316 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4576 4317
4577 if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9) 4318 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
4578 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 4319 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4579 4320
4580 if (IS_HASWELL(dev_priv)) 4321 if (IS_HASWELL(dev_priv))
@@ -4586,14 +4327,14 @@ i915_gem_init_hw(struct drm_device *dev)
4586 u32 temp = I915_READ(GEN7_MSG_CTL); 4327 u32 temp = I915_READ(GEN7_MSG_CTL);
4587 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 4328 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4588 I915_WRITE(GEN7_MSG_CTL, temp); 4329 I915_WRITE(GEN7_MSG_CTL, temp);
4589 } else if (INTEL_INFO(dev)->gen >= 7) { 4330 } else if (INTEL_GEN(dev_priv) >= 7) {
4590 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); 4331 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4591 temp &= ~RESET_PCH_HANDSHAKE_ENABLE; 4332 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4592 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); 4333 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4593 } 4334 }
4594 } 4335 }
4595 4336
4596 i915_gem_init_swizzling(dev); 4337 i915_gem_init_swizzling(dev_priv);
4597 4338
4598 /* 4339 /*
4599 * At least 830 can leave some of the unused rings 4340 * At least 830 can leave some of the unused rings
@@ -4605,7 +4346,7 @@ i915_gem_init_hw(struct drm_device *dev)
4605 4346
4606 BUG_ON(!dev_priv->kernel_context); 4347 BUG_ON(!dev_priv->kernel_context);
4607 4348
4608 ret = i915_ppgtt_init_hw(dev); 4349 ret = i915_ppgtt_init_hw(dev_priv);
4609 if (ret) { 4350 if (ret) {
4610 DRM_ERROR("PPGTT enable HW failed %d\n", ret); 4351 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4611 goto out; 4352 goto out;
@@ -4720,7 +4461,6 @@ i915_gem_cleanup_engines(struct drm_device *dev)
4720void 4461void
4721i915_gem_load_init_fences(struct drm_i915_private *dev_priv) 4462i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4722{ 4463{
4723 struct drm_device *dev = &dev_priv->drm;
4724 int i; 4464 int i;
4725 4465
4726 if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) && 4466 if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
@@ -4744,9 +4484,9 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4744 fence->id = i; 4484 fence->id = i;
4745 list_add_tail(&fence->link, &dev_priv->mm.fence_list); 4485 list_add_tail(&fence->link, &dev_priv->mm.fence_list);
4746 } 4486 }
4747 i915_gem_restore_fences(dev); 4487 i915_gem_restore_fences(dev_priv);
4748 4488
4749 i915_gem_detect_bit_6_swizzle(dev); 4489 i915_gem_detect_bit_6_swizzle(dev_priv);
4750} 4490}
4751 4491
4752int 4492int
@@ -4770,14 +4510,18 @@ i915_gem_load_init(struct drm_device *dev)
4770 if (!dev_priv->requests) 4510 if (!dev_priv->requests)
4771 goto err_vmas; 4511 goto err_vmas;
4772 4512
4513 dev_priv->dependencies = KMEM_CACHE(i915_dependency,
4514 SLAB_HWCACHE_ALIGN |
4515 SLAB_RECLAIM_ACCOUNT);
4516 if (!dev_priv->dependencies)
4517 goto err_requests;
4518
4773 mutex_lock(&dev_priv->drm.struct_mutex); 4519 mutex_lock(&dev_priv->drm.struct_mutex);
4774 INIT_LIST_HEAD(&dev_priv->gt.timelines); 4520 INIT_LIST_HEAD(&dev_priv->gt.timelines);
4775 err = i915_gem_timeline_init(dev_priv, 4521 err = i915_gem_timeline_init__global(dev_priv);
4776 &dev_priv->gt.global_timeline,
4777 "[execution]");
4778 mutex_unlock(&dev_priv->drm.struct_mutex); 4522 mutex_unlock(&dev_priv->drm.struct_mutex);
4779 if (err) 4523 if (err)
4780 goto err_requests; 4524 goto err_dependencies;
4781 4525
4782 INIT_LIST_HEAD(&dev_priv->context_list); 4526 INIT_LIST_HEAD(&dev_priv->context_list);
4783 INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); 4527 INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
@@ -4805,6 +4549,8 @@ i915_gem_load_init(struct drm_device *dev)
4805 4549
4806 return 0; 4550 return 0;
4807 4551
4552err_dependencies:
4553 kmem_cache_destroy(dev_priv->dependencies);
4808err_requests: 4554err_requests:
4809 kmem_cache_destroy(dev_priv->requests); 4555 kmem_cache_destroy(dev_priv->requests);
4810err_vmas: 4556err_vmas:
@@ -4821,6 +4567,12 @@ void i915_gem_load_cleanup(struct drm_device *dev)
4821 4567
4822 WARN_ON(!llist_empty(&dev_priv->mm.free_list)); 4568 WARN_ON(!llist_empty(&dev_priv->mm.free_list));
4823 4569
4570 mutex_lock(&dev_priv->drm.struct_mutex);
4571 i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
4572 WARN_ON(!list_empty(&dev_priv->gt.timelines));
4573 mutex_unlock(&dev_priv->drm.struct_mutex);
4574
4575 kmem_cache_destroy(dev_priv->dependencies);
4824 kmem_cache_destroy(dev_priv->requests); 4576 kmem_cache_destroy(dev_priv->requests);
4825 kmem_cache_destroy(dev_priv->vmas); 4577 kmem_cache_destroy(dev_priv->vmas);
4826 kmem_cache_destroy(dev_priv->objects); 4578 kmem_cache_destroy(dev_priv->objects);
@@ -4905,7 +4657,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4905 struct drm_i915_file_private *file_priv; 4657 struct drm_i915_file_private *file_priv;
4906 int ret; 4658 int ret;
4907 4659
4908 DRM_DEBUG_DRIVER("\n"); 4660 DRM_DEBUG("\n");
4909 4661
4910 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 4662 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4911 if (!file_priv) 4663 if (!file_priv)
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 735580d72eb1..51ec793f2e20 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -28,7 +28,7 @@
28#ifdef CONFIG_DRM_I915_DEBUG_GEM 28#ifdef CONFIG_DRM_I915_DEBUG_GEM
29#define GEM_BUG_ON(expr) BUG_ON(expr) 29#define GEM_BUG_ON(expr) BUG_ON(expr)
30#else 30#else
31#define GEM_BUG_ON(expr) 31#define GEM_BUG_ON(expr) do { } while (0)
32#endif 32#endif
33 33
34#define I915_NUM_ENGINES 5 34#define I915_NUM_ENGINES 5
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 6dd475735f0a..1f94b8d6d83d 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -476,6 +476,7 @@ int i915_gem_context_init(struct drm_device *dev)
476 return PTR_ERR(ctx); 476 return PTR_ERR(ctx);
477 } 477 }
478 478
479 ctx->priority = I915_PRIORITY_MIN; /* lowest priority; idle task */
479 dev_priv->kernel_context = ctx; 480 dev_priv->kernel_context = ctx;
480 481
481 DRM_DEBUG_DRIVER("%s context support initialized\n", 482 DRM_DEBUG_DRIVER("%s context support initialized\n",
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index fb5b44339f71..097d9d8c2315 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -287,7 +287,7 @@ static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
287 if (DBG_USE_CPU_RELOC) 287 if (DBG_USE_CPU_RELOC)
288 return DBG_USE_CPU_RELOC > 0; 288 return DBG_USE_CPU_RELOC > 0;
289 289
290 return (HAS_LLC(obj->base.dev) || 290 return (HAS_LLC(to_i915(obj->base.dev)) ||
291 obj->base.write_domain == I915_GEM_DOMAIN_CPU || 291 obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
292 obj->cache_level != I915_CACHE_NONE); 292 obj->cache_level != I915_CACHE_NONE);
293} 293}
@@ -833,7 +833,7 @@ need_reloc_mappable(struct i915_vma *vma)
833 return false; 833 return false;
834 834
835 /* See also use_cpu_reloc() */ 835 /* See also use_cpu_reloc() */
836 if (HAS_LLC(vma->obj->base.dev)) 836 if (HAS_LLC(to_i915(vma->obj->base.dev)))
837 return false; 837 return false;
838 838
839 if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU) 839 if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
@@ -1276,9 +1276,8 @@ void i915_vma_move_to_active(struct i915_vma *vma,
1276 list_move_tail(&vma->vm_link, &vma->vm->active_list); 1276 list_move_tail(&vma->vm_link, &vma->vm->active_list);
1277 1277
1278 if (flags & EXEC_OBJECT_WRITE) { 1278 if (flags & EXEC_OBJECT_WRITE) {
1279 i915_gem_active_set(&vma->last_write, req); 1279 if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
1280 1280 i915_gem_active_set(&obj->frontbuffer_write, req);
1281 intel_fb_obj_invalidate(obj, ORIGIN_CS);
1282 1281
1283 /* update for the implicit flush after a batch */ 1282 /* update for the implicit flush after a batch */
1284 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; 1283 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
@@ -1624,7 +1623,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1624 } 1623 }
1625 1624
1626 if (args->flags & I915_EXEC_RESOURCE_STREAMER) { 1625 if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1627 if (!HAS_RESOURCE_STREAMER(dev)) { 1626 if (!HAS_RESOURCE_STREAMER(dev_priv)) {
1628 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n"); 1627 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1629 return -EINVAL; 1628 return -EINVAL;
1630 } 1629 }
@@ -1878,7 +1877,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1878 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; 1877 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1879 exec2_list[i].alignment = exec_list[i].alignment; 1878 exec2_list[i].alignment = exec_list[i].alignment;
1880 exec2_list[i].offset = exec_list[i].offset; 1879 exec2_list[i].offset = exec_list[i].offset;
1881 if (INTEL_INFO(dev)->gen < 4) 1880 if (INTEL_GEN(to_i915(dev)) < 4)
1882 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; 1881 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1883 else 1882 else
1884 exec2_list[i].flags = 0; 1883 exec2_list[i].flags = 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
index cd59dbc6588c..0efa3571afc3 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
@@ -368,15 +368,14 @@ i915_vma_get_fence(struct i915_vma *vma)
368 368
369/** 369/**
370 * i915_gem_restore_fences - restore fence state 370 * i915_gem_restore_fences - restore fence state
371 * @dev: DRM device 371 * @dev_priv: i915 device private
372 * 372 *
373 * Restore the hw fence state to match the software tracking again, to be called 373 * Restore the hw fence state to match the software tracking again, to be called
374 * after a gpu reset and on resume. Note that on runtime suspend we only cancel 374 * after a gpu reset and on resume. Note that on runtime suspend we only cancel
375 * the fences, to be reacquired by the user later. 375 * the fences, to be reacquired by the user later.
376 */ 376 */
377void i915_gem_restore_fences(struct drm_device *dev) 377void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
378{ 378{
379 struct drm_i915_private *dev_priv = to_i915(dev);
380 int i; 379 int i;
381 380
382 for (i = 0; i < dev_priv->num_fence_regs; i++) { 381 for (i = 0; i < dev_priv->num_fence_regs; i++) {
@@ -451,15 +450,14 @@ void i915_gem_restore_fences(struct drm_device *dev)
451 450
452/** 451/**
453 * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern 452 * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern
454 * @dev: DRM device 453 * @dev_priv: i915 device private
455 * 454 *
456 * Detects bit 6 swizzling of address lookup between IGD access and CPU 455 * Detects bit 6 swizzling of address lookup between IGD access and CPU
457 * access through main memory. 456 * access through main memory.
458 */ 457 */
459void 458void
460i915_gem_detect_bit_6_swizzle(struct drm_device *dev) 459i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv)
461{ 460{
462 struct drm_i915_private *dev_priv = to_i915(dev);
463 uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; 461 uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
464 uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; 462 uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
465 463
@@ -473,7 +471,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
473 */ 471 */
474 swizzle_x = I915_BIT_6_SWIZZLE_NONE; 472 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
475 swizzle_y = I915_BIT_6_SWIZZLE_NONE; 473 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
476 } else if (INTEL_INFO(dev)->gen >= 6) { 474 } else if (INTEL_GEN(dev_priv) >= 6) {
477 if (dev_priv->preserve_bios_swizzle) { 475 if (dev_priv->preserve_bios_swizzle) {
478 if (I915_READ(DISP_ARB_CTL) & 476 if (I915_READ(DISP_ARB_CTL) &
479 DISP_TILE_SURFACE_SWIZZLING) { 477 DISP_TILE_SURFACE_SWIZZLING) {
diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.h b/drivers/gpu/drm/i915/i915_gem_fence_reg.h
new file mode 100644
index 000000000000..22c4a2d01adf
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.h
@@ -0,0 +1,51 @@
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#ifndef __I915_FENCE_REG_H__
26#define __I915_FENCE_REG_H__
27
28#include <linux/list.h>
29
30struct drm_i915_private;
31struct i915_vma;
32
33struct drm_i915_fence_reg {
34 struct list_head link;
35 struct drm_i915_private *i915;
36 struct i915_vma *vma;
37 int pin_count;
38 int id;
39 /**
40 * Whether the tiling parameters for the currently
41 * associated fence register have changed. Note that
42 * for the purposes of tracking tiling changes we also
43 * treat the unfenced register, the register slot that
44 * the object occupies whilst it executes a fenced
45 * command (such as BLT on gen2/3), as a "fence".
46 */
47 bool dirty;
48};
49
50#endif
51
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index a5fafa3d4fc8..b4bde1452f2a 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -96,13 +96,6 @@
96 * 96 *
97 */ 97 */
98 98
99static inline struct i915_ggtt *
100i915_vm_to_ggtt(struct i915_address_space *vm)
101{
102 GEM_BUG_ON(!i915_is_ggtt(vm));
103 return container_of(vm, struct i915_ggtt, base);
104}
105
106static int 99static int
107i915_get_ggtt_vma_pages(struct i915_vma *vma); 100i915_get_ggtt_vma_pages(struct i915_vma *vma);
108 101
@@ -327,10 +320,10 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
327 return pte; 320 return pte;
328} 321}
329 322
330static int __setup_page_dma(struct drm_device *dev, 323static int __setup_page_dma(struct drm_i915_private *dev_priv,
331 struct i915_page_dma *p, gfp_t flags) 324 struct i915_page_dma *p, gfp_t flags)
332{ 325{
333 struct device *kdev = &dev->pdev->dev; 326 struct device *kdev = &dev_priv->drm.pdev->dev;
334 327
335 p->page = alloc_page(flags); 328 p->page = alloc_page(flags);
336 if (!p->page) 329 if (!p->page)
@@ -347,14 +340,16 @@ static int __setup_page_dma(struct drm_device *dev,
347 return 0; 340 return 0;
348} 341}
349 342
350static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 343static int setup_page_dma(struct drm_i915_private *dev_priv,
344 struct i915_page_dma *p)
351{ 345{
352 return __setup_page_dma(dev, p, I915_GFP_DMA); 346 return __setup_page_dma(dev_priv, p, I915_GFP_DMA);
353} 347}
354 348
355static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) 349static void cleanup_page_dma(struct drm_i915_private *dev_priv,
350 struct i915_page_dma *p)
356{ 351{
357 struct pci_dev *pdev = dev->pdev; 352 struct pci_dev *pdev = dev_priv->drm.pdev;
358 353
359 if (WARN_ON(!p->page)) 354 if (WARN_ON(!p->page))
360 return; 355 return;
@@ -387,8 +382,8 @@ static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
387#define kunmap_px(ppgtt, vaddr) \ 382#define kunmap_px(ppgtt, vaddr) \
388 kunmap_page_dma(to_i915((ppgtt)->base.dev), (vaddr)) 383 kunmap_page_dma(to_i915((ppgtt)->base.dev), (vaddr))
389 384
390#define setup_px(dev, px) setup_page_dma((dev), px_base(px)) 385#define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px))
391#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) 386#define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px))
392#define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v)) 387#define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
393#define fill32_px(dev_priv, px, v) \ 388#define fill32_px(dev_priv, px, v) \
394 fill_page_dma_32((dev_priv), px_base(px), (v)) 389 fill_page_dma_32((dev_priv), px_base(px), (v))
@@ -416,24 +411,23 @@ static void fill_page_dma_32(struct drm_i915_private *dev_priv,
416} 411}
417 412
418static int 413static int
419setup_scratch_page(struct drm_device *dev, 414setup_scratch_page(struct drm_i915_private *dev_priv,
420 struct i915_page_dma *scratch, 415 struct i915_page_dma *scratch,
421 gfp_t gfp) 416 gfp_t gfp)
422{ 417{
423 return __setup_page_dma(dev, scratch, gfp | __GFP_ZERO); 418 return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO);
424} 419}
425 420
426static void cleanup_scratch_page(struct drm_device *dev, 421static void cleanup_scratch_page(struct drm_i915_private *dev_priv,
427 struct i915_page_dma *scratch) 422 struct i915_page_dma *scratch)
428{ 423{
429 cleanup_page_dma(dev, scratch); 424 cleanup_page_dma(dev_priv, scratch);
430} 425}
431 426
432static struct i915_page_table *alloc_pt(struct drm_device *dev) 427static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv)
433{ 428{
434 struct i915_page_table *pt; 429 struct i915_page_table *pt;
435 const size_t count = INTEL_INFO(dev)->gen >= 8 ? 430 const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES;
436 GEN8_PTES : GEN6_PTES;
437 int ret = -ENOMEM; 431 int ret = -ENOMEM;
438 432
439 pt = kzalloc(sizeof(*pt), GFP_KERNEL); 433 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
@@ -446,7 +440,7 @@ static struct i915_page_table *alloc_pt(struct drm_device *dev)
446 if (!pt->used_ptes) 440 if (!pt->used_ptes)
447 goto fail_bitmap; 441 goto fail_bitmap;
448 442
449 ret = setup_px(dev, pt); 443 ret = setup_px(dev_priv, pt);
450 if (ret) 444 if (ret)
451 goto fail_page_m; 445 goto fail_page_m;
452 446
@@ -460,9 +454,10 @@ fail_bitmap:
460 return ERR_PTR(ret); 454 return ERR_PTR(ret);
461} 455}
462 456
463static void free_pt(struct drm_device *dev, struct i915_page_table *pt) 457static void free_pt(struct drm_i915_private *dev_priv,
458 struct i915_page_table *pt)
464{ 459{
465 cleanup_px(dev, pt); 460 cleanup_px(dev_priv, pt);
466 kfree(pt->used_ptes); 461 kfree(pt->used_ptes);
467 kfree(pt); 462 kfree(pt);
468} 463}
@@ -491,7 +486,7 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
491 fill32_px(to_i915(vm->dev), pt, scratch_pte); 486 fill32_px(to_i915(vm->dev), pt, scratch_pte);
492} 487}
493 488
494static struct i915_page_directory *alloc_pd(struct drm_device *dev) 489static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv)
495{ 490{
496 struct i915_page_directory *pd; 491 struct i915_page_directory *pd;
497 int ret = -ENOMEM; 492 int ret = -ENOMEM;
@@ -505,7 +500,7 @@ static struct i915_page_directory *alloc_pd(struct drm_device *dev)
505 if (!pd->used_pdes) 500 if (!pd->used_pdes)
506 goto fail_bitmap; 501 goto fail_bitmap;
507 502
508 ret = setup_px(dev, pd); 503 ret = setup_px(dev_priv, pd);
509 if (ret) 504 if (ret)
510 goto fail_page_m; 505 goto fail_page_m;
511 506
@@ -519,10 +514,11 @@ fail_bitmap:
519 return ERR_PTR(ret); 514 return ERR_PTR(ret);
520} 515}
521 516
522static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) 517static void free_pd(struct drm_i915_private *dev_priv,
518 struct i915_page_directory *pd)
523{ 519{
524 if (px_page(pd)) { 520 if (px_page(pd)) {
525 cleanup_px(dev, pd); 521 cleanup_px(dev_priv, pd);
526 kfree(pd->used_pdes); 522 kfree(pd->used_pdes);
527 kfree(pd); 523 kfree(pd);
528 } 524 }
@@ -538,10 +534,10 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
538 fill_px(to_i915(vm->dev), pd, scratch_pde); 534 fill_px(to_i915(vm->dev), pd, scratch_pde);
539} 535}
540 536
541static int __pdp_init(struct drm_device *dev, 537static int __pdp_init(struct drm_i915_private *dev_priv,
542 struct i915_page_directory_pointer *pdp) 538 struct i915_page_directory_pointer *pdp)
543{ 539{
544 size_t pdpes = I915_PDPES_PER_PDP(dev); 540 size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
545 541
546 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), 542 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
547 sizeof(unsigned long), 543 sizeof(unsigned long),
@@ -570,22 +566,22 @@ static void __pdp_fini(struct i915_page_directory_pointer *pdp)
570} 566}
571 567
572static struct 568static struct
573i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) 569i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv)
574{ 570{
575 struct i915_page_directory_pointer *pdp; 571 struct i915_page_directory_pointer *pdp;
576 int ret = -ENOMEM; 572 int ret = -ENOMEM;
577 573
578 WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); 574 WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv));
579 575
580 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); 576 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
581 if (!pdp) 577 if (!pdp)
582 return ERR_PTR(-ENOMEM); 578 return ERR_PTR(-ENOMEM);
583 579
584 ret = __pdp_init(dev, pdp); 580 ret = __pdp_init(dev_priv, pdp);
585 if (ret) 581 if (ret)
586 goto fail_bitmap; 582 goto fail_bitmap;
587 583
588 ret = setup_px(dev, pdp); 584 ret = setup_px(dev_priv, pdp);
589 if (ret) 585 if (ret)
590 goto fail_page_m; 586 goto fail_page_m;
591 587
@@ -599,12 +595,12 @@ fail_bitmap:
599 return ERR_PTR(ret); 595 return ERR_PTR(ret);
600} 596}
601 597
602static void free_pdp(struct drm_device *dev, 598static void free_pdp(struct drm_i915_private *dev_priv,
603 struct i915_page_directory_pointer *pdp) 599 struct i915_page_directory_pointer *pdp)
604{ 600{
605 __pdp_fini(pdp); 601 __pdp_fini(pdp);
606 if (USES_FULL_48BIT_PPGTT(dev)) { 602 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
607 cleanup_px(dev, pdp); 603 cleanup_px(dev_priv, pdp);
608 kfree(pdp); 604 kfree(pdp);
609 } 605 }
610} 606}
@@ -638,7 +634,7 @@ gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
638{ 634{
639 gen8_ppgtt_pdpe_t *page_directorypo; 635 gen8_ppgtt_pdpe_t *page_directorypo;
640 636
641 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 637 if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
642 return; 638 return;
643 639
644 page_directorypo = kmap_px(pdp); 640 page_directorypo = kmap_px(pdp);
@@ -654,7 +650,7 @@ gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
654{ 650{
655 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); 651 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
656 652
657 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); 653 WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)));
658 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); 654 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
659 kunmap_px(ppgtt, pagemap); 655 kunmap_px(ppgtt, pagemap);
660} 656}
@@ -714,7 +710,7 @@ static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
714 */ 710 */
715static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) 711static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
716{ 712{
717 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; 713 ppgtt->pd_dirty_rings = INTEL_INFO(to_i915(ppgtt->base.dev))->ring_mask;
718} 714}
719 715
720/* Removes entries from a single page table, releasing it if it's empty. 716/* Removes entries from a single page table, releasing it if it's empty.
@@ -741,7 +737,7 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
741 bitmap_clear(pt->used_ptes, pte, num_entries); 737 bitmap_clear(pt->used_ptes, pte, num_entries);
742 738
743 if (bitmap_empty(pt->used_ptes, GEN8_PTES)) { 739 if (bitmap_empty(pt->used_ptes, GEN8_PTES)) {
744 free_pt(vm->dev, pt); 740 free_pt(to_i915(vm->dev), pt);
745 return true; 741 return true;
746 } 742 }
747 743
@@ -783,7 +779,7 @@ static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
783 } 779 }
784 780
785 if (bitmap_empty(pd->used_pdes, I915_PDES)) { 781 if (bitmap_empty(pd->used_pdes, I915_PDES)) {
786 free_pd(vm->dev, pd); 782 free_pd(to_i915(vm->dev), pd);
787 return true; 783 return true;
788 } 784 }
789 785
@@ -799,6 +795,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
799 uint64_t length) 795 uint64_t length)
800{ 796{
801 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 797 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
798 struct drm_i915_private *dev_priv = to_i915(vm->dev);
802 struct i915_page_directory *pd; 799 struct i915_page_directory *pd;
803 uint64_t pdpe; 800 uint64_t pdpe;
804 gen8_ppgtt_pdpe_t *pdpe_vaddr; 801 gen8_ppgtt_pdpe_t *pdpe_vaddr;
@@ -811,7 +808,7 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
811 808
812 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) { 809 if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
813 __clear_bit(pdpe, pdp->used_pdpes); 810 __clear_bit(pdpe, pdp->used_pdpes);
814 if (USES_FULL_48BIT_PPGTT(vm->dev)) { 811 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
815 pdpe_vaddr = kmap_px(pdp); 812 pdpe_vaddr = kmap_px(pdp);
816 pdpe_vaddr[pdpe] = scratch_pdpe; 813 pdpe_vaddr[pdpe] = scratch_pdpe;
817 kunmap_px(ppgtt, pdpe_vaddr); 814 kunmap_px(ppgtt, pdpe_vaddr);
@@ -821,9 +818,9 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
821 818
822 mark_tlbs_dirty(ppgtt); 819 mark_tlbs_dirty(ppgtt);
823 820
824 if (USES_FULL_48BIT_PPGTT(vm->dev) && 821 if (USES_FULL_48BIT_PPGTT(dev_priv) &&
825 bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev))) { 822 bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv))) {
826 free_pdp(vm->dev, pdp); 823 free_pdp(dev_priv, pdp);
827 return true; 824 return true;
828 } 825 }
829 826
@@ -846,7 +843,7 @@ static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
846 gen8_ppgtt_pml4e_t scratch_pml4e = 843 gen8_ppgtt_pml4e_t scratch_pml4e =
847 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC); 844 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
848 845
849 GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(vm->dev)); 846 GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(to_i915(vm->dev)));
850 847
851 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 848 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
852 if (WARN_ON(!pml4->pdps[pml4e])) 849 if (WARN_ON(!pml4->pdps[pml4e]))
@@ -866,7 +863,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
866{ 863{
867 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 864 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
868 865
869 if (USES_FULL_48BIT_PPGTT(vm->dev)) 866 if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev)))
870 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length); 867 gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
871 else 868 else
872 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length); 869 gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
@@ -901,7 +898,7 @@ gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
901 kunmap_px(ppgtt, pt_vaddr); 898 kunmap_px(ppgtt, pt_vaddr);
902 pt_vaddr = NULL; 899 pt_vaddr = NULL;
903 if (++pde == I915_PDES) { 900 if (++pde == I915_PDES) {
904 if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) 901 if (++pdpe == I915_PDPES_PER_PDP(to_i915(vm->dev)))
905 break; 902 break;
906 pde = 0; 903 pde = 0;
907 } 904 }
@@ -924,7 +921,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
924 921
925 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); 922 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
926 923
927 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 924 if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) {
928 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, 925 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
929 cache_level); 926 cache_level);
930 } else { 927 } else {
@@ -939,7 +936,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
939 } 936 }
940} 937}
941 938
942static void gen8_free_page_tables(struct drm_device *dev, 939static void gen8_free_page_tables(struct drm_i915_private *dev_priv,
943 struct i915_page_directory *pd) 940 struct i915_page_directory *pd)
944{ 941{
945 int i; 942 int i;
@@ -951,34 +948,34 @@ static void gen8_free_page_tables(struct drm_device *dev,
951 if (WARN_ON(!pd->page_table[i])) 948 if (WARN_ON(!pd->page_table[i]))
952 continue; 949 continue;
953 950
954 free_pt(dev, pd->page_table[i]); 951 free_pt(dev_priv, pd->page_table[i]);
955 pd->page_table[i] = NULL; 952 pd->page_table[i] = NULL;
956 } 953 }
957} 954}
958 955
959static int gen8_init_scratch(struct i915_address_space *vm) 956static int gen8_init_scratch(struct i915_address_space *vm)
960{ 957{
961 struct drm_device *dev = vm->dev; 958 struct drm_i915_private *dev_priv = to_i915(vm->dev);
962 int ret; 959 int ret;
963 960
964 ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA); 961 ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
965 if (ret) 962 if (ret)
966 return ret; 963 return ret;
967 964
968 vm->scratch_pt = alloc_pt(dev); 965 vm->scratch_pt = alloc_pt(dev_priv);
969 if (IS_ERR(vm->scratch_pt)) { 966 if (IS_ERR(vm->scratch_pt)) {
970 ret = PTR_ERR(vm->scratch_pt); 967 ret = PTR_ERR(vm->scratch_pt);
971 goto free_scratch_page; 968 goto free_scratch_page;
972 } 969 }
973 970
974 vm->scratch_pd = alloc_pd(dev); 971 vm->scratch_pd = alloc_pd(dev_priv);
975 if (IS_ERR(vm->scratch_pd)) { 972 if (IS_ERR(vm->scratch_pd)) {
976 ret = PTR_ERR(vm->scratch_pd); 973 ret = PTR_ERR(vm->scratch_pd);
977 goto free_pt; 974 goto free_pt;
978 } 975 }
979 976
980 if (USES_FULL_48BIT_PPGTT(dev)) { 977 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
981 vm->scratch_pdp = alloc_pdp(dev); 978 vm->scratch_pdp = alloc_pdp(dev_priv);
982 if (IS_ERR(vm->scratch_pdp)) { 979 if (IS_ERR(vm->scratch_pdp)) {
983 ret = PTR_ERR(vm->scratch_pdp); 980 ret = PTR_ERR(vm->scratch_pdp);
984 goto free_pd; 981 goto free_pd;
@@ -987,17 +984,17 @@ static int gen8_init_scratch(struct i915_address_space *vm)
987 984
988 gen8_initialize_pt(vm, vm->scratch_pt); 985 gen8_initialize_pt(vm, vm->scratch_pt);
989 gen8_initialize_pd(vm, vm->scratch_pd); 986 gen8_initialize_pd(vm, vm->scratch_pd);
990 if (USES_FULL_48BIT_PPGTT(dev)) 987 if (USES_FULL_48BIT_PPGTT(dev_priv))
991 gen8_initialize_pdp(vm, vm->scratch_pdp); 988 gen8_initialize_pdp(vm, vm->scratch_pdp);
992 989
993 return 0; 990 return 0;
994 991
995free_pd: 992free_pd:
996 free_pd(dev, vm->scratch_pd); 993 free_pd(dev_priv, vm->scratch_pd);
997free_pt: 994free_pt:
998 free_pt(dev, vm->scratch_pt); 995 free_pt(dev_priv, vm->scratch_pt);
999free_scratch_page: 996free_scratch_page:
1000 cleanup_scratch_page(dev, &vm->scratch_page); 997 cleanup_scratch_page(dev_priv, &vm->scratch_page);
1001 998
1002 return ret; 999 return ret;
1003} 1000}
@@ -1035,54 +1032,56 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1035 1032
1036static void gen8_free_scratch(struct i915_address_space *vm) 1033static void gen8_free_scratch(struct i915_address_space *vm)
1037{ 1034{
1038 struct drm_device *dev = vm->dev; 1035 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1039 1036
1040 if (USES_FULL_48BIT_PPGTT(dev)) 1037 if (USES_FULL_48BIT_PPGTT(dev_priv))
1041 free_pdp(dev, vm->scratch_pdp); 1038 free_pdp(dev_priv, vm->scratch_pdp);
1042 free_pd(dev, vm->scratch_pd); 1039 free_pd(dev_priv, vm->scratch_pd);
1043 free_pt(dev, vm->scratch_pt); 1040 free_pt(dev_priv, vm->scratch_pt);
1044 cleanup_scratch_page(dev, &vm->scratch_page); 1041 cleanup_scratch_page(dev_priv, &vm->scratch_page);
1045} 1042}
1046 1043
1047static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, 1044static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv,
1048 struct i915_page_directory_pointer *pdp) 1045 struct i915_page_directory_pointer *pdp)
1049{ 1046{
1050 int i; 1047 int i;
1051 1048
1052 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { 1049 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) {
1053 if (WARN_ON(!pdp->page_directory[i])) 1050 if (WARN_ON(!pdp->page_directory[i]))
1054 continue; 1051 continue;
1055 1052
1056 gen8_free_page_tables(dev, pdp->page_directory[i]); 1053 gen8_free_page_tables(dev_priv, pdp->page_directory[i]);
1057 free_pd(dev, pdp->page_directory[i]); 1054 free_pd(dev_priv, pdp->page_directory[i]);
1058 } 1055 }
1059 1056
1060 free_pdp(dev, pdp); 1057 free_pdp(dev_priv, pdp);
1061} 1058}
1062 1059
1063static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) 1060static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1064{ 1061{
1062 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1065 int i; 1063 int i;
1066 1064
1067 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { 1065 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
1068 if (WARN_ON(!ppgtt->pml4.pdps[i])) 1066 if (WARN_ON(!ppgtt->pml4.pdps[i]))
1069 continue; 1067 continue;
1070 1068
1071 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); 1069 gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]);
1072 } 1070 }
1073 1071
1074 cleanup_px(ppgtt->base.dev, &ppgtt->pml4); 1072 cleanup_px(dev_priv, &ppgtt->pml4);
1075} 1073}
1076 1074
1077static void gen8_ppgtt_cleanup(struct i915_address_space *vm) 1075static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1078{ 1076{
1077 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1079 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1078 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1080 1079
1081 if (intel_vgpu_active(to_i915(vm->dev))) 1080 if (intel_vgpu_active(dev_priv))
1082 gen8_ppgtt_notify_vgt(ppgtt, false); 1081 gen8_ppgtt_notify_vgt(ppgtt, false);
1083 1082
1084 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) 1083 if (!USES_FULL_48BIT_PPGTT(dev_priv))
1085 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); 1084 gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp);
1086 else 1085 else
1087 gen8_ppgtt_cleanup_4lvl(ppgtt); 1086 gen8_ppgtt_cleanup_4lvl(ppgtt);
1088 1087
@@ -1113,7 +1112,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1113 uint64_t length, 1112 uint64_t length,
1114 unsigned long *new_pts) 1113 unsigned long *new_pts)
1115{ 1114{
1116 struct drm_device *dev = vm->dev; 1115 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1117 struct i915_page_table *pt; 1116 struct i915_page_table *pt;
1118 uint32_t pde; 1117 uint32_t pde;
1119 1118
@@ -1125,7 +1124,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1125 continue; 1124 continue;
1126 } 1125 }
1127 1126
1128 pt = alloc_pt(dev); 1127 pt = alloc_pt(dev_priv);
1129 if (IS_ERR(pt)) 1128 if (IS_ERR(pt))
1130 goto unwind_out; 1129 goto unwind_out;
1131 1130
@@ -1139,7 +1138,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
1139 1138
1140unwind_out: 1139unwind_out:
1141 for_each_set_bit(pde, new_pts, I915_PDES) 1140 for_each_set_bit(pde, new_pts, I915_PDES)
1142 free_pt(dev, pd->page_table[pde]); 1141 free_pt(dev_priv, pd->page_table[pde]);
1143 1142
1144 return -ENOMEM; 1143 return -ENOMEM;
1145} 1144}
@@ -1174,10 +1173,10 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1174 uint64_t length, 1173 uint64_t length,
1175 unsigned long *new_pds) 1174 unsigned long *new_pds)
1176{ 1175{
1177 struct drm_device *dev = vm->dev; 1176 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1178 struct i915_page_directory *pd; 1177 struct i915_page_directory *pd;
1179 uint32_t pdpe; 1178 uint32_t pdpe;
1180 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1179 uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1181 1180
1182 WARN_ON(!bitmap_empty(new_pds, pdpes)); 1181 WARN_ON(!bitmap_empty(new_pds, pdpes));
1183 1182
@@ -1185,7 +1184,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1185 if (test_bit(pdpe, pdp->used_pdpes)) 1184 if (test_bit(pdpe, pdp->used_pdpes))
1186 continue; 1185 continue;
1187 1186
1188 pd = alloc_pd(dev); 1187 pd = alloc_pd(dev_priv);
1189 if (IS_ERR(pd)) 1188 if (IS_ERR(pd))
1190 goto unwind_out; 1189 goto unwind_out;
1191 1190
@@ -1199,7 +1198,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1199 1198
1200unwind_out: 1199unwind_out:
1201 for_each_set_bit(pdpe, new_pds, pdpes) 1200 for_each_set_bit(pdpe, new_pds, pdpes)
1202 free_pd(dev, pdp->page_directory[pdpe]); 1201 free_pd(dev_priv, pdp->page_directory[pdpe]);
1203 1202
1204 return -ENOMEM; 1203 return -ENOMEM;
1205} 1204}
@@ -1227,7 +1226,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1227 uint64_t length, 1226 uint64_t length,
1228 unsigned long *new_pdps) 1227 unsigned long *new_pdps)
1229{ 1228{
1230 struct drm_device *dev = vm->dev; 1229 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1231 struct i915_page_directory_pointer *pdp; 1230 struct i915_page_directory_pointer *pdp;
1232 uint32_t pml4e; 1231 uint32_t pml4e;
1233 1232
@@ -1235,7 +1234,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1235 1234
1236 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) { 1235 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1237 if (!test_bit(pml4e, pml4->used_pml4es)) { 1236 if (!test_bit(pml4e, pml4->used_pml4es)) {
1238 pdp = alloc_pdp(dev); 1237 pdp = alloc_pdp(dev_priv);
1239 if (IS_ERR(pdp)) 1238 if (IS_ERR(pdp))
1240 goto unwind_out; 1239 goto unwind_out;
1241 1240
@@ -1253,7 +1252,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1253 1252
1254unwind_out: 1253unwind_out:
1255 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1254 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1256 free_pdp(dev, pml4->pdps[pml4e]); 1255 free_pdp(dev_priv, pml4->pdps[pml4e]);
1257 1256
1258 return -ENOMEM; 1257 return -ENOMEM;
1259} 1258}
@@ -1302,12 +1301,12 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1302{ 1301{
1303 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1302 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1304 unsigned long *new_page_dirs, *new_page_tables; 1303 unsigned long *new_page_dirs, *new_page_tables;
1305 struct drm_device *dev = vm->dev; 1304 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1306 struct i915_page_directory *pd; 1305 struct i915_page_directory *pd;
1307 const uint64_t orig_start = start; 1306 const uint64_t orig_start = start;
1308 const uint64_t orig_length = length; 1307 const uint64_t orig_length = length;
1309 uint32_t pdpe; 1308 uint32_t pdpe;
1310 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1309 uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
1311 int ret; 1310 int ret;
1312 1311
1313 /* Wrap is never okay since we can only represent 48b, and we don't 1312 /* Wrap is never okay since we can only represent 48b, and we don't
@@ -1395,11 +1394,12 @@ err_out:
1395 1394
1396 for_each_set_bit(temp, new_page_tables + pdpe * 1395 for_each_set_bit(temp, new_page_tables + pdpe *
1397 BITS_TO_LONGS(I915_PDES), I915_PDES) 1396 BITS_TO_LONGS(I915_PDES), I915_PDES)
1398 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); 1397 free_pt(dev_priv,
1398 pdp->page_directory[pdpe]->page_table[temp]);
1399 } 1399 }
1400 1400
1401 for_each_set_bit(pdpe, new_page_dirs, pdpes) 1401 for_each_set_bit(pdpe, new_page_dirs, pdpes)
1402 free_pd(dev, pdp->page_directory[pdpe]); 1402 free_pd(dev_priv, pdp->page_directory[pdpe]);
1403 1403
1404 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); 1404 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
1405 mark_tlbs_dirty(ppgtt); 1405 mark_tlbs_dirty(ppgtt);
@@ -1450,7 +1450,7 @@ static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1450 1450
1451err_out: 1451err_out:
1452 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) 1452 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1453 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); 1453 gen8_ppgtt_cleanup_3lvl(to_i915(vm->dev), pml4->pdps[pml4e]);
1454 1454
1455 return ret; 1455 return ret;
1456} 1456}
@@ -1460,7 +1460,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
1460{ 1460{
1461 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1461 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1462 1462
1463 if (USES_FULL_48BIT_PPGTT(vm->dev)) 1463 if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev)))
1464 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); 1464 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1465 else 1465 else
1466 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); 1466 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
@@ -1531,7 +1531,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1531 gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr, 1531 gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
1532 I915_CACHE_LLC); 1532 I915_CACHE_LLC);
1533 1533
1534 if (!USES_FULL_48BIT_PPGTT(vm->dev)) { 1534 if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) {
1535 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); 1535 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1536 } else { 1536 } else {
1537 uint64_t pml4e; 1537 uint64_t pml4e;
@@ -1551,7 +1551,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1551static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) 1551static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1552{ 1552{
1553 unsigned long *new_page_dirs, *new_page_tables; 1553 unsigned long *new_page_dirs, *new_page_tables;
1554 uint32_t pdpes = I915_PDPES_PER_PDP(dev); 1554 uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
1555 int ret; 1555 int ret;
1556 1556
1557 /* We allocate temp bitmap for page tables for no gain 1557 /* We allocate temp bitmap for page tables for no gain
@@ -1584,6 +1584,7 @@ static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1584 */ 1584 */
1585static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 1585static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1586{ 1586{
1587 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1587 int ret; 1588 int ret;
1588 1589
1589 ret = gen8_init_scratch(&ppgtt->base); 1590 ret = gen8_init_scratch(&ppgtt->base);
@@ -1599,8 +1600,8 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1599 ppgtt->base.bind_vma = ppgtt_bind_vma; 1600 ppgtt->base.bind_vma = ppgtt_bind_vma;
1600 ppgtt->debug_dump = gen8_dump_ppgtt; 1601 ppgtt->debug_dump = gen8_dump_ppgtt;
1601 1602
1602 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { 1603 if (USES_FULL_48BIT_PPGTT(dev_priv)) {
1603 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); 1604 ret = setup_px(dev_priv, &ppgtt->pml4);
1604 if (ret) 1605 if (ret)
1605 goto free_scratch; 1606 goto free_scratch;
1606 1607
@@ -1609,7 +1610,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1609 ppgtt->base.total = 1ULL << 48; 1610 ppgtt->base.total = 1ULL << 48;
1610 ppgtt->switch_mm = gen8_48b_mm_switch; 1611 ppgtt->switch_mm = gen8_48b_mm_switch;
1611 } else { 1612 } else {
1612 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); 1613 ret = __pdp_init(dev_priv, &ppgtt->pdp);
1613 if (ret) 1614 if (ret)
1614 goto free_scratch; 1615 goto free_scratch;
1615 1616
@@ -1619,14 +1620,14 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1619 0, 0, 1620 0, 0,
1620 GEN8_PML4E_SHIFT); 1621 GEN8_PML4E_SHIFT);
1621 1622
1622 if (intel_vgpu_active(to_i915(ppgtt->base.dev))) { 1623 if (intel_vgpu_active(dev_priv)) {
1623 ret = gen8_preallocate_top_level_pdps(ppgtt); 1624 ret = gen8_preallocate_top_level_pdps(ppgtt);
1624 if (ret) 1625 if (ret)
1625 goto free_scratch; 1626 goto free_scratch;
1626 } 1627 }
1627 } 1628 }
1628 1629
1629 if (intel_vgpu_active(to_i915(ppgtt->base.dev))) 1630 if (intel_vgpu_active(dev_priv))
1630 gen8_ppgtt_notify_vgt(ppgtt, true); 1631 gen8_ppgtt_notify_vgt(ppgtt, true);
1631 1632
1632 return 0; 1633 return 0;
@@ -1801,22 +1802,21 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1801 return 0; 1802 return 0;
1802} 1803}
1803 1804
1804static void gen8_ppgtt_enable(struct drm_device *dev) 1805static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1805{ 1806{
1806 struct drm_i915_private *dev_priv = to_i915(dev);
1807 struct intel_engine_cs *engine; 1807 struct intel_engine_cs *engine;
1808 enum intel_engine_id id; 1808 enum intel_engine_id id;
1809 1809
1810 for_each_engine(engine, dev_priv, id) { 1810 for_each_engine(engine, dev_priv, id) {
1811 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; 1811 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1812 GEN8_GFX_PPGTT_48B : 0;
1812 I915_WRITE(RING_MODE_GEN7(engine), 1813 I915_WRITE(RING_MODE_GEN7(engine),
1813 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); 1814 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1814 } 1815 }
1815} 1816}
1816 1817
1817static void gen7_ppgtt_enable(struct drm_device *dev) 1818static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1818{ 1819{
1819 struct drm_i915_private *dev_priv = to_i915(dev);
1820 struct intel_engine_cs *engine; 1820 struct intel_engine_cs *engine;
1821 uint32_t ecochk, ecobits; 1821 uint32_t ecochk, ecobits;
1822 enum intel_engine_id id; 1822 enum intel_engine_id id;
@@ -1840,9 +1840,8 @@ static void gen7_ppgtt_enable(struct drm_device *dev)
1840 } 1840 }
1841} 1841}
1842 1842
1843static void gen6_ppgtt_enable(struct drm_device *dev) 1843static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1844{ 1844{
1845 struct drm_i915_private *dev_priv = to_i915(dev);
1846 uint32_t ecochk, gab_ctl, ecobits; 1845 uint32_t ecochk, gab_ctl, ecobits;
1847 1846
1848 ecobits = I915_READ(GAC_ECO_BITS); 1847 ecobits = I915_READ(GAC_ECO_BITS);
@@ -1928,8 +1927,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
1928 uint64_t start_in, uint64_t length_in) 1927 uint64_t start_in, uint64_t length_in)
1929{ 1928{
1930 DECLARE_BITMAP(new_page_tables, I915_PDES); 1929 DECLARE_BITMAP(new_page_tables, I915_PDES);
1931 struct drm_device *dev = vm->dev; 1930 struct drm_i915_private *dev_priv = to_i915(vm->dev);
1932 struct drm_i915_private *dev_priv = to_i915(dev);
1933 struct i915_ggtt *ggtt = &dev_priv->ggtt; 1931 struct i915_ggtt *ggtt = &dev_priv->ggtt;
1934 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1932 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1935 struct i915_page_table *pt; 1933 struct i915_page_table *pt;
@@ -1959,7 +1957,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
1959 /* We've already allocated a page table */ 1957 /* We've already allocated a page table */
1960 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); 1958 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1961 1959
1962 pt = alloc_pt(dev); 1960 pt = alloc_pt(dev_priv);
1963 if (IS_ERR(pt)) { 1961 if (IS_ERR(pt)) {
1964 ret = PTR_ERR(pt); 1962 ret = PTR_ERR(pt);
1965 goto unwind_out; 1963 goto unwind_out;
@@ -2007,7 +2005,7 @@ unwind_out:
2007 struct i915_page_table *pt = ppgtt->pd.page_table[pde]; 2005 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
2008 2006
2009 ppgtt->pd.page_table[pde] = vm->scratch_pt; 2007 ppgtt->pd.page_table[pde] = vm->scratch_pt;
2010 free_pt(vm->dev, pt); 2008 free_pt(dev_priv, pt);
2011 } 2009 }
2012 2010
2013 mark_tlbs_dirty(ppgtt); 2011 mark_tlbs_dirty(ppgtt);
@@ -2016,16 +2014,16 @@ unwind_out:
2016 2014
2017static int gen6_init_scratch(struct i915_address_space *vm) 2015static int gen6_init_scratch(struct i915_address_space *vm)
2018{ 2016{
2019 struct drm_device *dev = vm->dev; 2017 struct drm_i915_private *dev_priv = to_i915(vm->dev);
2020 int ret; 2018 int ret;
2021 2019
2022 ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA); 2020 ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
2023 if (ret) 2021 if (ret)
2024 return ret; 2022 return ret;
2025 2023
2026 vm->scratch_pt = alloc_pt(dev); 2024 vm->scratch_pt = alloc_pt(dev_priv);
2027 if (IS_ERR(vm->scratch_pt)) { 2025 if (IS_ERR(vm->scratch_pt)) {
2028 cleanup_scratch_page(dev, &vm->scratch_page); 2026 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2029 return PTR_ERR(vm->scratch_pt); 2027 return PTR_ERR(vm->scratch_pt);
2030 } 2028 }
2031 2029
@@ -2036,17 +2034,17 @@ static int gen6_init_scratch(struct i915_address_space *vm)
2036 2034
2037static void gen6_free_scratch(struct i915_address_space *vm) 2035static void gen6_free_scratch(struct i915_address_space *vm)
2038{ 2036{
2039 struct drm_device *dev = vm->dev; 2037 struct drm_i915_private *dev_priv = to_i915(vm->dev);
2040 2038
2041 free_pt(dev, vm->scratch_pt); 2039 free_pt(dev_priv, vm->scratch_pt);
2042 cleanup_scratch_page(dev, &vm->scratch_page); 2040 cleanup_scratch_page(dev_priv, &vm->scratch_page);
2043} 2041}
2044 2042
2045static void gen6_ppgtt_cleanup(struct i915_address_space *vm) 2043static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2046{ 2044{
2047 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 2045 struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
2048 struct i915_page_directory *pd = &ppgtt->pd; 2046 struct i915_page_directory *pd = &ppgtt->pd;
2049 struct drm_device *dev = vm->dev; 2047 struct drm_i915_private *dev_priv = to_i915(vm->dev);
2050 struct i915_page_table *pt; 2048 struct i915_page_table *pt;
2051 uint32_t pde; 2049 uint32_t pde;
2052 2050
@@ -2054,7 +2052,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2054 2052
2055 gen6_for_all_pdes(pt, pd, pde) 2053 gen6_for_all_pdes(pt, pd, pde)
2056 if (pt != vm->scratch_pt) 2054 if (pt != vm->scratch_pt)
2057 free_pt(dev, pt); 2055 free_pt(dev_priv, pt);
2058 2056
2059 gen6_free_scratch(vm); 2057 gen6_free_scratch(vm);
2060} 2058}
@@ -2062,8 +2060,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
2062static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) 2060static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
2063{ 2061{
2064 struct i915_address_space *vm = &ppgtt->base; 2062 struct i915_address_space *vm = &ppgtt->base;
2065 struct drm_device *dev = ppgtt->base.dev; 2063 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
2066 struct drm_i915_private *dev_priv = to_i915(dev);
2067 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2064 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2068 bool retried = false; 2065 bool retried = false;
2069 int ret; 2066 int ret;
@@ -2128,8 +2125,7 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2128 2125
2129static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 2126static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
2130{ 2127{
2131 struct drm_device *dev = ppgtt->base.dev; 2128 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
2132 struct drm_i915_private *dev_priv = to_i915(dev);
2133 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2129 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2134 int ret; 2130 int ret;
2135 2131
@@ -2200,10 +2196,15 @@ static void i915_address_space_init(struct i915_address_space *vm,
2200 list_add_tail(&vm->global_link, &dev_priv->vm_list); 2196 list_add_tail(&vm->global_link, &dev_priv->vm_list);
2201} 2197}
2202 2198
2203static void gtt_write_workarounds(struct drm_device *dev) 2199static void i915_address_space_fini(struct i915_address_space *vm)
2204{ 2200{
2205 struct drm_i915_private *dev_priv = to_i915(dev); 2201 i915_gem_timeline_fini(&vm->timeline);
2202 drm_mm_takedown(&vm->mm);
2203 list_del(&vm->global_link);
2204}
2206 2205
2206static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2207{
2207 /* This function is for gtt related workarounds. This function is 2208 /* This function is for gtt related workarounds. This function is
2208 * called on driver load and after a GPU reset, so you can place 2209 * called on driver load and after a GPU reset, so you can place
2209 * workarounds here even if they get overwritten by GPU reset. 2210 * workarounds here even if they get overwritten by GPU reset.
@@ -2236,11 +2237,9 @@ static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
2236 return ret; 2237 return ret;
2237} 2238}
2238 2239
2239int i915_ppgtt_init_hw(struct drm_device *dev) 2240int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2240{ 2241{
2241 struct drm_i915_private *dev_priv = to_i915(dev); 2242 gtt_write_workarounds(dev_priv);
2242
2243 gtt_write_workarounds(dev);
2244 2243
2245 /* In the case of execlists, PPGTT is enabled by the context descriptor 2244 /* In the case of execlists, PPGTT is enabled by the context descriptor
2246 * and the PDPs are contained within the context itself. We don't 2245 * and the PDPs are contained within the context itself. We don't
@@ -2248,17 +2247,17 @@ int i915_ppgtt_init_hw(struct drm_device *dev)
2248 if (i915.enable_execlists) 2247 if (i915.enable_execlists)
2249 return 0; 2248 return 0;
2250 2249
2251 if (!USES_PPGTT(dev)) 2250 if (!USES_PPGTT(dev_priv))
2252 return 0; 2251 return 0;
2253 2252
2254 if (IS_GEN6(dev_priv)) 2253 if (IS_GEN6(dev_priv))
2255 gen6_ppgtt_enable(dev); 2254 gen6_ppgtt_enable(dev_priv);
2256 else if (IS_GEN7(dev_priv)) 2255 else if (IS_GEN7(dev_priv))
2257 gen7_ppgtt_enable(dev); 2256 gen7_ppgtt_enable(dev_priv);
2258 else if (INTEL_INFO(dev)->gen >= 8) 2257 else if (INTEL_GEN(dev_priv) >= 8)
2259 gen8_ppgtt_enable(dev); 2258 gen8_ppgtt_enable(dev_priv);
2260 else 2259 else
2261 MISSING_CASE(INTEL_INFO(dev)->gen); 2260 MISSING_CASE(INTEL_GEN(dev_priv));
2262 2261
2263 return 0; 2262 return 0;
2264} 2263}
@@ -2286,7 +2285,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv,
2286 return ppgtt; 2285 return ppgtt;
2287} 2286}
2288 2287
2289void i915_ppgtt_release(struct kref *kref) 2288void i915_ppgtt_release(struct kref *kref)
2290{ 2289{
2291 struct i915_hw_ppgtt *ppgtt = 2290 struct i915_hw_ppgtt *ppgtt =
2292 container_of(kref, struct i915_hw_ppgtt, ref); 2291 container_of(kref, struct i915_hw_ppgtt, ref);
@@ -2298,9 +2297,7 @@ void i915_ppgtt_release(struct kref *kref)
2298 WARN_ON(!list_empty(&ppgtt->base.inactive_list)); 2297 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2299 WARN_ON(!list_empty(&ppgtt->base.unbound_list)); 2298 WARN_ON(!list_empty(&ppgtt->base.unbound_list));
2300 2299
2301 i915_gem_timeline_fini(&ppgtt->base.timeline); 2300 i915_address_space_fini(&ppgtt->base);
2302 list_del(&ppgtt->base.global_link);
2303 drm_mm_takedown(&ppgtt->base.mm);
2304 2301
2305 ppgtt->base.cleanup(&ppgtt->base); 2302 ppgtt->base.cleanup(&ppgtt->base);
2306 kfree(ppgtt); 2303 kfree(ppgtt);
@@ -2362,15 +2359,14 @@ static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2362 } 2359 }
2363} 2360}
2364 2361
2365void i915_gem_suspend_gtt_mappings(struct drm_device *dev) 2362void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2366{ 2363{
2367 struct drm_i915_private *dev_priv = to_i915(dev);
2368 struct i915_ggtt *ggtt = &dev_priv->ggtt; 2364 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2369 2365
2370 /* Don't bother messing with faults pre GEN6 as we have little 2366 /* Don't bother messing with faults pre GEN6 as we have little
2371 * documentation supporting that it's a good idea. 2367 * documentation supporting that it's a good idea.
2372 */ 2368 */
2373 if (INTEL_INFO(dev)->gen < 6) 2369 if (INTEL_GEN(dev_priv) < 6)
2374 return; 2370 return;
2375 2371
2376 i915_check_and_clear_faults(dev_priv); 2372 i915_check_and_clear_faults(dev_priv);
@@ -2842,8 +2838,9 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2842 if (drm_mm_initialized(&ggtt->base.mm)) { 2838 if (drm_mm_initialized(&ggtt->base.mm)) {
2843 intel_vgt_deballoon(dev_priv); 2839 intel_vgt_deballoon(dev_priv);
2844 2840
2845 drm_mm_takedown(&ggtt->base.mm); 2841 mutex_lock(&dev_priv->drm.struct_mutex);
2846 list_del(&ggtt->base.global_link); 2842 i915_address_space_fini(&ggtt->base);
2843 mutex_unlock(&dev_priv->drm.struct_mutex);
2847 } 2844 }
2848 2845
2849 ggtt->base.cleanup(&ggtt->base); 2846 ggtt->base.cleanup(&ggtt->base);
@@ -2932,6 +2929,7 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2932 2929
2933static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 2930static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2934{ 2931{
2932 struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev);
2935 struct pci_dev *pdev = ggtt->base.dev->pdev; 2933 struct pci_dev *pdev = ggtt->base.dev->pdev;
2936 phys_addr_t phys_addr; 2934 phys_addr_t phys_addr;
2937 int ret; 2935 int ret;
@@ -2946,7 +2944,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2946 * resort to an uncached mapping. The WC issue is easily caught by the 2944 * resort to an uncached mapping. The WC issue is easily caught by the
2947 * readback check when writing GTT PTE entries. 2945 * readback check when writing GTT PTE entries.
2948 */ 2946 */
2949 if (IS_BROXTON(to_i915(ggtt->base.dev))) 2947 if (IS_BROXTON(dev_priv))
2950 ggtt->gsm = ioremap_nocache(phys_addr, size); 2948 ggtt->gsm = ioremap_nocache(phys_addr, size);
2951 else 2949 else
2952 ggtt->gsm = ioremap_wc(phys_addr, size); 2950 ggtt->gsm = ioremap_wc(phys_addr, size);
@@ -2955,9 +2953,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
2955 return -ENOMEM; 2953 return -ENOMEM;
2956 } 2954 }
2957 2955
2958 ret = setup_scratch_page(ggtt->base.dev, 2956 ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32);
2959 &ggtt->base.scratch_page,
2960 GFP_DMA32);
2961 if (ret) { 2957 if (ret) {
2962 DRM_ERROR("Scratch setup failed\n"); 2958 DRM_ERROR("Scratch setup failed\n");
2963 /* iounmap will also get called at remove, but meh */ 2959 /* iounmap will also get called at remove, but meh */
@@ -3046,7 +3042,7 @@ static void gen6_gmch_remove(struct i915_address_space *vm)
3046 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 3042 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
3047 3043
3048 iounmap(ggtt->gsm); 3044 iounmap(ggtt->gsm);
3049 cleanup_scratch_page(vm->dev, &vm->scratch_page); 3045 cleanup_scratch_page(to_i915(vm->dev), &vm->scratch_page);
3050} 3046}
3051 3047
3052static int gen8_gmch_probe(struct i915_ggtt *ggtt) 3048static int gen8_gmch_probe(struct i915_ggtt *ggtt)
@@ -3262,7 +3258,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3262 * Initialise stolen early so that we may reserve preallocated 3258 * Initialise stolen early so that we may reserve preallocated
3263 * objects for the BIOS to KMS transition. 3259 * objects for the BIOS to KMS transition.
3264 */ 3260 */
3265 ret = i915_gem_init_stolen(&dev_priv->drm); 3261 ret = i915_gem_init_stolen(dev_priv);
3266 if (ret) 3262 if (ret)
3267 goto out_gtt_cleanup; 3263 goto out_gtt_cleanup;
3268 3264
@@ -3281,9 +3277,8 @@ int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3281 return 0; 3277 return 0;
3282} 3278}
3283 3279
3284void i915_gem_restore_gtt_mappings(struct drm_device *dev) 3280void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3285{ 3281{
3286 struct drm_i915_private *dev_priv = to_i915(dev);
3287 struct i915_ggtt *ggtt = &dev_priv->ggtt; 3282 struct i915_ggtt *ggtt = &dev_priv->ggtt;
3288 struct drm_i915_gem_object *obj, *on; 3283 struct drm_i915_gem_object *obj, *on;
3289 3284
@@ -3318,7 +3313,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
3318 3313
3319 ggtt->base.closed = false; 3314 ggtt->base.closed = false;
3320 3315
3321 if (INTEL_INFO(dev)->gen >= 8) { 3316 if (INTEL_GEN(dev_priv) >= 8) {
3322 if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv)) 3317 if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
3323 chv_setup_private_ppat(dev_priv); 3318 chv_setup_private_ppat(dev_priv);
3324 else 3319 else
@@ -3327,7 +3322,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
3327 return; 3322 return;
3328 } 3323 }
3329 3324
3330 if (USES_PPGTT(dev)) { 3325 if (USES_PPGTT(dev_priv)) {
3331 struct i915_address_space *vm; 3326 struct i915_address_space *vm;
3332 3327
3333 list_for_each_entry(vm, &dev_priv->vm_list, global_link) { 3328 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
@@ -3348,176 +3343,6 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
3348 i915_ggtt_flush(dev_priv); 3343 i915_ggtt_flush(dev_priv);
3349} 3344}
3350 3345
3351static void
3352i915_vma_retire(struct i915_gem_active *active,
3353 struct drm_i915_gem_request *rq)
3354{
3355 const unsigned int idx = rq->engine->id;
3356 struct i915_vma *vma =
3357 container_of(active, struct i915_vma, last_read[idx]);
3358 struct drm_i915_gem_object *obj = vma->obj;
3359
3360 GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
3361
3362 i915_vma_clear_active(vma, idx);
3363 if (i915_vma_is_active(vma))
3364 return;
3365
3366 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3367 if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
3368 WARN_ON(i915_vma_unbind(vma));
3369
3370 GEM_BUG_ON(!i915_gem_object_is_active(obj));
3371 if (--obj->active_count)
3372 return;
3373
3374 /* Bump our place on the bound list to keep it roughly in LRU order
3375 * so that we don't steal from recently used but inactive objects
3376 * (unless we are forced to ofc!)
3377 */
3378 if (obj->bind_count)
3379 list_move_tail(&obj->global_link, &rq->i915->mm.bound_list);
3380
3381 obj->mm.dirty = true; /* be paranoid */
3382
3383 if (i915_gem_object_has_active_reference(obj)) {
3384 i915_gem_object_clear_active_reference(obj);
3385 i915_gem_object_put(obj);
3386 }
3387}
3388
3389static void
3390i915_ggtt_retire__write(struct i915_gem_active *active,
3391 struct drm_i915_gem_request *request)
3392{
3393 struct i915_vma *vma =
3394 container_of(active, struct i915_vma, last_write);
3395
3396 intel_fb_obj_flush(vma->obj, true, ORIGIN_CS);
3397}
3398
3399void i915_vma_destroy(struct i915_vma *vma)
3400{
3401 GEM_BUG_ON(vma->node.allocated);
3402 GEM_BUG_ON(i915_vma_is_active(vma));
3403 GEM_BUG_ON(!i915_vma_is_closed(vma));
3404 GEM_BUG_ON(vma->fence);
3405
3406 list_del(&vma->vm_link);
3407 if (!i915_vma_is_ggtt(vma))
3408 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
3409
3410 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
3411}
3412
3413void i915_vma_close(struct i915_vma *vma)
3414{
3415 GEM_BUG_ON(i915_vma_is_closed(vma));
3416 vma->flags |= I915_VMA_CLOSED;
3417
3418 list_del(&vma->obj_link);
3419 rb_erase(&vma->obj_node, &vma->obj->vma_tree);
3420
3421 if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
3422 WARN_ON(i915_vma_unbind(vma));
3423}
3424
3425static inline long vma_compare(struct i915_vma *vma,
3426 struct i915_address_space *vm,
3427 const struct i915_ggtt_view *view)
3428{
3429 GEM_BUG_ON(view && !i915_is_ggtt(vm));
3430
3431 if (vma->vm != vm)
3432 return vma->vm - vm;
3433
3434 if (!view)
3435 return vma->ggtt_view.type;
3436
3437 if (vma->ggtt_view.type != view->type)
3438 return vma->ggtt_view.type - view->type;
3439
3440 return memcmp(&vma->ggtt_view.params,
3441 &view->params,
3442 sizeof(view->params));
3443}
3444
3445static struct i915_vma *
3446__i915_vma_create(struct drm_i915_gem_object *obj,
3447 struct i915_address_space *vm,
3448 const struct i915_ggtt_view *view)
3449{
3450 struct i915_vma *vma;
3451 struct rb_node *rb, **p;
3452 int i;
3453
3454 GEM_BUG_ON(vm->closed);
3455
3456 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
3457 if (vma == NULL)
3458 return ERR_PTR(-ENOMEM);
3459
3460 INIT_LIST_HEAD(&vma->exec_list);
3461 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
3462 init_request_active(&vma->last_read[i], i915_vma_retire);
3463 init_request_active(&vma->last_write,
3464 i915_is_ggtt(vm) ? i915_ggtt_retire__write : NULL);
3465 init_request_active(&vma->last_fence, NULL);
3466 list_add(&vma->vm_link, &vm->unbound_list);
3467 vma->vm = vm;
3468 vma->obj = obj;
3469 vma->size = obj->base.size;
3470
3471 if (view) {
3472 vma->ggtt_view = *view;
3473 if (view->type == I915_GGTT_VIEW_PARTIAL) {
3474 vma->size = view->params.partial.size;
3475 vma->size <<= PAGE_SHIFT;
3476 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
3477 vma->size =
3478 intel_rotation_info_size(&view->params.rotated);
3479 vma->size <<= PAGE_SHIFT;
3480 }
3481 }
3482
3483 if (i915_is_ggtt(vm)) {
3484 vma->flags |= I915_VMA_GGTT;
3485 list_add(&vma->obj_link, &obj->vma_list);
3486 } else {
3487 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
3488 list_add_tail(&vma->obj_link, &obj->vma_list);
3489 }
3490
3491 rb = NULL;
3492 p = &obj->vma_tree.rb_node;
3493 while (*p) {
3494 struct i915_vma *pos;
3495
3496 rb = *p;
3497 pos = rb_entry(rb, struct i915_vma, obj_node);
3498 if (vma_compare(pos, vm, view) < 0)
3499 p = &rb->rb_right;
3500 else
3501 p = &rb->rb_left;
3502 }
3503 rb_link_node(&vma->obj_node, rb, p);
3504 rb_insert_color(&vma->obj_node, &obj->vma_tree);
3505
3506 return vma;
3507}
3508
3509struct i915_vma *
3510i915_vma_create(struct drm_i915_gem_object *obj,
3511 struct i915_address_space *vm,
3512 const struct i915_ggtt_view *view)
3513{
3514 lockdep_assert_held(&obj->base.dev->struct_mutex);
3515 GEM_BUG_ON(view && !i915_is_ggtt(vm));
3516 GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
3517
3518 return __i915_vma_create(obj, vm, view);
3519}
3520
3521struct i915_vma * 3346struct i915_vma *
3522i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, 3347i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
3523 struct i915_address_space *vm, 3348 struct i915_address_space *vm,
@@ -3530,7 +3355,7 @@ i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
3530 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); 3355 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
3531 long cmp; 3356 long cmp;
3532 3357
3533 cmp = vma_compare(vma, vm, view); 3358 cmp = i915_vma_compare(vma, vm, view);
3534 if (cmp == 0) 3359 if (cmp == 0)
3535 return vma; 3360 return vma;
3536 3361
@@ -3555,7 +3380,7 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3555 3380
3556 vma = i915_gem_obj_to_vma(obj, vm, view); 3381 vma = i915_gem_obj_to_vma(obj, vm, view);
3557 if (!vma) { 3382 if (!vma) {
3558 vma = __i915_vma_create(obj, vm, view); 3383 vma = i915_vma_create(obj, vm, view);
3559 GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view)); 3384 GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view));
3560 } 3385 }
3561 3386
@@ -3747,99 +3572,3 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
3747 return ret; 3572 return ret;
3748} 3573}
3749 3574
3750/**
3751 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
3752 * @vma: VMA to map
3753 * @cache_level: mapping cache level
3754 * @flags: flags like global or local mapping
3755 *
3756 * DMA addresses are taken from the scatter-gather table of this object (or of
3757 * this VMA in case of non-default GGTT views) and PTE entries set up.
3758 * Note that DMA addresses are also the only part of the SG table we care about.
3759 */
3760int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
3761 u32 flags)
3762{
3763 u32 bind_flags;
3764 u32 vma_flags;
3765 int ret;
3766
3767 if (WARN_ON(flags == 0))
3768 return -EINVAL;
3769
3770 bind_flags = 0;
3771 if (flags & PIN_GLOBAL)
3772 bind_flags |= I915_VMA_GLOBAL_BIND;
3773 if (flags & PIN_USER)
3774 bind_flags |= I915_VMA_LOCAL_BIND;
3775
3776 vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
3777 if (flags & PIN_UPDATE)
3778 bind_flags |= vma_flags;
3779 else
3780 bind_flags &= ~vma_flags;
3781 if (bind_flags == 0)
3782 return 0;
3783
3784 if (vma_flags == 0 && vma->vm->allocate_va_range) {
3785 trace_i915_va_alloc(vma);
3786 ret = vma->vm->allocate_va_range(vma->vm,
3787 vma->node.start,
3788 vma->node.size);
3789 if (ret)
3790 return ret;
3791 }
3792
3793 ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
3794 if (ret)
3795 return ret;
3796
3797 vma->flags |= bind_flags;
3798 return 0;
3799}
3800
3801void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
3802{
3803 void __iomem *ptr;
3804
3805 /* Access through the GTT requires the device to be awake. */
3806 assert_rpm_wakelock_held(to_i915(vma->vm->dev));
3807
3808 lockdep_assert_held(&vma->vm->dev->struct_mutex);
3809 if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
3810 return IO_ERR_PTR(-ENODEV);
3811
3812 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
3813 GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
3814
3815 ptr = vma->iomap;
3816 if (ptr == NULL) {
3817 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
3818 vma->node.start,
3819 vma->node.size);
3820 if (ptr == NULL)
3821 return IO_ERR_PTR(-ENOMEM);
3822
3823 vma->iomap = ptr;
3824 }
3825
3826 __i915_vma_pin(vma);
3827 return ptr;
3828}
3829
3830void i915_vma_unpin_and_release(struct i915_vma **p_vma)
3831{
3832 struct i915_vma *vma;
3833 struct drm_i915_gem_object *obj;
3834
3835 vma = fetch_and_zero(p_vma);
3836 if (!vma)
3837 return;
3838
3839 obj = vma->obj;
3840
3841 i915_vma_unpin(vma);
3842 i915_vma_close(vma);
3843
3844 __i915_gem_object_release_unless_active(obj);
3845}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index c23ef9db1f53..4f35be4c26c7 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -35,7 +35,9 @@
35#define __I915_GEM_GTT_H__ 35#define __I915_GEM_GTT_H__
36 36
37#include <linux/io-mapping.h> 37#include <linux/io-mapping.h>
38#include <linux/mm.h>
38 39
40#include "i915_gem_timeline.h"
39#include "i915_gem_request.h" 41#include "i915_gem_request.h"
40 42
41#define I915_FENCE_REG_NONE -1 43#define I915_FENCE_REG_NONE -1
@@ -118,8 +120,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t;
118#define GEN8_LEGACY_PDPES 4 120#define GEN8_LEGACY_PDPES 4
119#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t)) 121#define GEN8_PTES I915_PTES(sizeof(gen8_pte_t))
120 122
121#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ 123#define I915_PDPES_PER_PDP(dev_priv) (USES_FULL_48BIT_PPGTT(dev_priv) ?\
122 GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES) 124 GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
123 125
124#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD) 126#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD)
125#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */ 127#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */
@@ -138,6 +140,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t;
138#define GEN8_PPAT_ELLC_OVERRIDE (0<<2) 140#define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
139#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8)) 141#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8))
140 142
143struct sg_table;
144
141enum i915_ggtt_view_type { 145enum i915_ggtt_view_type {
142 I915_GGTT_VIEW_NORMAL = 0, 146 I915_GGTT_VIEW_NORMAL = 0,
143 I915_GGTT_VIEW_ROTATED, 147 I915_GGTT_VIEW_ROTATED,
@@ -168,135 +172,7 @@ extern const struct i915_ggtt_view i915_ggtt_view_rotated;
168 172
169enum i915_cache_level; 173enum i915_cache_level;
170 174
171/** 175struct i915_vma;
172 * A VMA represents a GEM BO that is bound into an address space. Therefore, a
173 * VMA's presence cannot be guaranteed before binding, or after unbinding the
174 * object into/from the address space.
175 *
176 * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
177 * will always be <= an objects lifetime. So object refcounting should cover us.
178 */
179struct i915_vma {
180 struct drm_mm_node node;
181 struct drm_i915_gem_object *obj;
182 struct i915_address_space *vm;
183 struct drm_i915_fence_reg *fence;
184 struct sg_table *pages;
185 void __iomem *iomap;
186 u64 size;
187 u64 display_alignment;
188
189 unsigned int flags;
190 /**
191 * How many users have pinned this object in GTT space. The following
192 * users can each hold at most one reference: pwrite/pread, execbuffer
193 * (objects are not allowed multiple times for the same batchbuffer),
194 * and the framebuffer code. When switching/pageflipping, the
195 * framebuffer code has at most two buffers pinned per crtc.
196 *
197 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
198 * bits with absolutely no headroom. So use 4 bits.
199 */
200#define I915_VMA_PIN_MASK 0xf
201#define I915_VMA_PIN_OVERFLOW BIT(5)
202
203 /** Flags and address space this VMA is bound to */
204#define I915_VMA_GLOBAL_BIND BIT(6)
205#define I915_VMA_LOCAL_BIND BIT(7)
206#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
207
208#define I915_VMA_GGTT BIT(8)
209#define I915_VMA_CAN_FENCE BIT(9)
210#define I915_VMA_CLOSED BIT(10)
211
212 unsigned int active;
213 struct i915_gem_active last_read[I915_NUM_ENGINES];
214 struct i915_gem_active last_write;
215 struct i915_gem_active last_fence;
216
217 /**
218 * Support different GGTT views into the same object.
219 * This means there can be multiple VMA mappings per object and per VM.
220 * i915_ggtt_view_type is used to distinguish between those entries.
221 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
222 * assumed in GEM functions which take no ggtt view parameter.
223 */
224 struct i915_ggtt_view ggtt_view;
225
226 /** This object's place on the active/inactive lists */
227 struct list_head vm_link;
228
229 struct list_head obj_link; /* Link in the object's VMA list */
230 struct rb_node obj_node;
231
232 /** This vma's place in the batchbuffer or on the eviction list */
233 struct list_head exec_list;
234
235 /**
236 * Used for performing relocations during execbuffer insertion.
237 */
238 struct hlist_node exec_node;
239 unsigned long exec_handle;
240 struct drm_i915_gem_exec_object2 *exec_entry;
241};
242
243struct i915_vma *
244i915_vma_create(struct drm_i915_gem_object *obj,
245 struct i915_address_space *vm,
246 const struct i915_ggtt_view *view);
247void i915_vma_unpin_and_release(struct i915_vma **p_vma);
248
249static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
250{
251 return vma->flags & I915_VMA_GGTT;
252}
253
254static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
255{
256 return vma->flags & I915_VMA_CAN_FENCE;
257}
258
259static inline bool i915_vma_is_closed(const struct i915_vma *vma)
260{
261 return vma->flags & I915_VMA_CLOSED;
262}
263
264static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
265{
266 return vma->active;
267}
268
269static inline bool i915_vma_is_active(const struct i915_vma *vma)
270{
271 return i915_vma_get_active(vma);
272}
273
274static inline void i915_vma_set_active(struct i915_vma *vma,
275 unsigned int engine)
276{
277 vma->active |= BIT(engine);
278}
279
280static inline void i915_vma_clear_active(struct i915_vma *vma,
281 unsigned int engine)
282{
283 vma->active &= ~BIT(engine);
284}
285
286static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
287 unsigned int engine)
288{
289 return vma->active & BIT(engine);
290}
291
292static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
293{
294 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
295 GEM_BUG_ON(!vma->node.allocated);
296 GEM_BUG_ON(upper_32_bits(vma->node.start));
297 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
298 return lower_32_bits(vma->node.start);
299}
300 176
301struct i915_page_dma { 177struct i915_page_dma {
302 struct page *page; 178 struct page *page;
@@ -606,13 +482,20 @@ i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n)
606 px_dma(ppgtt->base.scratch_pd); 482 px_dma(ppgtt->base.scratch_pd);
607} 483}
608 484
485static inline struct i915_ggtt *
486i915_vm_to_ggtt(struct i915_address_space *vm)
487{
488 GEM_BUG_ON(!i915_is_ggtt(vm));
489 return container_of(vm, struct i915_ggtt, base);
490}
491
609int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv); 492int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv);
610int i915_ggtt_init_hw(struct drm_i915_private *dev_priv); 493int i915_ggtt_init_hw(struct drm_i915_private *dev_priv);
611int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv); 494int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv);
612int i915_gem_init_ggtt(struct drm_i915_private *dev_priv); 495int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
613void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); 496void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
614 497
615int i915_ppgtt_init_hw(struct drm_device *dev); 498int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
616void i915_ppgtt_release(struct kref *kref); 499void i915_ppgtt_release(struct kref *kref);
617struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv, 500struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
618 struct drm_i915_file_private *fpriv, 501 struct drm_i915_file_private *fpriv,
@@ -629,8 +512,8 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
629} 512}
630 513
631void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); 514void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
632void i915_gem_suspend_gtt_mappings(struct drm_device *dev); 515void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv);
633void i915_gem_restore_gtt_mappings(struct drm_device *dev); 516void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv);
634 517
635int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, 518int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
636 struct sg_table *pages); 519 struct sg_table *pages);
@@ -653,88 +536,4 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
653#define PIN_OFFSET_FIXED BIT(11) 536#define PIN_OFFSET_FIXED BIT(11)
654#define PIN_OFFSET_MASK (~4095) 537#define PIN_OFFSET_MASK (~4095)
655 538
656int __i915_vma_do_pin(struct i915_vma *vma,
657 u64 size, u64 alignment, u64 flags);
658static inline int __must_check
659i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
660{
661 BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
662 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
663 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
664
665 /* Pin early to prevent the shrinker/eviction logic from destroying
666 * our vma as we insert and bind.
667 */
668 if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
669 return 0;
670
671 return __i915_vma_do_pin(vma, size, alignment, flags);
672}
673
674static inline int i915_vma_pin_count(const struct i915_vma *vma)
675{
676 return vma->flags & I915_VMA_PIN_MASK;
677}
678
679static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
680{
681 return i915_vma_pin_count(vma);
682}
683
684static inline void __i915_vma_pin(struct i915_vma *vma)
685{
686 vma->flags++;
687 GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
688}
689
690static inline void __i915_vma_unpin(struct i915_vma *vma)
691{
692 GEM_BUG_ON(!i915_vma_is_pinned(vma));
693 vma->flags--;
694}
695
696static inline void i915_vma_unpin(struct i915_vma *vma)
697{
698 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
699 __i915_vma_unpin(vma);
700}
701
702/**
703 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
704 * @vma: VMA to iomap
705 *
706 * The passed in VMA has to be pinned in the global GTT mappable region.
707 * An extra pinning of the VMA is acquired for the return iomapping,
708 * the caller must call i915_vma_unpin_iomap to relinquish the pinning
709 * after the iomapping is no longer required.
710 *
711 * Callers must hold the struct_mutex.
712 *
713 * Returns a valid iomapped pointer or ERR_PTR.
714 */
715void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
716#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
717
718/**
719 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
720 * @vma: VMA to unpin
721 *
722 * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
723 *
724 * Callers must hold the struct_mutex. This function is only valid to be
725 * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
726 */
727static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
728{
729 lockdep_assert_held(&vma->vm->dev->struct_mutex);
730 GEM_BUG_ON(vma->iomap == NULL);
731 i915_vma_unpin(vma);
732}
733
734static inline struct page *i915_vma_first_page(struct i915_vma *vma)
735{
736 GEM_BUG_ON(!vma->pages);
737 return sg_page(vma->pages->sgl);
738}
739
740#endif 539#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
new file mode 100644
index 000000000000..6a368de9d81e
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -0,0 +1,338 @@
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#ifndef __I915_GEM_OBJECT_H__
26#define __I915_GEM_OBJECT_H__
27
28#include <linux/reservation.h>
29
30#include <drm/drm_vma_manager.h>
31#include <drm/drm_gem.h>
32#include <drm/drmP.h>
33
34#include <drm/i915_drm.h>
35
36struct drm_i915_gem_object_ops {
37 unsigned int flags;
38#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1
39#define I915_GEM_OBJECT_IS_SHRINKABLE 0x2
40
41 /* Interface between the GEM object and its backing storage.
42 * get_pages() is called once prior to the use of the associated set
43 * of pages before to binding them into the GTT, and put_pages() is
44 * called after we no longer need them. As we expect there to be
45 * associated cost with migrating pages between the backing storage
46 * and making them available for the GPU (e.g. clflush), we may hold
47 * onto the pages after they are no longer referenced by the GPU
48 * in case they may be used again shortly (for example migrating the
49 * pages to a different memory domain within the GTT). put_pages()
50 * will therefore most likely be called when the object itself is
51 * being released or under memory pressure (where we attempt to
52 * reap pages for the shrinker).
53 */
54 struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
55 void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
56
57 int (*dmabuf_export)(struct drm_i915_gem_object *);
58 void (*release)(struct drm_i915_gem_object *);
59};
60
61struct drm_i915_gem_object {
62 struct drm_gem_object base;
63
64 const struct drm_i915_gem_object_ops *ops;
65
66 /** List of VMAs backed by this object */
67 struct list_head vma_list;
68 struct rb_root vma_tree;
69
70 /** Stolen memory for this object, instead of being backed by shmem. */
71 struct drm_mm_node *stolen;
72 struct list_head global_link;
73 union {
74 struct rcu_head rcu;
75 struct llist_node freed;
76 };
77
78 /**
79 * Whether the object is currently in the GGTT mmap.
80 */
81 struct list_head userfault_link;
82
83 /** Used in execbuf to temporarily hold a ref */
84 struct list_head obj_exec_link;
85
86 struct list_head batch_pool_link;
87
88 unsigned long flags;
89
90 /**
91 * Have we taken a reference for the object for incomplete GPU
92 * activity?
93 */
94#define I915_BO_ACTIVE_REF 0
95
96 /*
97 * Is the object to be mapped as read-only to the GPU
98 * Only honoured if hardware has relevant pte bit
99 */
100 unsigned long gt_ro:1;
101 unsigned int cache_level:3;
102 unsigned int cache_dirty:1;
103
104 atomic_t frontbuffer_bits;
105 unsigned int frontbuffer_ggtt_origin; /* write once */
106 struct i915_gem_active frontbuffer_write;
107
108 /** Current tiling stride for the object, if it's tiled. */
109 unsigned int tiling_and_stride;
110#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
111#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
112#define STRIDE_MASK (~TILING_MASK)
113
114 /** Count of VMA actually bound by this object */
115 unsigned int bind_count;
116 unsigned int active_count;
117 unsigned int pin_display;
118
119 struct {
120 struct mutex lock; /* protects the pages and their use */
121 atomic_t pages_pin_count;
122
123 struct sg_table *pages;
124 void *mapping;
125
126 struct i915_gem_object_page_iter {
127 struct scatterlist *sg_pos;
128 unsigned int sg_idx; /* in pages, but 32bit eek! */
129
130 struct radix_tree_root radix;
131 struct mutex lock; /* protects this cache */
132 } get_page;
133
134 /**
135 * Advice: are the backing pages purgeable?
136 */
137 unsigned int madv:2;
138
139 /**
140 * This is set if the object has been written to since the
141 * pages were last acquired.
142 */
143 bool dirty:1;
144
145 /**
146 * This is set if the object has been pinned due to unknown
147 * swizzling.
148 */
149 bool quirked:1;
150 } mm;
151
152 /** Breadcrumb of last rendering to the buffer.
153 * There can only be one writer, but we allow for multiple readers.
154 * If there is a writer that necessarily implies that all other
155 * read requests are complete - but we may only be lazily clearing
156 * the read requests. A read request is naturally the most recent
157 * request on a ring, so we may have two different write and read
158 * requests on one ring where the write request is older than the
159 * read request. This allows for the CPU to read from an active
160 * buffer by only waiting for the write to complete.
161 */
162 struct reservation_object *resv;
163
164 /** References from framebuffers, locks out tiling changes. */
165 unsigned long framebuffer_references;
166
167 /** Record of address bit 17 of each page at last unbind. */
168 unsigned long *bit_17;
169
170 struct i915_gem_userptr {
171 uintptr_t ptr;
172 unsigned read_only :1;
173
174 struct i915_mm_struct *mm;
175 struct i915_mmu_object *mmu_object;
176 struct work_struct *work;
177 } userptr;
178
179 /** for phys allocated objects */
180 struct drm_dma_handle *phys_handle;
181
182 struct reservation_object __builtin_resv;
183};
184
185static inline struct drm_i915_gem_object *
186to_intel_bo(struct drm_gem_object *gem)
187{
188 /* Assert that to_intel_bo(NULL) == NULL */
189 BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
190
191 return container_of(gem, struct drm_i915_gem_object, base);
192}
193
194/**
195 * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
196 * @filp: DRM file private date
197 * @handle: userspace handle
198 *
199 * Returns:
200 *
201 * A pointer to the object named by the handle if such exists on @filp, NULL
202 * otherwise. This object is only valid whilst under the RCU read lock, and
203 * note carefully the object may be in the process of being destroyed.
204 */
205static inline struct drm_i915_gem_object *
206i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
207{
208#ifdef CONFIG_LOCKDEP
209 WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map));
210#endif
211 return idr_find(&file->object_idr, handle);
212}
213
214static inline struct drm_i915_gem_object *
215i915_gem_object_lookup(struct drm_file *file, u32 handle)
216{
217 struct drm_i915_gem_object *obj;
218
219 rcu_read_lock();
220 obj = i915_gem_object_lookup_rcu(file, handle);
221 if (obj && !kref_get_unless_zero(&obj->base.refcount))
222 obj = NULL;
223 rcu_read_unlock();
224
225 return obj;
226}
227
228__deprecated
229extern struct drm_gem_object *
230drm_gem_object_lookup(struct drm_file *file, u32 handle);
231
232__attribute__((nonnull))
233static inline struct drm_i915_gem_object *
234i915_gem_object_get(struct drm_i915_gem_object *obj)
235{
236 drm_gem_object_reference(&obj->base);
237 return obj;
238}
239
240__deprecated
241extern void drm_gem_object_reference(struct drm_gem_object *);
242
243__attribute__((nonnull))
244static inline void
245i915_gem_object_put(struct drm_i915_gem_object *obj)
246{
247 __drm_gem_object_unreference(&obj->base);
248}
249
250__deprecated
251extern void drm_gem_object_unreference(struct drm_gem_object *);
252
253__deprecated
254extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
255
256static inline bool
257i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
258{
259 return atomic_read(&obj->base.refcount.refcount) == 0;
260}
261
262static inline bool
263i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
264{
265 return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
266}
267
268static inline bool
269i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
270{
271 return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;
272}
273
274static inline bool
275i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
276{
277 return obj->active_count;
278}
279
280static inline bool
281i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
282{
283 return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
284}
285
286static inline void
287i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
288{
289 lockdep_assert_held(&obj->base.dev->struct_mutex);
290 __set_bit(I915_BO_ACTIVE_REF, &obj->flags);
291}
292
293static inline void
294i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
295{
296 lockdep_assert_held(&obj->base.dev->struct_mutex);
297 __clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
298}
299
300void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
301
302static inline unsigned int
303i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
304{
305 return obj->tiling_and_stride & TILING_MASK;
306}
307
308static inline bool
309i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
310{
311 return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
312}
313
314static inline unsigned int
315i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
316{
317 return obj->tiling_and_stride & STRIDE_MASK;
318}
319
320static inline struct intel_engine_cs *
321i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
322{
323 struct intel_engine_cs *engine = NULL;
324 struct dma_fence *fence;
325
326 rcu_read_lock();
327 fence = reservation_object_get_excl_rcu(obj->resv);
328 rcu_read_unlock();
329
330 if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))
331 engine = to_request(fence)->engine;
332 dma_fence_put(fence);
333
334 return engine;
335}
336
337#endif
338
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 0b3b051a5683..27e8f257fb39 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -113,6 +113,82 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
113 spin_unlock(&file_priv->mm.lock); 113 spin_unlock(&file_priv->mm.lock);
114} 114}
115 115
116static struct i915_dependency *
117i915_dependency_alloc(struct drm_i915_private *i915)
118{
119 return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
120}
121
122static void
123i915_dependency_free(struct drm_i915_private *i915,
124 struct i915_dependency *dep)
125{
126 kmem_cache_free(i915->dependencies, dep);
127}
128
129static void
130__i915_priotree_add_dependency(struct i915_priotree *pt,
131 struct i915_priotree *signal,
132 struct i915_dependency *dep,
133 unsigned long flags)
134{
135 INIT_LIST_HEAD(&dep->dfs_link);
136 list_add(&dep->wait_link, &signal->waiters_list);
137 list_add(&dep->signal_link, &pt->signalers_list);
138 dep->signaler = signal;
139 dep->flags = flags;
140}
141
142static int
143i915_priotree_add_dependency(struct drm_i915_private *i915,
144 struct i915_priotree *pt,
145 struct i915_priotree *signal)
146{
147 struct i915_dependency *dep;
148
149 dep = i915_dependency_alloc(i915);
150 if (!dep)
151 return -ENOMEM;
152
153 __i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
154 return 0;
155}
156
157static void
158i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
159{
160 struct i915_dependency *dep, *next;
161
162 GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node));
163
164 /* Everyone we depended upon (the fences we wait to be signaled)
165 * should retire before us and remove themselves from our list.
166 * However, retirement is run independently on each timeline and
167 * so we may be called out-of-order.
168 */
169 list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
170 list_del(&dep->wait_link);
171 if (dep->flags & I915_DEPENDENCY_ALLOC)
172 i915_dependency_free(i915, dep);
173 }
174
175 /* Remove ourselves from everyone who depends upon us */
176 list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
177 list_del(&dep->signal_link);
178 if (dep->flags & I915_DEPENDENCY_ALLOC)
179 i915_dependency_free(i915, dep);
180 }
181}
182
183static void
184i915_priotree_init(struct i915_priotree *pt)
185{
186 INIT_LIST_HEAD(&pt->signalers_list);
187 INIT_LIST_HEAD(&pt->waiters_list);
188 RB_CLEAR_NODE(&pt->node);
189 pt->priority = INT_MIN;
190}
191
116void i915_gem_retire_noop(struct i915_gem_active *active, 192void i915_gem_retire_noop(struct i915_gem_active *active,
117 struct drm_i915_gem_request *request) 193 struct drm_i915_gem_request *request)
118{ 194{
@@ -124,7 +200,10 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
124 struct i915_gem_active *active, *next; 200 struct i915_gem_active *active, *next;
125 201
126 lockdep_assert_held(&request->i915->drm.struct_mutex); 202 lockdep_assert_held(&request->i915->drm.struct_mutex);
203 GEM_BUG_ON(!i915_sw_fence_done(&request->submit));
204 GEM_BUG_ON(!i915_sw_fence_done(&request->execute));
127 GEM_BUG_ON(!i915_gem_request_completed(request)); 205 GEM_BUG_ON(!i915_gem_request_completed(request));
206 GEM_BUG_ON(!request->i915->gt.active_requests);
128 207
129 trace_i915_gem_request_retire(request); 208 trace_i915_gem_request_retire(request);
130 209
@@ -142,7 +221,12 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
142 */ 221 */
143 list_del(&request->ring_link); 222 list_del(&request->ring_link);
144 request->ring->last_retired_head = request->postfix; 223 request->ring->last_retired_head = request->postfix;
145 request->i915->gt.active_requests--; 224 if (!--request->i915->gt.active_requests) {
225 GEM_BUG_ON(!request->i915->gt.awake);
226 mod_delayed_work(request->i915->wq,
227 &request->i915->gt.idle_work,
228 msecs_to_jiffies(100));
229 }
146 230
147 /* Walk through the active list, calling retire on each. This allows 231 /* Walk through the active list, calling retire on each. This allows
148 * objects to track their GPU activity and mark themselves as idle 232 * objects to track their GPU activity and mark themselves as idle
@@ -182,6 +266,8 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
182 i915_gem_context_put(request->ctx); 266 i915_gem_context_put(request->ctx);
183 267
184 dma_fence_signal(&request->fence); 268 dma_fence_signal(&request->fence);
269
270 i915_priotree_fini(request->i915, &request->priotree);
185 i915_gem_request_put(request); 271 i915_gem_request_put(request);
186} 272}
187 273
@@ -241,9 +327,8 @@ static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno)
241 327
242 /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ 328 /* If the seqno wraps around, we need to clear the breadcrumb rbtree */
243 if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) { 329 if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) {
244 while (intel_kick_waiters(i915) || intel_kick_signalers(i915)) 330 while (intel_breadcrumbs_busy(i915))
245 yield(); 331 cond_resched(); /* spin until threads are complete */
246 yield();
247 } 332 }
248 atomic_set(&timeline->next_seqno, seqno); 333 atomic_set(&timeline->next_seqno, seqno);
249 334
@@ -307,25 +392,16 @@ static u32 timeline_get_seqno(struct i915_gem_timeline *tl)
307 return atomic_inc_return(&tl->next_seqno); 392 return atomic_inc_return(&tl->next_seqno);
308} 393}
309 394
310static int __i915_sw_fence_call 395void __i915_gem_request_submit(struct drm_i915_gem_request *request)
311submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
312{ 396{
313 struct drm_i915_gem_request *request =
314 container_of(fence, typeof(*request), submit);
315 struct intel_engine_cs *engine = request->engine; 397 struct intel_engine_cs *engine = request->engine;
316 struct intel_timeline *timeline; 398 struct intel_timeline *timeline;
317 unsigned long flags;
318 u32 seqno; 399 u32 seqno;
319 400
320 if (state != FENCE_COMPLETE)
321 return NOTIFY_DONE;
322
323 /* Transfer from per-context onto the global per-engine timeline */ 401 /* Transfer from per-context onto the global per-engine timeline */
324 timeline = engine->timeline; 402 timeline = engine->timeline;
325 GEM_BUG_ON(timeline == request->timeline); 403 GEM_BUG_ON(timeline == request->timeline);
326 404 assert_spin_locked(&timeline->lock);
327 /* Will be called from irq-context when using foreign DMA fences */
328 spin_lock_irqsave(&timeline->lock, flags);
329 405
330 seqno = timeline_get_seqno(timeline->common); 406 seqno = timeline_get_seqno(timeline->common);
331 GEM_BUG_ON(!seqno); 407 GEM_BUG_ON(!seqno);
@@ -345,14 +421,43 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
345 GEM_BUG_ON(!request->global_seqno); 421 GEM_BUG_ON(!request->global_seqno);
346 engine->emit_breadcrumb(request, 422 engine->emit_breadcrumb(request,
347 request->ring->vaddr + request->postfix); 423 request->ring->vaddr + request->postfix);
348 engine->submit_request(request);
349 424
350 spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING); 425 spin_lock(&request->timeline->lock);
351 list_move_tail(&request->link, &timeline->requests); 426 list_move_tail(&request->link, &timeline->requests);
352 spin_unlock(&request->timeline->lock); 427 spin_unlock(&request->timeline->lock);
353 428
354 spin_unlock_irqrestore(&timeline->lock, flags); 429 i915_sw_fence_commit(&request->execute);
430}
431
432void i915_gem_request_submit(struct drm_i915_gem_request *request)
433{
434 struct intel_engine_cs *engine = request->engine;
435 unsigned long flags;
436
437 /* Will be called from irq-context when using foreign fences. */
438 spin_lock_irqsave(&engine->timeline->lock, flags);
439
440 __i915_gem_request_submit(request);
441
442 spin_unlock_irqrestore(&engine->timeline->lock, flags);
443}
444
445static int __i915_sw_fence_call
446submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
447{
448 if (state == FENCE_COMPLETE) {
449 struct drm_i915_gem_request *request =
450 container_of(fence, typeof(*request), submit);
451
452 request->engine->submit_request(request);
453 }
454
455 return NOTIFY_DONE;
456}
355 457
458static int __i915_sw_fence_call
459execute_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
460{
356 return NOTIFY_DONE; 461 return NOTIFY_DONE;
357} 462}
358 463
@@ -441,6 +546,14 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
441 __timeline_get_seqno(req->timeline->common)); 546 __timeline_get_seqno(req->timeline->common));
442 547
443 i915_sw_fence_init(&req->submit, submit_notify); 548 i915_sw_fence_init(&req->submit, submit_notify);
549 i915_sw_fence_init(&req->execute, execute_notify);
550 /* Ensure that the execute fence completes after the submit fence -
551 * as we complete the execute fence from within the submit fence
552 * callback, its completion would otherwise be visible first.
553 */
554 i915_sw_fence_await_sw_fence(&req->execute, &req->submit, &req->execq);
555
556 i915_priotree_init(&req->priotree);
444 557
445 INIT_LIST_HEAD(&req->active_list); 558 INIT_LIST_HEAD(&req->active_list);
446 req->i915 = dev_priv; 559 req->i915 = dev_priv;
@@ -495,6 +608,14 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
495 608
496 GEM_BUG_ON(to == from); 609 GEM_BUG_ON(to == from);
497 610
611 if (to->engine->schedule) {
612 ret = i915_priotree_add_dependency(to->i915,
613 &to->priotree,
614 &from->priotree);
615 if (ret < 0)
616 return ret;
617 }
618
498 if (to->timeline == from->timeline) 619 if (to->timeline == from->timeline)
499 return 0; 620 return 0;
500 621
@@ -650,6 +771,8 @@ static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
650 if (dev_priv->gt.awake) 771 if (dev_priv->gt.awake)
651 return; 772 return;
652 773
774 GEM_BUG_ON(!dev_priv->gt.active_requests);
775
653 intel_runtime_pm_get_noresume(dev_priv); 776 intel_runtime_pm_get_noresume(dev_priv);
654 dev_priv->gt.awake = true; 777 dev_priv->gt.awake = true;
655 778
@@ -718,9 +841,15 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
718 841
719 prev = i915_gem_active_raw(&timeline->last_request, 842 prev = i915_gem_active_raw(&timeline->last_request,
720 &request->i915->drm.struct_mutex); 843 &request->i915->drm.struct_mutex);
721 if (prev) 844 if (prev) {
722 i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, 845 i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
723 &request->submitq); 846 &request->submitq);
847 if (engine->schedule)
848 __i915_priotree_add_dependency(&request->priotree,
849 &prev->priotree,
850 &request->dep,
851 0);
852 }
724 853
725 spin_lock_irq(&timeline->lock); 854 spin_lock_irq(&timeline->lock);
726 list_add_tail(&request->link, &timeline->requests); 855 list_add_tail(&request->link, &timeline->requests);
@@ -737,6 +866,19 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
737 866
738 i915_gem_mark_busy(engine); 867 i915_gem_mark_busy(engine);
739 868
869 /* Let the backend know a new request has arrived that may need
870 * to adjust the existing execution schedule due to a high priority
871 * request - i.e. we may want to preempt the current request in order
872 * to run a high priority dependency chain *before* we can execute this
873 * request.
874 *
875 * This is called before the request is ready to run so that we can
876 * decide whether to preempt the entire chain so that it is ready to
877 * run at the earliest possible convenience.
878 */
879 if (engine->schedule)
880 engine->schedule(request, request->ctx->priority);
881
740 local_bh_disable(); 882 local_bh_disable();
741 i915_sw_fence_commit(&request->submit); 883 i915_sw_fence_commit(&request->submit);
742 local_bh_enable(); /* Kick the execlists tasklet if just scheduled */ 884 local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
@@ -817,9 +959,9 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req,
817} 959}
818 960
819static long 961static long
820__i915_request_wait_for_submit(struct drm_i915_gem_request *request, 962__i915_request_wait_for_execute(struct drm_i915_gem_request *request,
821 unsigned int flags, 963 unsigned int flags,
822 long timeout) 964 long timeout)
823{ 965{
824 const int state = flags & I915_WAIT_INTERRUPTIBLE ? 966 const int state = flags & I915_WAIT_INTERRUPTIBLE ?
825 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; 967 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
@@ -831,9 +973,9 @@ __i915_request_wait_for_submit(struct drm_i915_gem_request *request,
831 add_wait_queue(q, &reset); 973 add_wait_queue(q, &reset);
832 974
833 do { 975 do {
834 prepare_to_wait(&request->submit.wait, &wait, state); 976 prepare_to_wait(&request->execute.wait, &wait, state);
835 977
836 if (i915_sw_fence_done(&request->submit)) 978 if (i915_sw_fence_done(&request->execute))
837 break; 979 break;
838 980
839 if (flags & I915_WAIT_LOCKED && 981 if (flags & I915_WAIT_LOCKED &&
@@ -851,7 +993,7 @@ __i915_request_wait_for_submit(struct drm_i915_gem_request *request,
851 993
852 timeout = io_schedule_timeout(timeout); 994 timeout = io_schedule_timeout(timeout);
853 } while (timeout); 995 } while (timeout);
854 finish_wait(&request->submit.wait, &wait); 996 finish_wait(&request->execute.wait, &wait);
855 997
856 if (flags & I915_WAIT_LOCKED) 998 if (flags & I915_WAIT_LOCKED)
857 remove_wait_queue(q, &reset); 999 remove_wait_queue(q, &reset);
@@ -903,13 +1045,14 @@ long i915_wait_request(struct drm_i915_gem_request *req,
903 1045
904 trace_i915_gem_request_wait_begin(req); 1046 trace_i915_gem_request_wait_begin(req);
905 1047
906 if (!i915_sw_fence_done(&req->submit)) { 1048 if (!i915_sw_fence_done(&req->execute)) {
907 timeout = __i915_request_wait_for_submit(req, flags, timeout); 1049 timeout = __i915_request_wait_for_execute(req, flags, timeout);
908 if (timeout < 0) 1050 if (timeout < 0)
909 goto complete; 1051 goto complete;
910 1052
911 GEM_BUG_ON(!i915_sw_fence_done(&req->submit)); 1053 GEM_BUG_ON(!i915_sw_fence_done(&req->execute));
912 } 1054 }
1055 GEM_BUG_ON(!i915_sw_fence_done(&req->submit));
913 GEM_BUG_ON(!req->global_seqno); 1056 GEM_BUG_ON(!req->global_seqno);
914 1057
915 /* Optimistic short spin before touching IRQs */ 1058 /* Optimistic short spin before touching IRQs */
@@ -1013,13 +1156,6 @@ void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
1013 if (!dev_priv->gt.active_requests) 1156 if (!dev_priv->gt.active_requests)
1014 return; 1157 return;
1015 1158
1016 GEM_BUG_ON(!dev_priv->gt.awake);
1017
1018 for_each_engine(engine, dev_priv, id) 1159 for_each_engine(engine, dev_priv, id)
1019 engine_retire_requests(engine); 1160 engine_retire_requests(engine);
1020
1021 if (!dev_priv->gt.active_requests)
1022 mod_delayed_work(dev_priv->wq,
1023 &dev_priv->gt.idle_work,
1024 msecs_to_jiffies(100));
1025} 1161}
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
index 75f8360b3421..e2b077df2da0 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -30,6 +30,9 @@
30#include "i915_gem.h" 30#include "i915_gem.h"
31#include "i915_sw_fence.h" 31#include "i915_sw_fence.h"
32 32
33struct drm_file;
34struct drm_i915_gem_object;
35
33struct intel_wait { 36struct intel_wait {
34 struct rb_node node; 37 struct rb_node node;
35 struct task_struct *tsk; 38 struct task_struct *tsk;
@@ -41,6 +44,33 @@ struct intel_signal_node {
41 struct intel_wait wait; 44 struct intel_wait wait;
42}; 45};
43 46
47struct i915_dependency {
48 struct i915_priotree *signaler;
49 struct list_head signal_link;
50 struct list_head wait_link;
51 struct list_head dfs_link;
52 unsigned long flags;
53#define I915_DEPENDENCY_ALLOC BIT(0)
54};
55
56/* Requests exist in a complex web of interdependencies. Each request
57 * has to wait for some other request to complete before it is ready to be run
58 * (e.g. we have to wait until the pixels have been rendering into a texture
59 * before we can copy from it). We track the readiness of a request in terms
60 * of fences, but we also need to keep the dependency tree for the lifetime
61 * of the request (beyond the life of an individual fence). We use the tree
62 * at various points to reorder the requests whilst keeping the requests
63 * in order with respect to their various dependencies.
64 */
65struct i915_priotree {
66 struct list_head signalers_list; /* those before us, we depend upon */
67 struct list_head waiters_list; /* those after us, they depend upon us */
68 struct rb_node node;
69 int priority;
70#define I915_PRIORITY_MAX 1024
71#define I915_PRIORITY_MIN (-I915_PRIORITY_MAX)
72};
73
44/** 74/**
45 * Request queue structure. 75 * Request queue structure.
46 * 76 *
@@ -84,8 +114,34 @@ struct drm_i915_gem_request {
84 struct intel_timeline *timeline; 114 struct intel_timeline *timeline;
85 struct intel_signal_node signaling; 115 struct intel_signal_node signaling;
86 116
117 /* Fences for the various phases in the request's lifetime.
118 *
119 * The submit fence is used to await upon all of the request's
120 * dependencies. When it is signaled, the request is ready to run.
121 * It is used by the driver to then queue the request for execution.
122 *
123 * The execute fence is used to signal when the request has been
124 * sent to hardware.
125 *
126 * It is illegal for the submit fence of one request to wait upon the
127 * execute fence of an earlier request. It should be sufficient to
128 * wait upon the submit fence of the earlier request.
129 */
87 struct i915_sw_fence submit; 130 struct i915_sw_fence submit;
131 struct i915_sw_fence execute;
88 wait_queue_t submitq; 132 wait_queue_t submitq;
133 wait_queue_t execq;
134
135 /* A list of everyone we wait upon, and everyone who waits upon us.
136 * Even though we will not be submitted to the hardware before the
137 * submit fence is signaled (it waits for all external events as well
138 * as our own requests), the scheduler still needs to know the
139 * dependency tree for the lifetime of the request (from execbuf
140 * to retirement), i.e. bidirectional dependency information for the
141 * request not tied to individual fences.
142 */
143 struct i915_priotree priotree;
144 struct i915_dependency dep;
89 145
90 u32 global_seqno; 146 u32 global_seqno;
91 147
@@ -143,9 +199,6 @@ struct drm_i915_gem_request {
143 struct drm_i915_file_private *file_priv; 199 struct drm_i915_file_private *file_priv;
144 /** file_priv list entry for this request */ 200 /** file_priv list entry for this request */
145 struct list_head client_list; 201 struct list_head client_list;
146
147 /** Link in the execlist submission queue, guarded by execlist_lock. */
148 struct list_head execlist_link;
149}; 202};
150 203
151extern const struct dma_fence_ops i915_fence_ops; 204extern const struct dma_fence_ops i915_fence_ops;
@@ -162,18 +215,6 @@ int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
162 struct drm_file *file); 215 struct drm_file *file);
163void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); 216void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
164 217
165static inline u32
166i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
167{
168 return req ? req->global_seqno : 0;
169}
170
171static inline struct intel_engine_cs *
172i915_gem_request_get_engine(struct drm_i915_gem_request *req)
173{
174 return req ? req->engine : NULL;
175}
176
177static inline struct drm_i915_gem_request * 218static inline struct drm_i915_gem_request *
178to_request(struct dma_fence *fence) 219to_request(struct dma_fence *fence)
179{ 220{
@@ -226,6 +267,9 @@ void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
226#define i915_add_request_no_flush(req) \ 267#define i915_add_request_no_flush(req) \
227 __i915_add_request(req, false) 268 __i915_add_request(req, false)
228 269
270void __i915_gem_request_submit(struct drm_i915_gem_request *request);
271void i915_gem_request_submit(struct drm_i915_gem_request *request);
272
229struct intel_rps_client; 273struct intel_rps_client;
230#define NO_WAITBOOST ERR_PTR(-1) 274#define NO_WAITBOOST ERR_PTR(-1)
231#define IS_RPS_CLIENT(p) (!IS_ERR(p)) 275#define IS_RPS_CLIENT(p) (!IS_ERR(p))
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index b1d367dba347..ebaa941c83af 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -89,9 +89,8 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
89 mutex_unlock(&dev_priv->mm.stolen_lock); 89 mutex_unlock(&dev_priv->mm.stolen_lock);
90} 90}
91 91
92static unsigned long i915_stolen_to_physical(struct drm_device *dev) 92static unsigned long i915_stolen_to_physical(struct drm_i915_private *dev_priv)
93{ 93{
94 struct drm_i915_private *dev_priv = to_i915(dev);
95 struct pci_dev *pdev = dev_priv->drm.pdev; 94 struct pci_dev *pdev = dev_priv->drm.pdev;
96 struct i915_ggtt *ggtt = &dev_priv->ggtt; 95 struct i915_ggtt *ggtt = &dev_priv->ggtt;
97 struct resource *r; 96 struct resource *r;
@@ -253,7 +252,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
253 * kernel. So if the region is already marked as busy, something 252 * kernel. So if the region is already marked as busy, something
254 * is seriously wrong. 253 * is seriously wrong.
255 */ 254 */
256 r = devm_request_mem_region(dev->dev, base, ggtt->stolen_size, 255 r = devm_request_mem_region(dev_priv->drm.dev, base, ggtt->stolen_size,
257 "Graphics Stolen Memory"); 256 "Graphics Stolen Memory");
258 if (r == NULL) { 257 if (r == NULL) {
259 /* 258 /*
@@ -264,7 +263,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
264 * PCI bus, but have an off-by-one error. Hence retry the 263 * PCI bus, but have an off-by-one error. Hence retry the
265 * reservation starting from 1 instead of 0. 264 * reservation starting from 1 instead of 0.
266 */ 265 */
267 r = devm_request_mem_region(dev->dev, base + 1, 266 r = devm_request_mem_region(dev_priv->drm.dev, base + 1,
268 ggtt->stolen_size - 1, 267 ggtt->stolen_size - 1,
269 "Graphics Stolen Memory"); 268 "Graphics Stolen Memory");
270 /* 269 /*
@@ -408,9 +407,8 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
408 *size = stolen_top - *base; 407 *size = stolen_top - *base;
409} 408}
410 409
411int i915_gem_init_stolen(struct drm_device *dev) 410int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
412{ 411{
413 struct drm_i915_private *dev_priv = to_i915(dev);
414 struct i915_ggtt *ggtt = &dev_priv->ggtt; 412 struct i915_ggtt *ggtt = &dev_priv->ggtt;
415 unsigned long reserved_total, reserved_base = 0, reserved_size; 413 unsigned long reserved_total, reserved_base = 0, reserved_size;
416 unsigned long stolen_top; 414 unsigned long stolen_top;
@@ -418,7 +416,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
418 mutex_init(&dev_priv->mm.stolen_lock); 416 mutex_init(&dev_priv->mm.stolen_lock);
419 417
420#ifdef CONFIG_INTEL_IOMMU 418#ifdef CONFIG_INTEL_IOMMU
421 if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) { 419 if (intel_iommu_gfx_mapped && INTEL_GEN(dev_priv) < 8) {
422 DRM_INFO("DMAR active, disabling use of stolen memory\n"); 420 DRM_INFO("DMAR active, disabling use of stolen memory\n");
423 return 0; 421 return 0;
424 } 422 }
@@ -427,7 +425,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
427 if (ggtt->stolen_size == 0) 425 if (ggtt->stolen_size == 0)
428 return 0; 426 return 0;
429 427
430 dev_priv->mm.stolen_base = i915_stolen_to_physical(dev); 428 dev_priv->mm.stolen_base = i915_stolen_to_physical(dev_priv);
431 if (dev_priv->mm.stolen_base == 0) 429 if (dev_priv->mm.stolen_base == 0)
432 return 0; 430 return 0;
433 431
@@ -515,12 +513,10 @@ i915_pages_create_for_stolen(struct drm_device *dev,
515 u32 offset, u32 size) 513 u32 offset, u32 size)
516{ 514{
517 struct drm_i915_private *dev_priv = to_i915(dev); 515 struct drm_i915_private *dev_priv = to_i915(dev);
518 struct i915_ggtt *ggtt = &dev_priv->ggtt;
519 struct sg_table *st; 516 struct sg_table *st;
520 struct scatterlist *sg; 517 struct scatterlist *sg;
521 518
522 DRM_DEBUG_DRIVER("offset=0x%x, size=%d\n", offset, size); 519 GEM_BUG_ON(offset > dev_priv->ggtt.stolen_size - size);
523 BUG_ON(offset > ggtt->stolen_size - size);
524 520
525 /* We hide that we have no struct page backing our stolen object 521 /* We hide that we have no struct page backing our stolen object
526 * by wrapping the contiguous physical allocation with a fake 522 * by wrapping the contiguous physical allocation with a fake
@@ -529,11 +525,11 @@ i915_pages_create_for_stolen(struct drm_device *dev,
529 525
530 st = kmalloc(sizeof(*st), GFP_KERNEL); 526 st = kmalloc(sizeof(*st), GFP_KERNEL);
531 if (st == NULL) 527 if (st == NULL)
532 return NULL; 528 return ERR_PTR(-ENOMEM);
533 529
534 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 530 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
535 kfree(st); 531 kfree(st);
536 return NULL; 532 return ERR_PTR(-ENOMEM);
537 } 533 }
538 534
539 sg = st->sgl; 535 sg = st->sgl;
@@ -557,7 +553,7 @@ i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
557static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, 553static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
558 struct sg_table *pages) 554 struct sg_table *pages)
559{ 555{
560 /* Should only be called during free */ 556 /* Should only be called from i915_gem_object_release_stolen() */
561 sg_free_table(pages); 557 sg_free_table(pages);
562 kfree(pages); 558 kfree(pages);
563} 559}
@@ -566,15 +562,16 @@ static void
566i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) 562i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
567{ 563{
568 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 564 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
565 struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
566
567 GEM_BUG_ON(!stolen);
569 568
570 __i915_gem_object_unpin_pages(obj); 569 __i915_gem_object_unpin_pages(obj);
571 570
572 if (obj->stolen) { 571 i915_gem_stolen_remove_node(dev_priv, stolen);
573 i915_gem_stolen_remove_node(dev_priv, obj->stolen); 572 kfree(stolen);
574 kfree(obj->stolen);
575 obj->stolen = NULL;
576 }
577} 573}
574
578static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = { 575static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
579 .get_pages = i915_gem_object_get_pages_stolen, 576 .get_pages = i915_gem_object_get_pages_stolen,
580 .put_pages = i915_gem_object_put_pages_stolen, 577 .put_pages = i915_gem_object_put_pages_stolen,
@@ -596,7 +593,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
596 593
597 obj->stolen = stolen; 594 obj->stolen = stolen;
598 obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; 595 obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
599 obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE; 596 obj->cache_level = HAS_LLC(to_i915(dev)) ?
597 I915_CACHE_LLC : I915_CACHE_NONE;
600 598
601 if (i915_gem_object_pin_pages(obj)) 599 if (i915_gem_object_pin_pages(obj))
602 goto cleanup; 600 goto cleanup;
@@ -619,7 +617,6 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size)
619 if (!drm_mm_initialized(&dev_priv->mm.stolen)) 617 if (!drm_mm_initialized(&dev_priv->mm.stolen))
620 return NULL; 618 return NULL;
621 619
622 DRM_DEBUG_KMS("creating stolen object: size=%x\n", size);
623 if (size == 0) 620 if (size == 0)
624 return NULL; 621 return NULL;
625 622
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 251d51b01174..c85e7b06bdba 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -60,9 +60,9 @@
60 60
61/* Check pitch constriants for all chips & tiling formats */ 61/* Check pitch constriants for all chips & tiling formats */
62static bool 62static bool
63i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) 63i915_tiling_ok(struct drm_i915_private *dev_priv,
64 int stride, int size, int tiling_mode)
64{ 65{
65 struct drm_i915_private *dev_priv = to_i915(dev);
66 int tile_width; 66 int tile_width;
67 67
68 /* Linear is always fine */ 68 /* Linear is always fine */
@@ -81,10 +81,10 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
81 /* check maximum stride & object size */ 81 /* check maximum stride & object size */
82 /* i965+ stores the end address of the gtt mapping in the fence 82 /* i965+ stores the end address of the gtt mapping in the fence
83 * reg, so dont bother to check the size */ 83 * reg, so dont bother to check the size */
84 if (INTEL_INFO(dev)->gen >= 7) { 84 if (INTEL_GEN(dev_priv) >= 7) {
85 if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL) 85 if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
86 return false; 86 return false;
87 } else if (INTEL_INFO(dev)->gen >= 4) { 87 } else if (INTEL_GEN(dev_priv) >= 4) {
88 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) 88 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
89 return false; 89 return false;
90 } else { 90 } else {
@@ -104,7 +104,7 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
104 return false; 104 return false;
105 105
106 /* 965+ just needs multiples of tile width */ 106 /* 965+ just needs multiples of tile width */
107 if (INTEL_INFO(dev)->gen >= 4) { 107 if (INTEL_GEN(dev_priv) >= 4) {
108 if (stride & (tile_width - 1)) 108 if (stride & (tile_width - 1))
109 return false; 109 return false;
110 return true; 110 return true;
@@ -199,7 +199,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
199 if (!obj) 199 if (!obj)
200 return -ENOENT; 200 return -ENOENT;
201 201
202 if (!i915_tiling_ok(dev, 202 if (!i915_tiling_ok(dev_priv,
203 args->stride, obj->base.size, args->tiling_mode)) { 203 args->stride, obj->base.size, args->tiling_mode)) {
204 i915_gem_object_put(obj); 204 i915_gem_object_put(obj);
205 return -EINVAL; 205 return -EINVAL;
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
index fc8f13a79f8f..bf8a471b61e6 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.c
@@ -24,9 +24,11 @@
24 24
25#include "i915_drv.h" 25#include "i915_drv.h"
26 26
27int i915_gem_timeline_init(struct drm_i915_private *i915, 27static int __i915_gem_timeline_init(struct drm_i915_private *i915,
28 struct i915_gem_timeline *timeline, 28 struct i915_gem_timeline *timeline,
29 const char *name) 29 const char *name,
30 struct lock_class_key *lockclass,
31 const char *lockname)
30{ 32{
31 unsigned int i; 33 unsigned int i;
32 u64 fences; 34 u64 fences;
@@ -47,8 +49,11 @@ int i915_gem_timeline_init(struct drm_i915_private *i915,
47 49
48 tl->fence_context = fences++; 50 tl->fence_context = fences++;
49 tl->common = timeline; 51 tl->common = timeline;
50 52#ifdef CONFIG_DEBUG_SPINLOCK
53 __raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass);
54#else
51 spin_lock_init(&tl->lock); 55 spin_lock_init(&tl->lock);
56#endif
52 init_request_active(&tl->last_request, NULL); 57 init_request_active(&tl->last_request, NULL);
53 INIT_LIST_HEAD(&tl->requests); 58 INIT_LIST_HEAD(&tl->requests);
54 } 59 }
@@ -56,6 +61,26 @@ int i915_gem_timeline_init(struct drm_i915_private *i915,
56 return 0; 61 return 0;
57} 62}
58 63
64int i915_gem_timeline_init(struct drm_i915_private *i915,
65 struct i915_gem_timeline *timeline,
66 const char *name)
67{
68 static struct lock_class_key class;
69
70 return __i915_gem_timeline_init(i915, timeline, name,
71 &class, "&timeline->lock");
72}
73
74int i915_gem_timeline_init__global(struct drm_i915_private *i915)
75{
76 static struct lock_class_key class;
77
78 return __i915_gem_timeline_init(i915,
79 &i915->gt.global_timeline,
80 "[execution]",
81 &class, "&global_timeline->lock");
82}
83
59void i915_gem_timeline_fini(struct i915_gem_timeline *tl) 84void i915_gem_timeline_fini(struct i915_gem_timeline *tl)
60{ 85{
61 lockdep_assert_held(&tl->i915->drm.struct_mutex); 86 lockdep_assert_held(&tl->i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
index f2bf7b1d49a1..98d99a62b4ae 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -67,6 +67,7 @@ struct i915_gem_timeline {
67int i915_gem_timeline_init(struct drm_i915_private *i915, 67int i915_gem_timeline_init(struct drm_i915_private *i915,
68 struct i915_gem_timeline *tl, 68 struct i915_gem_timeline *tl,
69 const char *name); 69 const char *name);
70int i915_gem_timeline_init__global(struct drm_i915_private *i915);
70void i915_gem_timeline_fini(struct i915_gem_timeline *tl); 71void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
71 72
72#endif 73#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 64261639f547..107ddf51065e 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -753,12 +753,13 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
753int 753int
754i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 754i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
755{ 755{
756 struct drm_i915_private *dev_priv = to_i915(dev);
756 struct drm_i915_gem_userptr *args = data; 757 struct drm_i915_gem_userptr *args = data;
757 struct drm_i915_gem_object *obj; 758 struct drm_i915_gem_object *obj;
758 int ret; 759 int ret;
759 u32 handle; 760 u32 handle;
760 761
761 if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) { 762 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
762 /* We cannot support coherent userptr objects on hw without 763 /* We cannot support coherent userptr objects on hw without
763 * LLC and broken snooping. 764 * LLC and broken snooping.
764 */ 765 */
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 204093f3eaa5..ae84aa4b1467 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -528,8 +528,7 @@ static void err_print_capabilities(struct drm_i915_error_state_buf *m,
528int i915_error_state_to_str(struct drm_i915_error_state_buf *m, 528int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
529 const struct i915_error_state_file_priv *error_priv) 529 const struct i915_error_state_file_priv *error_priv)
530{ 530{
531 struct drm_device *dev = error_priv->dev; 531 struct drm_i915_private *dev_priv = to_i915(error_priv->dev);
532 struct drm_i915_private *dev_priv = to_i915(dev);
533 struct pci_dev *pdev = dev_priv->drm.pdev; 532 struct pci_dev *pdev = dev_priv->drm.pdev;
534 struct drm_i915_error_state *error = error_priv->error; 533 struct drm_i915_error_state *error = error_priv->error;
535 struct drm_i915_error_object *obj; 534 struct drm_i915_error_object *obj;
@@ -573,7 +572,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
573 pdev->subsystem_device); 572 pdev->subsystem_device);
574 err_printf(m, "IOMMU enabled?: %d\n", error->iommu); 573 err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
575 574
576 if (HAS_CSR(dev)) { 575 if (HAS_CSR(dev_priv)) {
577 struct intel_csr *csr = &dev_priv->csr; 576 struct intel_csr *csr = &dev_priv->csr;
578 577
579 err_printf(m, "DMC loaded: %s\n", 578 err_printf(m, "DMC loaded: %s\n",
@@ -585,7 +584,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
585 584
586 err_printf(m, "EIR: 0x%08x\n", error->eir); 585 err_printf(m, "EIR: 0x%08x\n", error->eir);
587 err_printf(m, "IER: 0x%08x\n", error->ier); 586 err_printf(m, "IER: 0x%08x\n", error->ier);
588 if (INTEL_INFO(dev)->gen >= 8) { 587 if (INTEL_GEN(dev_priv) >= 8) {
589 for (i = 0; i < 4; i++) 588 for (i = 0; i < 4; i++)
590 err_printf(m, "GTIER gt %d: 0x%08x\n", i, 589 err_printf(m, "GTIER gt %d: 0x%08x\n", i,
591 error->gtier[i]); 590 error->gtier[i]);
@@ -600,10 +599,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
600 for (i = 0; i < dev_priv->num_fence_regs; i++) 599 for (i = 0; i < dev_priv->num_fence_regs; i++)
601 err_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]); 600 err_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]);
602 601
603 if (INTEL_INFO(dev)->gen >= 6) { 602 if (INTEL_GEN(dev_priv) >= 6) {
604 err_printf(m, "ERROR: 0x%08x\n", error->error); 603 err_printf(m, "ERROR: 0x%08x\n", error->error);
605 604
606 if (INTEL_INFO(dev)->gen >= 8) 605 if (INTEL_GEN(dev_priv) >= 8)
607 err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n", 606 err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
608 error->fault_data1, error->fault_data0); 607 error->fault_data1, error->fault_data0);
609 608
@@ -708,7 +707,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
708 intel_overlay_print_error_state(m, error->overlay); 707 intel_overlay_print_error_state(m, error->overlay);
709 708
710 if (error->display) 709 if (error->display)
711 intel_display_print_error_state(m, dev, error->display); 710 intel_display_print_error_state(m, dev_priv, error->display);
712 711
713out: 712out:
714 if (m->bytes == 0 && m->err) 713 if (m->bytes == 0 && m->err)
@@ -861,16 +860,19 @@ out:
861static inline uint32_t 860static inline uint32_t
862__active_get_seqno(struct i915_gem_active *active) 861__active_get_seqno(struct i915_gem_active *active)
863{ 862{
864 return i915_gem_request_get_seqno(__i915_gem_active_peek(active)); 863 struct drm_i915_gem_request *request;
864
865 request = __i915_gem_active_peek(active);
866 return request ? request->global_seqno : 0;
865} 867}
866 868
867static inline int 869static inline int
868__active_get_engine_id(struct i915_gem_active *active) 870__active_get_engine_id(struct i915_gem_active *active)
869{ 871{
870 struct intel_engine_cs *engine; 872 struct drm_i915_gem_request *request;
871 873
872 engine = i915_gem_request_get_engine(__i915_gem_active_peek(active)); 874 request = __i915_gem_active_peek(active);
873 return engine ? engine->id : -1; 875 return request ? request->engine->id : -1;
874} 876}
875 877
876static void capture_bo(struct drm_i915_error_buffer *err, 878static void capture_bo(struct drm_i915_error_buffer *err,
@@ -884,8 +886,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
884 886
885 for (i = 0; i < I915_NUM_ENGINES; i++) 887 for (i = 0; i < I915_NUM_ENGINES; i++)
886 err->rseqno[i] = __active_get_seqno(&vma->last_read[i]); 888 err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
887 err->wseqno = __active_get_seqno(&vma->last_write); 889 err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
888 err->engine = __active_get_engine_id(&vma->last_write); 890 err->engine = __active_get_engine_id(&obj->frontbuffer_write);
889 891
890 err->gtt_offset = vma->node.start; 892 err->gtt_offset = vma->node.start;
891 err->read_domains = obj->base.read_domains; 893 err->read_domains = obj->base.read_domains;
@@ -1440,7 +1442,6 @@ static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv,
1440static void i915_capture_reg_state(struct drm_i915_private *dev_priv, 1442static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
1441 struct drm_i915_error_state *error) 1443 struct drm_i915_error_state *error)
1442{ 1444{
1443 struct drm_device *dev = &dev_priv->drm;
1444 int i; 1445 int i;
1445 1446
1446 /* General organization 1447 /* General organization
@@ -1461,7 +1462,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
1461 if (IS_GEN7(dev_priv)) 1462 if (IS_GEN7(dev_priv))
1462 error->err_int = I915_READ(GEN7_ERR_INT); 1463 error->err_int = I915_READ(GEN7_ERR_INT);
1463 1464
1464 if (INTEL_INFO(dev)->gen >= 8) { 1465 if (INTEL_GEN(dev_priv) >= 8) {
1465 error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0); 1466 error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
1466 error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1); 1467 error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
1467 } 1468 }
@@ -1473,10 +1474,10 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
1473 } 1474 }
1474 1475
1475 /* 2: Registers which belong to multiple generations */ 1476 /* 2: Registers which belong to multiple generations */
1476 if (INTEL_INFO(dev)->gen >= 7) 1477 if (INTEL_GEN(dev_priv) >= 7)
1477 error->forcewake = I915_READ_FW(FORCEWAKE_MT); 1478 error->forcewake = I915_READ_FW(FORCEWAKE_MT);
1478 1479
1479 if (INTEL_INFO(dev)->gen >= 6) { 1480 if (INTEL_GEN(dev_priv) >= 6) {
1480 error->derrmr = I915_READ(DERRMR); 1481 error->derrmr = I915_READ(DERRMR);
1481 error->error = I915_READ(ERROR_GEN6); 1482 error->error = I915_READ(ERROR_GEN6);
1482 error->done_reg = I915_READ(DONE_REG); 1483 error->done_reg = I915_READ(DONE_REG);
@@ -1489,10 +1490,10 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
1489 } 1490 }
1490 1491
1491 /* 4: Everything else */ 1492 /* 4: Everything else */
1492 if (HAS_HW_CONTEXTS(dev)) 1493 if (HAS_HW_CONTEXTS(dev_priv))
1493 error->ccid = I915_READ(CCID); 1494 error->ccid = I915_READ(CCID);
1494 1495
1495 if (INTEL_INFO(dev)->gen >= 8) { 1496 if (INTEL_GEN(dev_priv) >= 8) {
1496 error->ier = I915_READ(GEN8_DE_MISC_IER); 1497 error->ier = I915_READ(GEN8_DE_MISC_IER);
1497 for (i = 0; i < 4; i++) 1498 for (i = 0; i < 4; i++)
1498 error->gtier[i] = I915_READ(GEN8_GT_IER(i)); 1499 error->gtier[i] = I915_READ(GEN8_GT_IER(i));
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index 666dab7a675a..4462112725ef 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -629,11 +629,23 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
629static void i915_guc_submit(struct drm_i915_gem_request *rq) 629static void i915_guc_submit(struct drm_i915_gem_request *rq)
630{ 630{
631 struct drm_i915_private *dev_priv = rq->i915; 631 struct drm_i915_private *dev_priv = rq->i915;
632 unsigned int engine_id = rq->engine->id; 632 struct intel_engine_cs *engine = rq->engine;
633 unsigned int engine_id = engine->id;
633 struct intel_guc *guc = &rq->i915->guc; 634 struct intel_guc *guc = &rq->i915->guc;
634 struct i915_guc_client *client = guc->execbuf_client; 635 struct i915_guc_client *client = guc->execbuf_client;
635 int b_ret; 636 int b_ret;
636 637
638 /* We keep the previous context alive until we retire the following
639 * request. This ensures that any the context object is still pinned
640 * for any residual writes the HW makes into it on the context switch
641 * into the next object following the breadcrumb. Otherwise, we may
642 * retire the context too early.
643 */
644 rq->previous_context = engine->last_context;
645 engine->last_context = rq->ctx;
646
647 i915_gem_request_submit(rq);
648
637 spin_lock(&client->wq_lock); 649 spin_lock(&client->wq_lock);
638 guc_wq_item_append(client, rq); 650 guc_wq_item_append(client, rq);
639 651
@@ -1520,6 +1532,7 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
1520 /* Take over from manual control of ELSP (execlists) */ 1532 /* Take over from manual control of ELSP (execlists) */
1521 for_each_engine(engine, dev_priv, id) { 1533 for_each_engine(engine, dev_priv, id) {
1522 engine->submit_request = i915_guc_submit; 1534 engine->submit_request = i915_guc_submit;
1535 engine->schedule = NULL;
1523 1536
1524 /* Replay the current set of previously submitted requests */ 1537 /* Replay the current set of previously submitted requests */
1525 list_for_each_entry(request, 1538 list_for_each_entry(request,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 6d7505b5c5e7..07ca71cabb2b 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -2848,10 +2848,8 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2848 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); 2848 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2849} 2849}
2850 2850
2851static void ibx_irq_reset(struct drm_device *dev) 2851static void ibx_irq_reset(struct drm_i915_private *dev_priv)
2852{ 2852{
2853 struct drm_i915_private *dev_priv = to_i915(dev);
2854
2855 if (HAS_PCH_NOP(dev_priv)) 2853 if (HAS_PCH_NOP(dev_priv))
2856 return; 2854 return;
2857 2855
@@ -2881,12 +2879,10 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
2881 POSTING_READ(SDEIER); 2879 POSTING_READ(SDEIER);
2882} 2880}
2883 2881
2884static void gen5_gt_irq_reset(struct drm_device *dev) 2882static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
2885{ 2883{
2886 struct drm_i915_private *dev_priv = to_i915(dev);
2887
2888 GEN5_IRQ_RESET(GT); 2884 GEN5_IRQ_RESET(GT);
2889 if (INTEL_INFO(dev)->gen >= 6) 2885 if (INTEL_GEN(dev_priv) >= 6)
2890 GEN5_IRQ_RESET(GEN6_PM); 2886 GEN5_IRQ_RESET(GEN6_PM);
2891} 2887}
2892 2888
@@ -2951,9 +2947,9 @@ static void ironlake_irq_reset(struct drm_device *dev)
2951 if (IS_GEN7(dev_priv)) 2947 if (IS_GEN7(dev_priv))
2952 I915_WRITE(GEN7_ERR_INT, 0xffffffff); 2948 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
2953 2949
2954 gen5_gt_irq_reset(dev); 2950 gen5_gt_irq_reset(dev_priv);
2955 2951
2956 ibx_irq_reset(dev); 2952 ibx_irq_reset(dev_priv);
2957} 2953}
2958 2954
2959static void valleyview_irq_preinstall(struct drm_device *dev) 2955static void valleyview_irq_preinstall(struct drm_device *dev)
@@ -2963,7 +2959,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
2963 I915_WRITE(VLV_MASTER_IER, 0); 2959 I915_WRITE(VLV_MASTER_IER, 0);
2964 POSTING_READ(VLV_MASTER_IER); 2960 POSTING_READ(VLV_MASTER_IER);
2965 2961
2966 gen5_gt_irq_reset(dev); 2962 gen5_gt_irq_reset(dev_priv);
2967 2963
2968 spin_lock_irq(&dev_priv->irq_lock); 2964 spin_lock_irq(&dev_priv->irq_lock);
2969 if (dev_priv->display_irqs_enabled) 2965 if (dev_priv->display_irqs_enabled)
@@ -2999,7 +2995,7 @@ static void gen8_irq_reset(struct drm_device *dev)
2999 GEN5_IRQ_RESET(GEN8_PCU_); 2995 GEN5_IRQ_RESET(GEN8_PCU_);
3000 2996
3001 if (HAS_PCH_SPLIT(dev_priv)) 2997 if (HAS_PCH_SPLIT(dev_priv))
3002 ibx_irq_reset(dev); 2998 ibx_irq_reset(dev_priv);
3003} 2999}
3004 3000
3005void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, 3001void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
@@ -3222,7 +3218,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
3222 3218
3223 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs); 3219 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3224 3220
3225 if (INTEL_INFO(dev)->gen >= 6) { 3221 if (INTEL_GEN(dev_priv) >= 6) {
3226 /* 3222 /*
3227 * RPS interrupts will get enabled/disabled on demand when RPS 3223 * RPS interrupts will get enabled/disabled on demand when RPS
3228 * itself is enabled/disabled. 3224 * itself is enabled/disabled.
@@ -3242,7 +3238,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
3242 struct drm_i915_private *dev_priv = to_i915(dev); 3238 struct drm_i915_private *dev_priv = to_i915(dev);
3243 u32 display_mask, extra_mask; 3239 u32 display_mask, extra_mask;
3244 3240
3245 if (INTEL_INFO(dev)->gen >= 7) { 3241 if (INTEL_GEN(dev_priv) >= 7) {
3246 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | 3242 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3247 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB | 3243 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3248 DE_PLANEB_FLIP_DONE_IVB | 3244 DE_PLANEB_FLIP_DONE_IVB |
@@ -3466,7 +3462,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
3466 I915_WRITE(VLV_MASTER_IER, 0); 3462 I915_WRITE(VLV_MASTER_IER, 0);
3467 POSTING_READ(VLV_MASTER_IER); 3463 POSTING_READ(VLV_MASTER_IER);
3468 3464
3469 gen5_gt_irq_reset(dev); 3465 gen5_gt_irq_reset(dev_priv);
3470 3466
3471 I915_WRITE(HWSTAM, 0xffffffff); 3467 I915_WRITE(HWSTAM, 0xffffffff);
3472 3468
@@ -3678,7 +3674,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
3678 struct drm_i915_private *dev_priv = to_i915(dev); 3674 struct drm_i915_private *dev_priv = to_i915(dev);
3679 int pipe; 3675 int pipe;
3680 3676
3681 if (I915_HAS_HOTPLUG(dev)) { 3677 if (I915_HAS_HOTPLUG(dev_priv)) {
3682 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3678 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3683 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3679 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3684 } 3680 }
@@ -3712,7 +3708,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
3712 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | 3708 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3713 I915_USER_INTERRUPT; 3709 I915_USER_INTERRUPT;
3714 3710
3715 if (I915_HAS_HOTPLUG(dev)) { 3711 if (I915_HAS_HOTPLUG(dev_priv)) {
3716 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3712 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3717 POSTING_READ(PORT_HOTPLUG_EN); 3713 POSTING_READ(PORT_HOTPLUG_EN);
3718 3714
@@ -3880,7 +3876,7 @@ static void i915_irq_uninstall(struct drm_device * dev)
3880 struct drm_i915_private *dev_priv = to_i915(dev); 3876 struct drm_i915_private *dev_priv = to_i915(dev);
3881 int pipe; 3877 int pipe;
3882 3878
3883 if (I915_HAS_HOTPLUG(dev)) { 3879 if (I915_HAS_HOTPLUG(dev_priv)) {
3884 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); 3880 i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3885 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT)); 3881 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3886 } 3882 }
@@ -4145,7 +4141,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
4145 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); 4141 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4146 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); 4142 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4147 4143
4148 if (HAS_GUC_SCHED(dev)) 4144 if (HAS_GUC_SCHED(dev_priv))
4149 dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT; 4145 dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;
4150 4146
4151 /* Let's track the enabled rps events */ 4147 /* Let's track the enabled rps events */
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 629e4334719c..d46ffe7086bc 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -39,7 +39,7 @@ struct i915_params i915 __read_mostly = {
39 .enable_hangcheck = true, 39 .enable_hangcheck = true,
40 .enable_ppgtt = -1, 40 .enable_ppgtt = -1,
41 .enable_psr = -1, 41 .enable_psr = -1,
42 .preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT), 42 .alpha_support = IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT),
43 .disable_power_well = -1, 43 .disable_power_well = -1,
44 .enable_ips = 1, 44 .enable_ips = 1,
45 .fastboot = 0, 45 .fastboot = 0,
@@ -145,9 +145,10 @@ MODULE_PARM_DESC(enable_psr, "Enable PSR "
145 "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) " 145 "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
146 "Default: -1 (use per-chip default)"); 146 "Default: -1 (use per-chip default)");
147 147
148module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, int, 0400); 148module_param_named_unsafe(alpha_support, i915.alpha_support, int, 0400);
149MODULE_PARM_DESC(preliminary_hw_support, 149MODULE_PARM_DESC(alpha_support,
150 "Enable preliminary hardware support."); 150 "Enable alpha quality driver support for latest hardware. "
151 "See also CONFIG_DRM_I915_ALPHA_SUPPORT.");
151 152
152module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400); 153module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400);
153MODULE_PARM_DESC(disable_power_well, 154MODULE_PARM_DESC(disable_power_well,
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 94efc899c1ef..817ad959941e 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -40,7 +40,7 @@ struct i915_params {
40 int enable_ppgtt; 40 int enable_ppgtt;
41 int enable_execlists; 41 int enable_execlists;
42 int enable_psr; 42 int enable_psr;
43 unsigned int preliminary_hw_support; 43 unsigned int alpha_support;
44 int disable_power_well; 44 int disable_power_well;
45 int enable_ips; 45 int enable_ips;
46 int invert_brightness; 46 int invert_brightness;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 2a419500b81a..fce8e198bc76 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -363,6 +363,7 @@ static const struct intel_device_info intel_broxton_info = {
363 .has_hw_contexts = 1, 363 .has_hw_contexts = 1,
364 .has_logical_ring_contexts = 1, 364 .has_logical_ring_contexts = 1,
365 .has_guc = 1, 365 .has_guc = 1,
366 .has_decoupled_mmio = 1,
366 .ddb_size = 512, 367 .ddb_size = 512,
367 GEN_DEFAULT_PIPEOFFSETS, 368 GEN_DEFAULT_PIPEOFFSETS,
368 IVB_CURSOR_OFFSETS, 369 IVB_CURSOR_OFFSETS,
@@ -439,9 +440,10 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
439 struct intel_device_info *intel_info = 440 struct intel_device_info *intel_info =
440 (struct intel_device_info *) ent->driver_data; 441 (struct intel_device_info *) ent->driver_data;
441 442
442 if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) { 443 if (IS_ALPHA_SUPPORT(intel_info) && !i915.alpha_support) {
443 DRM_INFO("This hardware requires preliminary hardware support.\n" 444 DRM_INFO("The driver support for your hardware in this kernel version is alpha quality\n"
444 "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n"); 445 "See CONFIG_DRM_I915_ALPHA_SUPPORT or i915.alpha_support module parameter\n"
446 "to enable support in this kernel version, or check for kernel updates.\n");
445 return -ENODEV; 447 return -ENODEV;
446 } 448 }
447 449
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 3361d7ffc63e..c70c07a7b586 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -7342,6 +7342,13 @@ enum {
7342#define SKL_FUSE_PG1_DIST_STATUS (1<<26) 7342#define SKL_FUSE_PG1_DIST_STATUS (1<<26)
7343#define SKL_FUSE_PG2_DIST_STATUS (1<<25) 7343#define SKL_FUSE_PG2_DIST_STATUS (1<<25)
7344 7344
7345/* Decoupled MMIO register pair for kernel driver */
7346#define GEN9_DECOUPLED_REG0_DW0 _MMIO(0xF00)
7347#define GEN9_DECOUPLED_REG0_DW1 _MMIO(0xF04)
7348#define GEN9_DECOUPLED_DW1_GO (1<<31)
7349#define GEN9_DECOUPLED_PD_SHIFT 28
7350#define GEN9_DECOUPLED_OP_SHIFT 24
7351
7345/* Per-pipe DDI Function Control */ 7352/* Per-pipe DDI Function Control */
7346#define _TRANS_DDI_FUNC_CTL_A 0x60400 7353#define _TRANS_DDI_FUNC_CTL_A 0x60400
7347#define _TRANS_DDI_FUNC_CTL_B 0x61400 7354#define _TRANS_DDI_FUNC_CTL_B 0x61400
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 344cbf39cfa9..b0e1e7ca75da 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -29,12 +29,10 @@
29#include "intel_drv.h" 29#include "intel_drv.h"
30#include "i915_reg.h" 30#include "i915_reg.h"
31 31
32static void i915_save_display(struct drm_device *dev) 32static void i915_save_display(struct drm_i915_private *dev_priv)
33{ 33{
34 struct drm_i915_private *dev_priv = to_i915(dev);
35
36 /* Display arbitration control */ 34 /* Display arbitration control */
37 if (INTEL_INFO(dev)->gen <= 4) 35 if (INTEL_GEN(dev_priv) <= 4)
38 dev_priv->regfile.saveDSPARB = I915_READ(DSPARB); 36 dev_priv->regfile.saveDSPARB = I915_READ(DSPARB);
39 37
40 /* save FBC interval */ 38 /* save FBC interval */
@@ -42,12 +40,10 @@ static void i915_save_display(struct drm_device *dev)
42 dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL); 40 dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
43} 41}
44 42
45static void i915_restore_display(struct drm_device *dev) 43static void i915_restore_display(struct drm_i915_private *dev_priv)
46{ 44{
47 struct drm_i915_private *dev_priv = to_i915(dev);
48
49 /* Display arbitration */ 45 /* Display arbitration */
50 if (INTEL_INFO(dev)->gen <= 4) 46 if (INTEL_GEN(dev_priv) <= 4)
51 I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB); 47 I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB);
52 48
53 /* only restore FBC info on the platform that supports FBC*/ 49 /* only restore FBC info on the platform that supports FBC*/
@@ -57,7 +53,7 @@ static void i915_restore_display(struct drm_device *dev)
57 if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv)) 53 if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv))
58 I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL); 54 I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL);
59 55
60 i915_redisable_vga(dev); 56 i915_redisable_vga(dev_priv);
61} 57}
62 58
63int i915_save_state(struct drm_device *dev) 59int i915_save_state(struct drm_device *dev)
@@ -68,14 +64,14 @@ int i915_save_state(struct drm_device *dev)
68 64
69 mutex_lock(&dev->struct_mutex); 65 mutex_lock(&dev->struct_mutex);
70 66
71 i915_save_display(dev); 67 i915_save_display(dev_priv);
72 68
73 if (IS_GEN4(dev_priv)) 69 if (IS_GEN4(dev_priv))
74 pci_read_config_word(pdev, GCDGMBUS, 70 pci_read_config_word(pdev, GCDGMBUS,
75 &dev_priv->regfile.saveGCDGMBUS); 71 &dev_priv->regfile.saveGCDGMBUS);
76 72
77 /* Cache mode state */ 73 /* Cache mode state */
78 if (INTEL_INFO(dev)->gen < 7) 74 if (INTEL_GEN(dev_priv) < 7)
79 dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); 75 dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
80 76
81 /* Memory Arbitration state */ 77 /* Memory Arbitration state */
@@ -114,15 +110,15 @@ int i915_restore_state(struct drm_device *dev)
114 110
115 mutex_lock(&dev->struct_mutex); 111 mutex_lock(&dev->struct_mutex);
116 112
117 i915_gem_restore_fences(dev); 113 i915_gem_restore_fences(dev_priv);
118 114
119 if (IS_GEN4(dev_priv)) 115 if (IS_GEN4(dev_priv))
120 pci_write_config_word(pdev, GCDGMBUS, 116 pci_write_config_word(pdev, GCDGMBUS,
121 dev_priv->regfile.saveGCDGMBUS); 117 dev_priv->regfile.saveGCDGMBUS);
122 i915_restore_display(dev); 118 i915_restore_display(dev_priv);
123 119
124 /* Cache mode state */ 120 /* Cache mode state */
125 if (INTEL_INFO(dev)->gen < 7) 121 if (INTEL_GEN(dev_priv) < 7)
126 I915_WRITE(CACHE_MODE_0, dev_priv->regfile.saveCACHE_MODE_0 | 122 I915_WRITE(CACHE_MODE_0, dev_priv->regfile.saveCACHE_MODE_0 |
127 0xffff0000); 123 0xffff0000);
128 124
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index 95f2f12e0917..147420ccf49c 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -116,11 +116,14 @@ static void i915_sw_fence_await(struct i915_sw_fence *fence)
116 WARN_ON(atomic_inc_return(&fence->pending) <= 1); 116 WARN_ON(atomic_inc_return(&fence->pending) <= 1);
117} 117}
118 118
119void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn) 119void __i915_sw_fence_init(struct i915_sw_fence *fence,
120 i915_sw_fence_notify_t fn,
121 const char *name,
122 struct lock_class_key *key)
120{ 123{
121 BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK); 124 BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
122 125
123 init_waitqueue_head(&fence->wait); 126 __init_waitqueue_head(&fence->wait, name, key);
124 kref_init(&fence->kref); 127 kref_init(&fence->kref);
125 atomic_set(&fence->pending, 1); 128 atomic_set(&fence->pending, 1);
126 fence->flags = (unsigned long)fn; 129 fence->flags = (unsigned long)fn;
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
index 707dfc4f0da5..7508d23f823b 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -40,7 +40,22 @@ typedef int (*i915_sw_fence_notify_t)(struct i915_sw_fence *,
40 enum i915_sw_fence_notify state); 40 enum i915_sw_fence_notify state);
41#define __i915_sw_fence_call __aligned(4) 41#define __i915_sw_fence_call __aligned(4)
42 42
43void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn); 43void __i915_sw_fence_init(struct i915_sw_fence *fence,
44 i915_sw_fence_notify_t fn,
45 const char *name,
46 struct lock_class_key *key);
47#ifdef CONFIG_LOCKDEP
48#define i915_sw_fence_init(fence, fn) \
49do { \
50 static struct lock_class_key __key; \
51 \
52 __i915_sw_fence_init((fence), (fn), #fence, &__key); \
53} while (0)
54#else
55#define i915_sw_fence_init(fence, fn) \
56 __i915_sw_fence_init((fence), (fn), NULL, NULL)
57#endif
58
44void i915_sw_fence_commit(struct i915_sw_fence *fence); 59void i915_sw_fence_commit(struct i915_sw_fence *fence);
45 60
46int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, 61int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
new file mode 100644
index 000000000000..a792dcb902b5
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -0,0 +1,638 @@
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "i915_vma.h"
26
27#include "i915_drv.h"
28#include "intel_ringbuffer.h"
29#include "intel_frontbuffer.h"
30
31#include <drm/drm_gem.h>
32
33static void
34i915_vma_retire(struct i915_gem_active *active,
35 struct drm_i915_gem_request *rq)
36{
37 const unsigned int idx = rq->engine->id;
38 struct i915_vma *vma =
39 container_of(active, struct i915_vma, last_read[idx]);
40 struct drm_i915_gem_object *obj = vma->obj;
41
42 GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
43
44 i915_vma_clear_active(vma, idx);
45 if (i915_vma_is_active(vma))
46 return;
47
48 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
49 if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
50 WARN_ON(i915_vma_unbind(vma));
51
52 GEM_BUG_ON(!i915_gem_object_is_active(obj));
53 if (--obj->active_count)
54 return;
55
56 /* Bump our place on the bound list to keep it roughly in LRU order
57 * so that we don't steal from recently used but inactive objects
58 * (unless we are forced to ofc!)
59 */
60 if (obj->bind_count)
61 list_move_tail(&obj->global_link, &rq->i915->mm.bound_list);
62
63 obj->mm.dirty = true; /* be paranoid */
64
65 if (i915_gem_object_has_active_reference(obj)) {
66 i915_gem_object_clear_active_reference(obj);
67 i915_gem_object_put(obj);
68 }
69}
70
71static struct i915_vma *
72__i915_vma_create(struct drm_i915_gem_object *obj,
73 struct i915_address_space *vm,
74 const struct i915_ggtt_view *view)
75{
76 struct i915_vma *vma;
77 struct rb_node *rb, **p;
78 int i;
79
80 GEM_BUG_ON(vm->closed);
81
82 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
83 if (vma == NULL)
84 return ERR_PTR(-ENOMEM);
85
86 INIT_LIST_HEAD(&vma->exec_list);
87 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
88 init_request_active(&vma->last_read[i], i915_vma_retire);
89 init_request_active(&vma->last_fence, NULL);
90 list_add(&vma->vm_link, &vm->unbound_list);
91 vma->vm = vm;
92 vma->obj = obj;
93 vma->size = obj->base.size;
94
95 if (view) {
96 vma->ggtt_view = *view;
97 if (view->type == I915_GGTT_VIEW_PARTIAL) {
98 vma->size = view->params.partial.size;
99 vma->size <<= PAGE_SHIFT;
100 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
101 vma->size =
102 intel_rotation_info_size(&view->params.rotated);
103 vma->size <<= PAGE_SHIFT;
104 }
105 }
106
107 if (i915_is_ggtt(vm)) {
108 vma->flags |= I915_VMA_GGTT;
109 list_add(&vma->obj_link, &obj->vma_list);
110 } else {
111 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
112 list_add_tail(&vma->obj_link, &obj->vma_list);
113 }
114
115 rb = NULL;
116 p = &obj->vma_tree.rb_node;
117 while (*p) {
118 struct i915_vma *pos;
119
120 rb = *p;
121 pos = rb_entry(rb, struct i915_vma, obj_node);
122 if (i915_vma_compare(pos, vm, view) < 0)
123 p = &rb->rb_right;
124 else
125 p = &rb->rb_left;
126 }
127 rb_link_node(&vma->obj_node, rb, p);
128 rb_insert_color(&vma->obj_node, &obj->vma_tree);
129
130 return vma;
131}
132
133struct i915_vma *
134i915_vma_create(struct drm_i915_gem_object *obj,
135 struct i915_address_space *vm,
136 const struct i915_ggtt_view *view)
137{
138 lockdep_assert_held(&obj->base.dev->struct_mutex);
139 GEM_BUG_ON(view && !i915_is_ggtt(vm));
140 GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
141
142 return __i915_vma_create(obj, vm, view);
143}
144
145/**
146 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
147 * @vma: VMA to map
148 * @cache_level: mapping cache level
149 * @flags: flags like global or local mapping
150 *
151 * DMA addresses are taken from the scatter-gather table of this object (or of
152 * this VMA in case of non-default GGTT views) and PTE entries set up.
153 * Note that DMA addresses are also the only part of the SG table we care about.
154 */
155int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
156 u32 flags)
157{
158 u32 bind_flags;
159 u32 vma_flags;
160 int ret;
161
162 if (WARN_ON(flags == 0))
163 return -EINVAL;
164
165 bind_flags = 0;
166 if (flags & PIN_GLOBAL)
167 bind_flags |= I915_VMA_GLOBAL_BIND;
168 if (flags & PIN_USER)
169 bind_flags |= I915_VMA_LOCAL_BIND;
170
171 vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
172 if (flags & PIN_UPDATE)
173 bind_flags |= vma_flags;
174 else
175 bind_flags &= ~vma_flags;
176 if (bind_flags == 0)
177 return 0;
178
179 if (vma_flags == 0 && vma->vm->allocate_va_range) {
180 trace_i915_va_alloc(vma);
181 ret = vma->vm->allocate_va_range(vma->vm,
182 vma->node.start,
183 vma->node.size);
184 if (ret)
185 return ret;
186 }
187
188 ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
189 if (ret)
190 return ret;
191
192 vma->flags |= bind_flags;
193 return 0;
194}
195
196void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
197{
198 void __iomem *ptr;
199
200 /* Access through the GTT requires the device to be awake. */
201 assert_rpm_wakelock_held(to_i915(vma->vm->dev));
202
203 lockdep_assert_held(&vma->vm->dev->struct_mutex);
204 if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
205 return IO_ERR_PTR(-ENODEV);
206
207 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
208 GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
209
210 ptr = vma->iomap;
211 if (ptr == NULL) {
212 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
213 vma->node.start,
214 vma->node.size);
215 if (ptr == NULL)
216 return IO_ERR_PTR(-ENOMEM);
217
218 vma->iomap = ptr;
219 }
220
221 __i915_vma_pin(vma);
222 return ptr;
223}
224
225void i915_vma_unpin_and_release(struct i915_vma **p_vma)
226{
227 struct i915_vma *vma;
228 struct drm_i915_gem_object *obj;
229
230 vma = fetch_and_zero(p_vma);
231 if (!vma)
232 return;
233
234 obj = vma->obj;
235
236 i915_vma_unpin(vma);
237 i915_vma_close(vma);
238
239 __i915_gem_object_release_unless_active(obj);
240}
241
242bool
243i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
244{
245 if (!drm_mm_node_allocated(&vma->node))
246 return false;
247
248 if (vma->node.size < size)
249 return true;
250
251 if (alignment && vma->node.start & (alignment - 1))
252 return true;
253
254 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
255 return true;
256
257 if (flags & PIN_OFFSET_BIAS &&
258 vma->node.start < (flags & PIN_OFFSET_MASK))
259 return true;
260
261 if (flags & PIN_OFFSET_FIXED &&
262 vma->node.start != (flags & PIN_OFFSET_MASK))
263 return true;
264
265 return false;
266}
267
268void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
269{
270 struct drm_i915_gem_object *obj = vma->obj;
271 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
272 bool mappable, fenceable;
273 u32 fence_size, fence_alignment;
274
275 fence_size = i915_gem_get_ggtt_size(dev_priv,
276 vma->size,
277 i915_gem_object_get_tiling(obj));
278 fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
279 vma->size,
280 i915_gem_object_get_tiling(obj),
281 true);
282
283 fenceable = (vma->node.size == fence_size &&
284 (vma->node.start & (fence_alignment - 1)) == 0);
285
286 mappable = (vma->node.start + fence_size <=
287 dev_priv->ggtt.mappable_end);
288
289 /*
290 * Explicitly disable for rotated VMA since the display does not
291 * need the fence and the VMA is not accessible to other users.
292 */
293 if (mappable && fenceable &&
294 vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED)
295 vma->flags |= I915_VMA_CAN_FENCE;
296 else
297 vma->flags &= ~I915_VMA_CAN_FENCE;
298}
299
300bool i915_gem_valid_gtt_space(struct i915_vma *vma,
301 unsigned long cache_level)
302{
303 struct drm_mm_node *gtt_space = &vma->node;
304 struct drm_mm_node *other;
305
306 /*
307 * On some machines we have to be careful when putting differing types
308 * of snoopable memory together to avoid the prefetcher crossing memory
309 * domains and dying. During vm initialisation, we decide whether or not
310 * these constraints apply and set the drm_mm.color_adjust
311 * appropriately.
312 */
313 if (vma->vm->mm.color_adjust == NULL)
314 return true;
315
316 if (!drm_mm_node_allocated(gtt_space))
317 return true;
318
319 if (list_empty(&gtt_space->node_list))
320 return true;
321
322 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
323 if (other->allocated && !other->hole_follows && other->color != cache_level)
324 return false;
325
326 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
327 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
328 return false;
329
330 return true;
331}
332
333/**
334 * i915_vma_insert - finds a slot for the vma in its address space
335 * @vma: the vma
336 * @size: requested size in bytes (can be larger than the VMA)
337 * @alignment: required alignment
338 * @flags: mask of PIN_* flags to use
339 *
340 * First we try to allocate some free space that meets the requirements for
341 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
342 * preferrably the oldest idle entry to make room for the new VMA.
343 *
344 * Returns:
345 * 0 on success, negative error code otherwise.
346 */
347static int
348i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
349{
350 struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
351 struct drm_i915_gem_object *obj = vma->obj;
352 u64 start, end;
353 int ret;
354
355 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
356 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
357
358 size = max(size, vma->size);
359 if (flags & PIN_MAPPABLE)
360 size = i915_gem_get_ggtt_size(dev_priv, size,
361 i915_gem_object_get_tiling(obj));
362
363 alignment = max(max(alignment, vma->display_alignment),
364 i915_gem_get_ggtt_alignment(dev_priv, size,
365 i915_gem_object_get_tiling(obj),
366 flags & PIN_MAPPABLE));
367
368 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
369
370 end = vma->vm->total;
371 if (flags & PIN_MAPPABLE)
372 end = min_t(u64, end, dev_priv->ggtt.mappable_end);
373 if (flags & PIN_ZONE_4G)
374 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
375
376 /* If binding the object/GGTT view requires more space than the entire
377 * aperture has, reject it early before evicting everything in a vain
378 * attempt to find space.
379 */
380 if (size > end) {
381 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
382 size, obj->base.size,
383 flags & PIN_MAPPABLE ? "mappable" : "total",
384 end);
385 return -E2BIG;
386 }
387
388 ret = i915_gem_object_pin_pages(obj);
389 if (ret)
390 return ret;
391
392 if (flags & PIN_OFFSET_FIXED) {
393 u64 offset = flags & PIN_OFFSET_MASK;
394 if (offset & (alignment - 1) || offset > end - size) {
395 ret = -EINVAL;
396 goto err_unpin;
397 }
398
399 vma->node.start = offset;
400 vma->node.size = size;
401 vma->node.color = obj->cache_level;
402 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
403 if (ret) {
404 ret = i915_gem_evict_for_vma(vma);
405 if (ret == 0)
406 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
407 if (ret)
408 goto err_unpin;
409 }
410 } else {
411 u32 search_flag, alloc_flag;
412
413 if (flags & PIN_HIGH) {
414 search_flag = DRM_MM_SEARCH_BELOW;
415 alloc_flag = DRM_MM_CREATE_TOP;
416 } else {
417 search_flag = DRM_MM_SEARCH_DEFAULT;
418 alloc_flag = DRM_MM_CREATE_DEFAULT;
419 }
420
421 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
422 * so we know that we always have a minimum alignment of 4096.
423 * The drm_mm range manager is optimised to return results
424 * with zero alignment, so where possible use the optimal
425 * path.
426 */
427 if (alignment <= 4096)
428 alignment = 0;
429
430search_free:
431 ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
432 &vma->node,
433 size, alignment,
434 obj->cache_level,
435 start, end,
436 search_flag,
437 alloc_flag);
438 if (ret) {
439 ret = i915_gem_evict_something(vma->vm, size, alignment,
440 obj->cache_level,
441 start, end,
442 flags);
443 if (ret == 0)
444 goto search_free;
445
446 goto err_unpin;
447 }
448
449 GEM_BUG_ON(vma->node.start < start);
450 GEM_BUG_ON(vma->node.start + vma->node.size > end);
451 }
452 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
453
454 list_move_tail(&obj->global_link, &dev_priv->mm.bound_list);
455 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
456 obj->bind_count++;
457 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
458
459 return 0;
460
461err_unpin:
462 i915_gem_object_unpin_pages(obj);
463 return ret;
464}
465
466int __i915_vma_do_pin(struct i915_vma *vma,
467 u64 size, u64 alignment, u64 flags)
468{
469 unsigned int bound = vma->flags;
470 int ret;
471
472 lockdep_assert_held(&vma->vm->dev->struct_mutex);
473 GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
474 GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
475
476 if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
477 ret = -EBUSY;
478 goto err;
479 }
480
481 if ((bound & I915_VMA_BIND_MASK) == 0) {
482 ret = i915_vma_insert(vma, size, alignment, flags);
483 if (ret)
484 goto err;
485 }
486
487 ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
488 if (ret)
489 goto err;
490
491 if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
492 __i915_vma_set_map_and_fenceable(vma);
493
494 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
495 return 0;
496
497err:
498 __i915_vma_unpin(vma);
499 return ret;
500}
501
502void i915_vma_destroy(struct i915_vma *vma)
503{
504 GEM_BUG_ON(vma->node.allocated);
505 GEM_BUG_ON(i915_vma_is_active(vma));
506 GEM_BUG_ON(!i915_vma_is_closed(vma));
507 GEM_BUG_ON(vma->fence);
508
509 list_del(&vma->vm_link);
510 if (!i915_vma_is_ggtt(vma))
511 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
512
513 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
514}
515
516void i915_vma_close(struct i915_vma *vma)
517{
518 GEM_BUG_ON(i915_vma_is_closed(vma));
519 vma->flags |= I915_VMA_CLOSED;
520
521 list_del(&vma->obj_link);
522 rb_erase(&vma->obj_node, &vma->obj->vma_tree);
523
524 if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
525 WARN_ON(i915_vma_unbind(vma));
526}
527
528static void __i915_vma_iounmap(struct i915_vma *vma)
529{
530 GEM_BUG_ON(i915_vma_is_pinned(vma));
531
532 if (vma->iomap == NULL)
533 return;
534
535 io_mapping_unmap(vma->iomap);
536 vma->iomap = NULL;
537}
538
539int i915_vma_unbind(struct i915_vma *vma)
540{
541 struct drm_i915_gem_object *obj = vma->obj;
542 unsigned long active;
543 int ret;
544
545 lockdep_assert_held(&obj->base.dev->struct_mutex);
546
547 /* First wait upon any activity as retiring the request may
548 * have side-effects such as unpinning or even unbinding this vma.
549 */
550 active = i915_vma_get_active(vma);
551 if (active) {
552 int idx;
553
554 /* When a closed VMA is retired, it is unbound - eek.
555 * In order to prevent it from being recursively closed,
556 * take a pin on the vma so that the second unbind is
557 * aborted.
558 *
559 * Even more scary is that the retire callback may free
560 * the object (last active vma). To prevent the explosion
561 * we defer the actual object free to a worker that can
562 * only proceed once it acquires the struct_mutex (which
563 * we currently hold, therefore it cannot free this object
564 * before we are finished).
565 */
566 __i915_vma_pin(vma);
567
568 for_each_active(active, idx) {
569 ret = i915_gem_active_retire(&vma->last_read[idx],
570 &vma->vm->dev->struct_mutex);
571 if (ret)
572 break;
573 }
574
575 __i915_vma_unpin(vma);
576 if (ret)
577 return ret;
578
579 GEM_BUG_ON(i915_vma_is_active(vma));
580 }
581
582 if (i915_vma_is_pinned(vma))
583 return -EBUSY;
584
585 if (!drm_mm_node_allocated(&vma->node))
586 goto destroy;
587
588 GEM_BUG_ON(obj->bind_count == 0);
589 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
590
591 if (i915_vma_is_map_and_fenceable(vma)) {
592 /* release the fence reg _after_ flushing */
593 ret = i915_vma_put_fence(vma);
594 if (ret)
595 return ret;
596
597 /* Force a pagefault for domain tracking on next user access */
598 i915_gem_release_mmap(obj);
599
600 __i915_vma_iounmap(vma);
601 vma->flags &= ~I915_VMA_CAN_FENCE;
602 }
603
604 if (likely(!vma->vm->closed)) {
605 trace_i915_vma_unbind(vma);
606 vma->vm->unbind_vma(vma);
607 }
608 vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
609
610 drm_mm_remove_node(&vma->node);
611 list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
612
613 if (vma->pages != obj->mm.pages) {
614 GEM_BUG_ON(!vma->pages);
615 sg_free_table(vma->pages);
616 kfree(vma->pages);
617 }
618 vma->pages = NULL;
619
620 /* Since the unbound list is global, only move to that list if
621 * no more VMAs exist. */
622 if (--obj->bind_count == 0)
623 list_move_tail(&obj->global_link,
624 &to_i915(obj->base.dev)->mm.unbound_list);
625
626 /* And finally now the object is completely decoupled from this vma,
627 * we can drop its hold on the backing storage and allow it to be
628 * reaped by the shrinker.
629 */
630 i915_gem_object_unpin_pages(obj);
631
632destroy:
633 if (unlikely(i915_vma_is_closed(vma)))
634 i915_vma_destroy(vma);
635
636 return 0;
637}
638
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
new file mode 100644
index 000000000000..85446f0b0b3f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -0,0 +1,341 @@
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#ifndef __I915_VMA_H__
26#define __I915_VMA_H__
27
28#include <linux/io-mapping.h>
29
30#include <drm/drm_mm.h>
31
32#include "i915_gem_gtt.h"
33#include "i915_gem_fence_reg.h"
34#include "i915_gem_object.h"
35#include "i915_gem_request.h"
36
37
38enum i915_cache_level;
39
40/**
41 * A VMA represents a GEM BO that is bound into an address space. Therefore, a
42 * VMA's presence cannot be guaranteed before binding, or after unbinding the
43 * object into/from the address space.
44 *
45 * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
46 * will always be <= an objects lifetime. So object refcounting should cover us.
47 */
48struct i915_vma {
49 struct drm_mm_node node;
50 struct drm_i915_gem_object *obj;
51 struct i915_address_space *vm;
52 struct drm_i915_fence_reg *fence;
53 struct sg_table *pages;
54 void __iomem *iomap;
55 u64 size;
56 u64 display_alignment;
57
58 unsigned int flags;
59 /**
60 * How many users have pinned this object in GTT space. The following
61 * users can each hold at most one reference: pwrite/pread, execbuffer
62 * (objects are not allowed multiple times for the same batchbuffer),
63 * and the framebuffer code. When switching/pageflipping, the
64 * framebuffer code has at most two buffers pinned per crtc.
65 *
66 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
67 * bits with absolutely no headroom. So use 4 bits.
68 */
69#define I915_VMA_PIN_MASK 0xf
70#define I915_VMA_PIN_OVERFLOW BIT(5)
71
72 /** Flags and address space this VMA is bound to */
73#define I915_VMA_GLOBAL_BIND BIT(6)
74#define I915_VMA_LOCAL_BIND BIT(7)
75#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
76
77#define I915_VMA_GGTT BIT(8)
78#define I915_VMA_CAN_FENCE BIT(9)
79#define I915_VMA_CLOSED BIT(10)
80
81 unsigned int active;
82 struct i915_gem_active last_read[I915_NUM_ENGINES];
83 struct i915_gem_active last_fence;
84
85 /**
86 * Support different GGTT views into the same object.
87 * This means there can be multiple VMA mappings per object and per VM.
88 * i915_ggtt_view_type is used to distinguish between those entries.
89 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
90 * assumed in GEM functions which take no ggtt view parameter.
91 */
92 struct i915_ggtt_view ggtt_view;
93
94 /** This object's place on the active/inactive lists */
95 struct list_head vm_link;
96
97 struct list_head obj_link; /* Link in the object's VMA list */
98 struct rb_node obj_node;
99
100 /** This vma's place in the batchbuffer or on the eviction list */
101 struct list_head exec_list;
102
103 /**
104 * Used for performing relocations during execbuffer insertion.
105 */
106 struct hlist_node exec_node;
107 unsigned long exec_handle;
108 struct drm_i915_gem_exec_object2 *exec_entry;
109};
110
111struct i915_vma *
112i915_vma_create(struct drm_i915_gem_object *obj,
113 struct i915_address_space *vm,
114 const struct i915_ggtt_view *view);
115
116void i915_vma_unpin_and_release(struct i915_vma **p_vma);
117
118static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
119{
120 return vma->flags & I915_VMA_GGTT;
121}
122
123static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
124{
125 return vma->flags & I915_VMA_CAN_FENCE;
126}
127
128static inline bool i915_vma_is_closed(const struct i915_vma *vma)
129{
130 return vma->flags & I915_VMA_CLOSED;
131}
132
133static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
134{
135 return vma->active;
136}
137
138static inline bool i915_vma_is_active(const struct i915_vma *vma)
139{
140 return i915_vma_get_active(vma);
141}
142
143static inline void i915_vma_set_active(struct i915_vma *vma,
144 unsigned int engine)
145{
146 vma->active |= BIT(engine);
147}
148
149static inline void i915_vma_clear_active(struct i915_vma *vma,
150 unsigned int engine)
151{
152 vma->active &= ~BIT(engine);
153}
154
155static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
156 unsigned int engine)
157{
158 return vma->active & BIT(engine);
159}
160
161static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
162{
163 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
164 GEM_BUG_ON(!vma->node.allocated);
165 GEM_BUG_ON(upper_32_bits(vma->node.start));
166 GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
167 return lower_32_bits(vma->node.start);
168}
169
170static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
171{
172 i915_gem_object_get(vma->obj);
173 return vma;
174}
175
176static inline void i915_vma_put(struct i915_vma *vma)
177{
178 i915_gem_object_put(vma->obj);
179}
180
181static inline long
182i915_vma_compare(struct i915_vma *vma,
183 struct i915_address_space *vm,
184 const struct i915_ggtt_view *view)
185{
186 GEM_BUG_ON(view && !i915_is_ggtt(vm));
187
188 if (vma->vm != vm)
189 return vma->vm - vm;
190
191 if (!view)
192 return vma->ggtt_view.type;
193
194 if (vma->ggtt_view.type != view->type)
195 return vma->ggtt_view.type - view->type;
196
197 return memcmp(&vma->ggtt_view.params,
198 &view->params,
199 sizeof(view->params));
200}
201
202int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
203 u32 flags);
204bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
205bool
206i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
207void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
208int __must_check i915_vma_unbind(struct i915_vma *vma);
209void i915_vma_close(struct i915_vma *vma);
210void i915_vma_destroy(struct i915_vma *vma);
211
212int __i915_vma_do_pin(struct i915_vma *vma,
213 u64 size, u64 alignment, u64 flags);
214static inline int __must_check
215i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
216{
217 BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
218 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
219 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
220
221 /* Pin early to prevent the shrinker/eviction logic from destroying
222 * our vma as we insert and bind.
223 */
224 if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
225 return 0;
226
227 return __i915_vma_do_pin(vma, size, alignment, flags);
228}
229
230static inline int i915_vma_pin_count(const struct i915_vma *vma)
231{
232 return vma->flags & I915_VMA_PIN_MASK;
233}
234
235static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
236{
237 return i915_vma_pin_count(vma);
238}
239
240static inline void __i915_vma_pin(struct i915_vma *vma)
241{
242 vma->flags++;
243 GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
244}
245
246static inline void __i915_vma_unpin(struct i915_vma *vma)
247{
248 GEM_BUG_ON(!i915_vma_is_pinned(vma));
249 vma->flags--;
250}
251
252static inline void i915_vma_unpin(struct i915_vma *vma)
253{
254 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
255 __i915_vma_unpin(vma);
256}
257
258/**
259 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
260 * @vma: VMA to iomap
261 *
262 * The passed in VMA has to be pinned in the global GTT mappable region.
263 * An extra pinning of the VMA is acquired for the return iomapping,
264 * the caller must call i915_vma_unpin_iomap to relinquish the pinning
265 * after the iomapping is no longer required.
266 *
267 * Callers must hold the struct_mutex.
268 *
269 * Returns a valid iomapped pointer or ERR_PTR.
270 */
271void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
272#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
273
274/**
275 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
276 * @vma: VMA to unpin
277 *
278 * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
279 *
280 * Callers must hold the struct_mutex. This function is only valid to be
281 * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
282 */
283static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
284{
285 lockdep_assert_held(&vma->vm->dev->struct_mutex);
286 GEM_BUG_ON(vma->iomap == NULL);
287 i915_vma_unpin(vma);
288}
289
290static inline struct page *i915_vma_first_page(struct i915_vma *vma)
291{
292 GEM_BUG_ON(!vma->pages);
293 return sg_page(vma->pages->sgl);
294}
295
296/**
297 * i915_vma_pin_fence - pin fencing state
298 * @vma: vma to pin fencing for
299 *
300 * This pins the fencing state (whether tiled or untiled) to make sure the
301 * vma (and its object) is ready to be used as a scanout target. Fencing
302 * status must be synchronize first by calling i915_vma_get_fence():
303 *
304 * The resulting fence pin reference must be released again with
305 * i915_vma_unpin_fence().
306 *
307 * Returns:
308 *
309 * True if the vma has a fence, false otherwise.
310 */
311static inline bool
312i915_vma_pin_fence(struct i915_vma *vma)
313{
314 lockdep_assert_held(&vma->vm->dev->struct_mutex);
315 if (vma->fence) {
316 vma->fence->pin_count++;
317 return true;
318 } else
319 return false;
320}
321
322/**
323 * i915_vma_unpin_fence - unpin fencing state
324 * @vma: vma to unpin fencing for
325 *
326 * This releases the fence pin reference acquired through
327 * i915_vma_pin_fence. It will handle both objects with and without an
328 * attached fence correctly, callers do not need to distinguish this.
329 */
330static inline void
331i915_vma_unpin_fence(struct i915_vma *vma)
332{
333 lockdep_assert_held(&vma->vm->dev->struct_mutex);
334 if (vma->fence) {
335 GEM_BUG_ON(vma->fence->pin_count <= 0);
336 vma->fence->pin_count--;
337 }
338}
339
340#endif
341
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index 984a6b75c118..ff821649486e 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -106,6 +106,7 @@ intel_plane_destroy_state(struct drm_plane *plane,
106static int intel_plane_atomic_check(struct drm_plane *plane, 106static int intel_plane_atomic_check(struct drm_plane *plane,
107 struct drm_plane_state *state) 107 struct drm_plane_state *state)
108{ 108{
109 struct drm_i915_private *dev_priv = to_i915(plane->dev);
109 struct drm_crtc *crtc = state->crtc; 110 struct drm_crtc *crtc = state->crtc;
110 struct intel_crtc *intel_crtc; 111 struct intel_crtc *intel_crtc;
111 struct intel_crtc_state *crtc_state; 112 struct intel_crtc_state *crtc_state;
@@ -167,6 +168,14 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
167 } 168 }
168 } 169 }
169 170
171 /* CHV ignores the mirror bit when the rotate bit is set :( */
172 if (IS_CHERRYVIEW(dev_priv) &&
173 state->rotation & DRM_ROTATE_180 &&
174 state->rotation & DRM_REFLECT_X) {
175 DRM_DEBUG_KMS("Cannot rotate and reflect at the same time\n");
176 return -EINVAL;
177 }
178
170 intel_state->base.visible = false; 179 intel_state->base.visible = false;
171 ret = intel_plane->check_plane(plane, crtc_state, intel_state); 180 ret = intel_plane->check_plane(plane, crtc_state, intel_state);
172 if (ret) 181 if (ret)
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 813fd74d9c8d..1c509f7410f5 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -574,23 +574,26 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
574/** 574/**
575 * intel_audio_codec_enable - Enable the audio codec for HD audio 575 * intel_audio_codec_enable - Enable the audio codec for HD audio
576 * @intel_encoder: encoder on which to enable audio 576 * @intel_encoder: encoder on which to enable audio
577 * @crtc_state: pointer to the current crtc state.
578 * @conn_state: pointer to the current connector state.
577 * 579 *
578 * The enable sequences may only be performed after enabling the transcoder and 580 * The enable sequences may only be performed after enabling the transcoder and
579 * port, and after completed link training. 581 * port, and after completed link training.
580 */ 582 */
581void intel_audio_codec_enable(struct intel_encoder *intel_encoder) 583void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
584 const struct intel_crtc_state *crtc_state,
585 const struct drm_connector_state *conn_state)
582{ 586{
583 struct drm_encoder *encoder = &intel_encoder->base; 587 struct drm_encoder *encoder = &intel_encoder->base;
584 struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); 588 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
585 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
586 struct drm_connector *connector; 589 struct drm_connector *connector;
587 struct drm_i915_private *dev_priv = to_i915(encoder->dev); 590 struct drm_i915_private *dev_priv = to_i915(encoder->dev);
588 struct i915_audio_component *acomp = dev_priv->audio_component; 591 struct i915_audio_component *acomp = dev_priv->audio_component;
589 enum port port = intel_encoder->port; 592 enum port port = intel_encoder->port;
590 enum pipe pipe = crtc->pipe; 593 enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
591 594
592 connector = drm_select_eld(encoder); 595 connector = conn_state->connector;
593 if (!connector) 596 if (!connector || !connector->eld[0])
594 return; 597 return;
595 598
596 DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n", 599 DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
@@ -601,7 +604,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
601 604
602 /* ELD Conn_Type */ 605 /* ELD Conn_Type */
603 connector->eld[5] &= ~(3 << 2); 606 connector->eld[5] &= ~(3 << 2);
604 if (intel_crtc_has_dp_encoder(crtc->config)) 607 if (intel_crtc_has_dp_encoder(crtc_state))
605 connector->eld[5] |= (1 << 2); 608 connector->eld[5] |= (1 << 2);
606 609
607 connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; 610 connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 5ab646ef8c9f..7ffab1abc518 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -1147,7 +1147,7 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
1147 if (!child) 1147 if (!child)
1148 return; 1148 return;
1149 1149
1150 aux_channel = child->raw[25]; 1150 aux_channel = child->common.aux_channel;
1151 ddc_pin = child->common.ddc_pin; 1151 ddc_pin = child->common.ddc_pin;
1152 1152
1153 is_dvi = child->common.device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING; 1153 is_dvi = child->common.device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING;
@@ -1677,7 +1677,8 @@ bool intel_bios_is_port_edp(struct drm_i915_private *dev_priv, enum port port)
1677 return false; 1677 return false;
1678} 1678}
1679 1679
1680bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum port port) 1680static bool child_dev_is_dp_dual_mode(const union child_device_config *p_child,
1681 enum port port)
1681{ 1682{
1682 static const struct { 1683 static const struct {
1683 u16 dp, hdmi; 1684 u16 dp, hdmi;
@@ -1691,22 +1692,35 @@ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum por
1691 [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, }, 1692 [PORT_D] = { DVO_PORT_DPD, DVO_PORT_HDMID, },
1692 [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, }, 1693 [PORT_E] = { DVO_PORT_DPE, DVO_PORT_HDMIE, },
1693 }; 1694 };
1694 int i;
1695 1695
1696 if (port == PORT_A || port >= ARRAY_SIZE(port_mapping)) 1696 if (port == PORT_A || port >= ARRAY_SIZE(port_mapping))
1697 return false; 1697 return false;
1698 1698
1699 if (!dev_priv->vbt.child_dev_num) 1699 if ((p_child->common.device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) !=
1700 (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS))
1700 return false; 1701 return false;
1701 1702
1703 if (p_child->common.dvo_port == port_mapping[port].dp)
1704 return true;
1705
1706 /* Only accept a HDMI dvo_port as DP++ if it has an AUX channel */
1707 if (p_child->common.dvo_port == port_mapping[port].hdmi &&
1708 p_child->common.aux_channel != 0)
1709 return true;
1710
1711 return false;
1712}
1713
1714bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv,
1715 enum port port)
1716{
1717 int i;
1718
1702 for (i = 0; i < dev_priv->vbt.child_dev_num; i++) { 1719 for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
1703 const union child_device_config *p_child = 1720 const union child_device_config *p_child =
1704 &dev_priv->vbt.child_dev[i]; 1721 &dev_priv->vbt.child_dev[i];
1705 1722
1706 if ((p_child->common.dvo_port == port_mapping[port].dp || 1723 if (child_dev_is_dp_dual_mode(p_child, port))
1707 p_child->common.dvo_port == port_mapping[port].hdmi) &&
1708 (p_child->common.device_type & DEVICE_TYPE_DP_DUAL_MODE_BITS) ==
1709 (DEVICE_TYPE_DP_DUAL_MODE & DEVICE_TYPE_DP_DUAL_MODE_BITS))
1710 return true; 1724 return true;
1711 } 1725 }
1712 1726
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index c410d3d6465f..c9c46a538edb 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -629,35 +629,28 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
629 cancel_fake_irq(engine); 629 cancel_fake_irq(engine);
630} 630}
631 631
632unsigned int intel_kick_waiters(struct drm_i915_private *i915) 632unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915)
633{ 633{
634 struct intel_engine_cs *engine; 634 struct intel_engine_cs *engine;
635 enum intel_engine_id id; 635 enum intel_engine_id id;
636 unsigned int mask = 0; 636 unsigned int mask = 0;
637 637
638 /* To avoid the task_struct disappearing beneath us as we wake up 638 for_each_engine(engine, i915, id) {
639 * the process, we must first inspect the task_struct->state under the 639 struct intel_breadcrumbs *b = &engine->breadcrumbs;
640 * RCU lock, i.e. as we call wake_up_process() we must be holding the
641 * rcu_read_lock().
642 */
643 for_each_engine(engine, i915, id)
644 if (unlikely(intel_engine_wakeup(engine)))
645 mask |= intel_engine_flag(engine);
646 640
647 return mask; 641 spin_lock_irq(&b->lock);
648}
649 642
650unsigned int intel_kick_signalers(struct drm_i915_private *i915) 643 if (b->first_wait) {
651{ 644 wake_up_process(b->first_wait->tsk);
652 struct intel_engine_cs *engine; 645 mask |= intel_engine_flag(engine);
653 enum intel_engine_id id; 646 }
654 unsigned int mask = 0;
655 647
656 for_each_engine(engine, i915, id) { 648 if (b->first_signal) {
657 if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) { 649 wake_up_process(b->signaler);
658 wake_up_process(engine->breadcrumbs.signaler);
659 mask |= intel_engine_flag(engine); 650 mask |= intel_engine_flag(engine);
660 } 651 }
652
653 spin_unlock_irq(&b->lock);
661 } 654 }
662 655
663 return mask; 656 return mask;
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 445108855275..d81232b79f00 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -95,8 +95,7 @@ static void ctm_mult_by_limited(uint64_t *result, int64_t *input)
95static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state) 95static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
96{ 96{
97 struct drm_crtc *crtc = crtc_state->crtc; 97 struct drm_crtc *crtc = crtc_state->crtc;
98 struct drm_device *dev = crtc->dev; 98 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
99 struct drm_i915_private *dev_priv = to_i915(dev);
100 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 99 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
101 int i, pipe = intel_crtc->pipe; 100 int i, pipe = intel_crtc->pipe;
102 uint16_t coeffs[9] = { 0, }; 101 uint16_t coeffs[9] = { 0, };
@@ -180,7 +179,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
180 I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0); 179 I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
181 I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0); 180 I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
182 181
183 if (INTEL_INFO(dev)->gen > 6) { 182 if (INTEL_GEN(dev_priv) > 6) {
184 uint16_t postoff = 0; 183 uint16_t postoff = 0;
185 184
186 if (intel_crtc_state->limited_color_range) 185 if (intel_crtc_state->limited_color_range)
@@ -345,11 +344,10 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
345static void broadwell_load_luts(struct drm_crtc_state *state) 344static void broadwell_load_luts(struct drm_crtc_state *state)
346{ 345{
347 struct drm_crtc *crtc = state->crtc; 346 struct drm_crtc *crtc = state->crtc;
348 struct drm_device *dev = crtc->dev; 347 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
349 struct drm_i915_private *dev_priv = to_i915(dev);
350 struct intel_crtc_state *intel_state = to_intel_crtc_state(state); 348 struct intel_crtc_state *intel_state = to_intel_crtc_state(state);
351 enum pipe pipe = to_intel_crtc(crtc)->pipe; 349 enum pipe pipe = to_intel_crtc(crtc)->pipe;
352 uint32_t i, lut_size = INTEL_INFO(dev)->color.degamma_lut_size; 350 uint32_t i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
353 351
354 if (crtc_state_is_legacy(state)) { 352 if (crtc_state_is_legacy(state)) {
355 haswell_load_luts(state); 353 haswell_load_luts(state);
@@ -428,8 +426,7 @@ static void broadwell_load_luts(struct drm_crtc_state *state)
428static void cherryview_load_luts(struct drm_crtc_state *state) 426static void cherryview_load_luts(struct drm_crtc_state *state)
429{ 427{
430 struct drm_crtc *crtc = state->crtc; 428 struct drm_crtc *crtc = state->crtc;
431 struct drm_device *dev = crtc->dev; 429 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
432 struct drm_i915_private *dev_priv = to_i915(dev);
433 enum pipe pipe = to_intel_crtc(crtc)->pipe; 430 enum pipe pipe = to_intel_crtc(crtc)->pipe;
434 struct drm_color_lut *lut; 431 struct drm_color_lut *lut;
435 uint32_t i, lut_size; 432 uint32_t i, lut_size;
@@ -446,7 +443,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
446 443
447 if (state->degamma_lut) { 444 if (state->degamma_lut) {
448 lut = (struct drm_color_lut *) state->degamma_lut->data; 445 lut = (struct drm_color_lut *) state->degamma_lut->data;
449 lut_size = INTEL_INFO(dev)->color.degamma_lut_size; 446 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
450 for (i = 0; i < lut_size; i++) { 447 for (i = 0; i < lut_size; i++) {
451 /* Write LUT in U0.14 format. */ 448 /* Write LUT in U0.14 format. */
452 word0 = 449 word0 =
@@ -461,7 +458,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
461 458
462 if (state->gamma_lut) { 459 if (state->gamma_lut) {
463 lut = (struct drm_color_lut *) state->gamma_lut->data; 460 lut = (struct drm_color_lut *) state->gamma_lut->data;
464 lut_size = INTEL_INFO(dev)->color.gamma_lut_size; 461 lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size;
465 for (i = 0; i < lut_size; i++) { 462 for (i = 0; i < lut_size; i++) {
466 /* Write LUT in U0.10 format. */ 463 /* Write LUT in U0.10 format. */
467 word0 = 464 word0 =
@@ -497,12 +494,12 @@ void intel_color_load_luts(struct drm_crtc_state *crtc_state)
497int intel_color_check(struct drm_crtc *crtc, 494int intel_color_check(struct drm_crtc *crtc,
498 struct drm_crtc_state *crtc_state) 495 struct drm_crtc_state *crtc_state)
499{ 496{
500 struct drm_device *dev = crtc->dev; 497 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
501 size_t gamma_length, degamma_length; 498 size_t gamma_length, degamma_length;
502 499
503 degamma_length = INTEL_INFO(dev)->color.degamma_lut_size * 500 degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size *
504 sizeof(struct drm_color_lut); 501 sizeof(struct drm_color_lut);
505 gamma_length = INTEL_INFO(dev)->color.gamma_lut_size * 502 gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size *
506 sizeof(struct drm_color_lut); 503 sizeof(struct drm_color_lut);
507 504
508 /* 505 /*
@@ -529,8 +526,7 @@ int intel_color_check(struct drm_crtc *crtc,
529 526
530void intel_color_init(struct drm_crtc *crtc) 527void intel_color_init(struct drm_crtc *crtc)
531{ 528{
532 struct drm_device *dev = crtc->dev; 529 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
533 struct drm_i915_private *dev_priv = to_i915(dev);
534 530
535 drm_mode_crtc_set_gamma_size(crtc, 256); 531 drm_mode_crtc_set_gamma_size(crtc, 256);
536 532
@@ -549,10 +545,10 @@ void intel_color_init(struct drm_crtc *crtc)
549 } 545 }
550 546
551 /* Enable color management support when we have degamma & gamma LUTs. */ 547 /* Enable color management support when we have degamma & gamma LUTs. */
552 if (INTEL_INFO(dev)->color.degamma_lut_size != 0 && 548 if (INTEL_INFO(dev_priv)->color.degamma_lut_size != 0 &&
553 INTEL_INFO(dev)->color.gamma_lut_size != 0) 549 INTEL_INFO(dev_priv)->color.gamma_lut_size != 0)
554 drm_crtc_enable_color_mgmt(crtc, 550 drm_crtc_enable_color_mgmt(crtc,
555 INTEL_INFO(dev)->color.degamma_lut_size, 551 INTEL_INFO(dev_priv)->color.degamma_lut_size,
556 true, 552 true,
557 INTEL_INFO(dev)->color.gamma_lut_size); 553 INTEL_INFO(dev_priv)->color.gamma_lut_size);
558} 554}
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 30eb95b54dcf..86ecec5601d4 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -147,14 +147,13 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder,
147 struct intel_crtc_state *crtc_state, 147 struct intel_crtc_state *crtc_state,
148 int mode) 148 int mode)
149{ 149{
150 struct drm_device *dev = encoder->base.dev; 150 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
151 struct drm_i915_private *dev_priv = to_i915(dev);
152 struct intel_crt *crt = intel_encoder_to_crt(encoder); 151 struct intel_crt *crt = intel_encoder_to_crt(encoder);
153 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 152 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
154 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; 153 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
155 u32 adpa; 154 u32 adpa;
156 155
157 if (INTEL_INFO(dev)->gen >= 5) 156 if (INTEL_GEN(dev_priv) >= 5)
158 adpa = ADPA_HOTPLUG_BITS; 157 adpa = ADPA_HOTPLUG_BITS;
159 else 158 else
160 adpa = 0; 159 adpa = 0;
@@ -673,8 +672,7 @@ static const struct dmi_system_id intel_spurious_crt_detect[] = {
673static enum drm_connector_status 672static enum drm_connector_status
674intel_crt_detect(struct drm_connector *connector, bool force) 673intel_crt_detect(struct drm_connector *connector, bool force)
675{ 674{
676 struct drm_device *dev = connector->dev; 675 struct drm_i915_private *dev_priv = to_i915(connector->dev);
677 struct drm_i915_private *dev_priv = to_i915(dev);
678 struct intel_crt *crt = intel_attached_crt(connector); 676 struct intel_crt *crt = intel_attached_crt(connector);
679 struct intel_encoder *intel_encoder = &crt->base; 677 struct intel_encoder *intel_encoder = &crt->base;
680 enum intel_display_power_domain power_domain; 678 enum intel_display_power_domain power_domain;
@@ -693,7 +691,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
693 power_domain = intel_display_port_power_domain(intel_encoder); 691 power_domain = intel_display_port_power_domain(intel_encoder);
694 intel_display_power_get(dev_priv, power_domain); 692 intel_display_power_get(dev_priv, power_domain);
695 693
696 if (I915_HAS_HOTPLUG(dev)) { 694 if (I915_HAS_HOTPLUG(dev_priv)) {
697 /* We can not rely on the HPD pin always being correctly wired 695 /* We can not rely on the HPD pin always being correctly wired
698 * up, for example many KVM do not pass it through, and so 696 * up, for example many KVM do not pass it through, and so
699 * only trust an assertion that the monitor is connected. 697 * only trust an assertion that the monitor is connected.
@@ -715,7 +713,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
715 * broken monitor (without edid) to work behind a broken kvm (that fails 713 * broken monitor (without edid) to work behind a broken kvm (that fails
716 * to have the right resistors for HP detection) needs to fix this up. 714 * to have the right resistors for HP detection) needs to fix this up.
717 * For now just bail out. */ 715 * For now just bail out. */
718 if (I915_HAS_HOTPLUG(dev) && !i915.load_detect_test) { 716 if (I915_HAS_HOTPLUG(dev_priv) && !i915.load_detect_test) {
719 status = connector_status_disconnected; 717 status = connector_status_disconnected;
720 goto out; 718 goto out;
721 } 719 }
@@ -731,7 +729,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
731 if (intel_get_load_detect_pipe(connector, NULL, &tmp, &ctx)) { 729 if (intel_get_load_detect_pipe(connector, NULL, &tmp, &ctx)) {
732 if (intel_crt_detect_ddc(connector)) 730 if (intel_crt_detect_ddc(connector))
733 status = connector_status_connected; 731 status = connector_status_connected;
734 else if (INTEL_INFO(dev)->gen < 4) 732 else if (INTEL_GEN(dev_priv) < 4)
735 status = intel_crt_load_detect(crt, 733 status = intel_crt_load_detect(crt,
736 to_intel_crtc(connector->state->crtc)->pipe); 734 to_intel_crtc(connector->state->crtc)->pipe);
737 else if (i915.load_detect_test) 735 else if (i915.load_detect_test)
@@ -793,11 +791,10 @@ static int intel_crt_set_property(struct drm_connector *connector,
793 791
794void intel_crt_reset(struct drm_encoder *encoder) 792void intel_crt_reset(struct drm_encoder *encoder)
795{ 793{
796 struct drm_device *dev = encoder->dev; 794 struct drm_i915_private *dev_priv = to_i915(encoder->dev);
797 struct drm_i915_private *dev_priv = to_i915(dev);
798 struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder)); 795 struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder));
799 796
800 if (INTEL_INFO(dev)->gen >= 5) { 797 if (INTEL_GEN(dev_priv) >= 5) {
801 u32 adpa; 798 u32 adpa;
802 799
803 adpa = I915_READ(crt->adpa_reg); 800 adpa = I915_READ(crt->adpa_reg);
@@ -915,7 +912,7 @@ void intel_crt_init(struct drm_device *dev)
915 crt->base.disable = intel_disable_crt; 912 crt->base.disable = intel_disable_crt;
916 } 913 }
917 crt->base.enable = intel_enable_crt; 914 crt->base.enable = intel_enable_crt;
918 if (I915_HAS_HOTPLUG(dev) && 915 if (I915_HAS_HOTPLUG(dev_priv) &&
919 !dmi_check_system(intel_spurious_crt_detect)) 916 !dmi_check_system(intel_spurious_crt_detect))
920 crt->base.hpd_pin = HPD_CRT; 917 crt->base.hpd_pin = HPD_CRT;
921 if (HAS_DDI(dev_priv)) { 918 if (HAS_DDI(dev_priv)) {
@@ -932,7 +929,7 @@ void intel_crt_init(struct drm_device *dev)
932 929
933 drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs); 930 drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
934 931
935 if (!I915_HAS_HOTPLUG(dev)) 932 if (!I915_HAS_HOTPLUG(dev_priv))
936 intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT; 933 intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT;
937 934
938 /* 935 /*
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 938ac4dbcb45..10ec9d4b7d45 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -1753,8 +1753,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
1753 struct drm_connector_state *old_conn_state) 1753 struct drm_connector_state *old_conn_state)
1754{ 1754{
1755 struct drm_encoder *encoder = &intel_encoder->base; 1755 struct drm_encoder *encoder = &intel_encoder->base;
1756 struct drm_device *dev = encoder->dev; 1756 struct drm_i915_private *dev_priv = to_i915(encoder->dev);
1757 struct drm_i915_private *dev_priv = to_i915(dev);
1758 enum port port = intel_ddi_get_encoder_port(intel_encoder); 1757 enum port port = intel_ddi_get_encoder_port(intel_encoder);
1759 int type = intel_encoder->type; 1758 int type = intel_encoder->type;
1760 uint32_t val; 1759 uint32_t val;
@@ -1787,7 +1786,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
1787 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) 1786 if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
1788 I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) | 1787 I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) |
1789 DPLL_CTRL2_DDI_CLK_OFF(port))); 1788 DPLL_CTRL2_DDI_CLK_OFF(port)));
1790 else if (INTEL_INFO(dev)->gen < 9) 1789 else if (INTEL_GEN(dev_priv) < 9)
1791 I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE); 1790 I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE);
1792 1791
1793 if (type == INTEL_OUTPUT_HDMI) { 1792 if (type == INTEL_OUTPUT_HDMI) {
@@ -1837,8 +1836,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
1837 struct drm_encoder *encoder = &intel_encoder->base; 1836 struct drm_encoder *encoder = &intel_encoder->base;
1838 struct drm_crtc *crtc = encoder->crtc; 1837 struct drm_crtc *crtc = encoder->crtc;
1839 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1838 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1840 struct drm_device *dev = encoder->dev; 1839 struct drm_i915_private *dev_priv = to_i915(encoder->dev);
1841 struct drm_i915_private *dev_priv = to_i915(dev);
1842 enum port port = intel_ddi_get_encoder_port(intel_encoder); 1840 enum port port = intel_ddi_get_encoder_port(intel_encoder);
1843 int type = intel_encoder->type; 1841 int type = intel_encoder->type;
1844 1842
@@ -1856,7 +1854,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
1856 } else if (type == INTEL_OUTPUT_EDP) { 1854 } else if (type == INTEL_OUTPUT_EDP) {
1857 struct intel_dp *intel_dp = enc_to_intel_dp(encoder); 1855 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
1858 1856
1859 if (port == PORT_A && INTEL_INFO(dev)->gen < 9) 1857 if (port == PORT_A && INTEL_GEN(dev_priv) < 9)
1860 intel_dp_stop_link_train(intel_dp); 1858 intel_dp_stop_link_train(intel_dp);
1861 1859
1862 intel_edp_backlight_on(intel_dp); 1860 intel_edp_backlight_on(intel_dp);
@@ -1866,7 +1864,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
1866 1864
1867 if (intel_crtc->config->has_audio) { 1865 if (intel_crtc->config->has_audio) {
1868 intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); 1866 intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
1869 intel_audio_codec_enable(intel_encoder); 1867 intel_audio_codec_enable(intel_encoder, pipe_config, conn_state);
1870 } 1868 }
1871} 1869}
1872 1870
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 6fd6283fa1f8..8d270f7650de 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -1035,9 +1035,8 @@ enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1035 return crtc->config->cpu_transcoder; 1035 return crtc->config->cpu_transcoder;
1036} 1036}
1037 1037
1038static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) 1038static bool pipe_dsl_stopped(struct drm_i915_private *dev_priv, enum pipe pipe)
1039{ 1039{
1040 struct drm_i915_private *dev_priv = to_i915(dev);
1041 i915_reg_t reg = PIPEDSL(pipe); 1040 i915_reg_t reg = PIPEDSL(pipe);
1042 u32 line1, line2; 1041 u32 line1, line2;
1043 u32 line_mask; 1042 u32 line_mask;
@@ -1072,12 +1071,11 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
1072 */ 1071 */
1073static void intel_wait_for_pipe_off(struct intel_crtc *crtc) 1072static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1074{ 1073{
1075 struct drm_device *dev = crtc->base.dev; 1074 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1076 struct drm_i915_private *dev_priv = to_i915(dev);
1077 enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; 1075 enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1078 enum pipe pipe = crtc->pipe; 1076 enum pipe pipe = crtc->pipe;
1079 1077
1080 if (INTEL_INFO(dev)->gen >= 4) { 1078 if (INTEL_GEN(dev_priv) >= 4) {
1081 i915_reg_t reg = PIPECONF(cpu_transcoder); 1079 i915_reg_t reg = PIPECONF(cpu_transcoder);
1082 1080
1083 /* Wait for the Pipe State to go off */ 1081 /* Wait for the Pipe State to go off */
@@ -1087,7 +1085,7 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1087 WARN(1, "pipe_off wait timed out\n"); 1085 WARN(1, "pipe_off wait timed out\n");
1088 } else { 1086 } else {
1089 /* Wait for the display line to settle */ 1087 /* Wait for the display line to settle */
1090 if (wait_for(pipe_dsl_stopped(dev, pipe), 100)) 1088 if (wait_for(pipe_dsl_stopped(dev_priv, pipe), 100))
1091 WARN(1, "pipe_off wait timed out\n"); 1089 WARN(1, "pipe_off wait timed out\n");
1092 } 1090 }
1093} 1091}
@@ -1293,11 +1291,10 @@ static void assert_plane(struct drm_i915_private *dev_priv,
1293static void assert_planes_disabled(struct drm_i915_private *dev_priv, 1291static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1294 enum pipe pipe) 1292 enum pipe pipe)
1295{ 1293{
1296 struct drm_device *dev = &dev_priv->drm;
1297 int i; 1294 int i;
1298 1295
1299 /* Primary planes are fixed to pipes on gen4+ */ 1296 /* Primary planes are fixed to pipes on gen4+ */
1300 if (INTEL_INFO(dev)->gen >= 4) { 1297 if (INTEL_GEN(dev_priv) >= 4) {
1301 u32 val = I915_READ(DSPCNTR(pipe)); 1298 u32 val = I915_READ(DSPCNTR(pipe));
1302 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE, 1299 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
1303 "plane %c assertion failure, should be disabled but not\n", 1300 "plane %c assertion failure, should be disabled but not\n",
@@ -1319,10 +1316,9 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1319static void assert_sprites_disabled(struct drm_i915_private *dev_priv, 1316static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1320 enum pipe pipe) 1317 enum pipe pipe)
1321{ 1318{
1322 struct drm_device *dev = &dev_priv->drm;
1323 int sprite; 1319 int sprite;
1324 1320
1325 if (INTEL_INFO(dev)->gen >= 9) { 1321 if (INTEL_GEN(dev_priv) >= 9) {
1326 for_each_sprite(dev_priv, pipe, sprite) { 1322 for_each_sprite(dev_priv, pipe, sprite) {
1327 u32 val = I915_READ(PLANE_CTL(pipe, sprite)); 1323 u32 val = I915_READ(PLANE_CTL(pipe, sprite));
1328 I915_STATE_WARN(val & PLANE_CTL_ENABLE, 1324 I915_STATE_WARN(val & PLANE_CTL_ENABLE,
@@ -1336,12 +1332,12 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1336 "sprite %c assertion failure, should be off on pipe %c but is still active\n", 1332 "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1337 sprite_name(pipe, sprite), pipe_name(pipe)); 1333 sprite_name(pipe, sprite), pipe_name(pipe));
1338 } 1334 }
1339 } else if (INTEL_INFO(dev)->gen >= 7) { 1335 } else if (INTEL_GEN(dev_priv) >= 7) {
1340 u32 val = I915_READ(SPRCTL(pipe)); 1336 u32 val = I915_READ(SPRCTL(pipe));
1341 I915_STATE_WARN(val & SPRITE_ENABLE, 1337 I915_STATE_WARN(val & SPRITE_ENABLE,
1342 "sprite %c assertion failure, should be off on pipe %c but is still active\n", 1338 "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1343 plane_name(pipe), pipe_name(pipe)); 1339 plane_name(pipe), pipe_name(pipe));
1344 } else if (INTEL_INFO(dev)->gen >= 5) { 1340 } else if (INTEL_GEN(dev_priv) >= 5) {
1345 u32 val = I915_READ(DVSCNTR(pipe)); 1341 u32 val = I915_READ(DVSCNTR(pipe));
1346 I915_STATE_WARN(val & DVS_ENABLE, 1342 I915_STATE_WARN(val & DVS_ENABLE,
1347 "sprite %c assertion failure, should be off on pipe %c but is still active\n", 1343 "sprite %c assertion failure, should be off on pipe %c but is still active\n",
@@ -1595,12 +1591,12 @@ static void chv_enable_pll(struct intel_crtc *crtc,
1595 } 1591 }
1596} 1592}
1597 1593
1598static int intel_num_dvo_pipes(struct drm_device *dev) 1594static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
1599{ 1595{
1600 struct intel_crtc *crtc; 1596 struct intel_crtc *crtc;
1601 int count = 0; 1597 int count = 0;
1602 1598
1603 for_each_intel_crtc(dev, crtc) { 1599 for_each_intel_crtc(&dev_priv->drm, crtc) {
1604 count += crtc->base.state->active && 1600 count += crtc->base.state->active &&
1605 intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO); 1601 intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
1606 } 1602 }
@@ -1610,8 +1606,7 @@ static int intel_num_dvo_pipes(struct drm_device *dev)
1610 1606
1611static void i9xx_enable_pll(struct intel_crtc *crtc) 1607static void i9xx_enable_pll(struct intel_crtc *crtc)
1612{ 1608{
1613 struct drm_device *dev = crtc->base.dev; 1609 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1614 struct drm_i915_private *dev_priv = to_i915(dev);
1615 i915_reg_t reg = DPLL(crtc->pipe); 1610 i915_reg_t reg = DPLL(crtc->pipe);
1616 u32 dpll = crtc->config->dpll_hw_state.dpll; 1611 u32 dpll = crtc->config->dpll_hw_state.dpll;
1617 1612
@@ -1622,7 +1617,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
1622 assert_panel_unlocked(dev_priv, crtc->pipe); 1617 assert_panel_unlocked(dev_priv, crtc->pipe);
1623 1618
1624 /* Enable DVO 2x clock on both PLLs if necessary */ 1619 /* Enable DVO 2x clock on both PLLs if necessary */
1625 if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev) > 0) { 1620 if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
1626 /* 1621 /*
1627 * It appears to be important that we don't enable this 1622 * It appears to be important that we don't enable this
1628 * for the current pipe before otherwise configuring the 1623 * for the current pipe before otherwise configuring the
@@ -1647,7 +1642,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
1647 POSTING_READ(reg); 1642 POSTING_READ(reg);
1648 udelay(150); 1643 udelay(150);
1649 1644
1650 if (INTEL_INFO(dev)->gen >= 4) { 1645 if (INTEL_GEN(dev_priv) >= 4) {
1651 I915_WRITE(DPLL_MD(crtc->pipe), 1646 I915_WRITE(DPLL_MD(crtc->pipe),
1652 crtc->config->dpll_hw_state.dpll_md); 1647 crtc->config->dpll_hw_state.dpll_md);
1653 } else { 1648 } else {
@@ -1682,14 +1677,13 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
1682 */ 1677 */
1683static void i9xx_disable_pll(struct intel_crtc *crtc) 1678static void i9xx_disable_pll(struct intel_crtc *crtc)
1684{ 1679{
1685 struct drm_device *dev = crtc->base.dev; 1680 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1686 struct drm_i915_private *dev_priv = to_i915(dev);
1687 enum pipe pipe = crtc->pipe; 1681 enum pipe pipe = crtc->pipe;
1688 1682
1689 /* Disable DVO 2x clock on both PLLs if necessary */ 1683 /* Disable DVO 2x clock on both PLLs if necessary */
1690 if (IS_I830(dev_priv) && 1684 if (IS_I830(dev_priv) &&
1691 intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) && 1685 intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
1692 !intel_num_dvo_pipes(dev)) { 1686 !intel_num_dvo_pipes(dev_priv)) {
1693 I915_WRITE(DPLL(PIPE_B), 1687 I915_WRITE(DPLL(PIPE_B),
1694 I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); 1688 I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1695 I915_WRITE(DPLL(PIPE_A), 1689 I915_WRITE(DPLL(PIPE_A),
@@ -3004,11 +2998,9 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
3004 const struct intel_crtc_state *crtc_state, 2998 const struct intel_crtc_state *crtc_state,
3005 const struct intel_plane_state *plane_state) 2999 const struct intel_plane_state *plane_state)
3006{ 3000{
3007 struct drm_device *dev = primary->dev; 3001 struct drm_i915_private *dev_priv = to_i915(primary->dev);
3008 struct drm_i915_private *dev_priv = to_i915(dev);
3009 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); 3002 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3010 struct drm_framebuffer *fb = plane_state->base.fb; 3003 struct drm_framebuffer *fb = plane_state->base.fb;
3011 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3012 int plane = intel_crtc->plane; 3004 int plane = intel_crtc->plane;
3013 u32 linear_offset; 3005 u32 linear_offset;
3014 u32 dspcntr; 3006 u32 dspcntr;
@@ -3021,7 +3013,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
3021 3013
3022 dspcntr |= DISPLAY_PLANE_ENABLE; 3014 dspcntr |= DISPLAY_PLANE_ENABLE;
3023 3015
3024 if (INTEL_INFO(dev)->gen < 4) { 3016 if (INTEL_GEN(dev_priv) < 4) {
3025 if (intel_crtc->pipe == PIPE_B) 3017 if (intel_crtc->pipe == PIPE_B)
3026 dspcntr |= DISPPLANE_SEL_PIPE_B; 3018 dspcntr |= DISPPLANE_SEL_PIPE_B;
3027 3019
@@ -3070,25 +3062,31 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
3070 fb->modifier[0] == I915_FORMAT_MOD_X_TILED) 3062 fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
3071 dspcntr |= DISPPLANE_TILED; 3063 dspcntr |= DISPPLANE_TILED;
3072 3064
3065 if (rotation & DRM_ROTATE_180)
3066 dspcntr |= DISPPLANE_ROTATE_180;
3067
3068 if (rotation & DRM_REFLECT_X)
3069 dspcntr |= DISPPLANE_MIRROR;
3070
3073 if (IS_G4X(dev_priv)) 3071 if (IS_G4X(dev_priv))
3074 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; 3072 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3075 3073
3076 intel_add_fb_offsets(&x, &y, plane_state, 0); 3074 intel_add_fb_offsets(&x, &y, plane_state, 0);
3077 3075
3078 if (INTEL_INFO(dev)->gen >= 4) 3076 if (INTEL_GEN(dev_priv) >= 4)
3079 intel_crtc->dspaddr_offset = 3077 intel_crtc->dspaddr_offset =
3080 intel_compute_tile_offset(&x, &y, plane_state, 0); 3078 intel_compute_tile_offset(&x, &y, plane_state, 0);
3081 3079
3082 if (rotation == DRM_ROTATE_180) { 3080 if (rotation & DRM_ROTATE_180) {
3083 dspcntr |= DISPPLANE_ROTATE_180; 3081 x += crtc_state->pipe_src_w - 1;
3084 3082 y += crtc_state->pipe_src_h - 1;
3085 x += (crtc_state->pipe_src_w - 1); 3083 } else if (rotation & DRM_REFLECT_X) {
3086 y += (crtc_state->pipe_src_h - 1); 3084 x += crtc_state->pipe_src_w - 1;
3087 } 3085 }
3088 3086
3089 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); 3087 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
3090 3088
3091 if (INTEL_INFO(dev)->gen < 4) 3089 if (INTEL_GEN(dev_priv) < 4)
3092 intel_crtc->dspaddr_offset = linear_offset; 3090 intel_crtc->dspaddr_offset = linear_offset;
3093 3091
3094 intel_crtc->adjusted_x = x; 3092 intel_crtc->adjusted_x = x;
@@ -3097,14 +3095,17 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
3097 I915_WRITE(reg, dspcntr); 3095 I915_WRITE(reg, dspcntr);
3098 3096
3099 I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]); 3097 I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
3100 if (INTEL_INFO(dev)->gen >= 4) { 3098 if (INTEL_GEN(dev_priv) >= 4) {
3101 I915_WRITE(DSPSURF(plane), 3099 I915_WRITE(DSPSURF(plane),
3102 intel_fb_gtt_offset(fb, rotation) + 3100 intel_fb_gtt_offset(fb, rotation) +
3103 intel_crtc->dspaddr_offset); 3101 intel_crtc->dspaddr_offset);
3104 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x); 3102 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
3105 I915_WRITE(DSPLINOFF(plane), linear_offset); 3103 I915_WRITE(DSPLINOFF(plane), linear_offset);
3106 } else 3104 } else {
3107 I915_WRITE(DSPADDR(plane), i915_gem_object_ggtt_offset(obj, NULL) + linear_offset); 3105 I915_WRITE(DSPADDR(plane),
3106 intel_fb_gtt_offset(fb, rotation) +
3107 intel_crtc->dspaddr_offset);
3108 }
3108 POSTING_READ(reg); 3109 POSTING_READ(reg);
3109} 3110}
3110 3111
@@ -3172,6 +3173,9 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
3172 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) 3173 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
3173 dspcntr |= DISPPLANE_TILED; 3174 dspcntr |= DISPPLANE_TILED;
3174 3175
3176 if (rotation & DRM_ROTATE_180)
3177 dspcntr |= DISPPLANE_ROTATE_180;
3178
3175 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) 3179 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv))
3176 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE; 3180 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
3177 3181
@@ -3180,13 +3184,11 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
3180 intel_crtc->dspaddr_offset = 3184 intel_crtc->dspaddr_offset =
3181 intel_compute_tile_offset(&x, &y, plane_state, 0); 3185 intel_compute_tile_offset(&x, &y, plane_state, 0);
3182 3186
3183 if (rotation == DRM_ROTATE_180) { 3187 /* HSW+ does this automagically in hardware */
3184 dspcntr |= DISPPLANE_ROTATE_180; 3188 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) &&
3185 3189 rotation & DRM_ROTATE_180) {
3186 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) { 3190 x += crtc_state->pipe_src_w - 1;
3187 x += (crtc_state->pipe_src_w - 1); 3191 y += crtc_state->pipe_src_h - 1;
3188 y += (crtc_state->pipe_src_h - 1);
3189 }
3190 } 3192 }
3191 3193
3192 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); 3194 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -3376,9 +3378,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
3376 struct drm_i915_private *dev_priv = to_i915(dev); 3378 struct drm_i915_private *dev_priv = to_i915(dev);
3377 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); 3379 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3378 struct drm_framebuffer *fb = plane_state->base.fb; 3380 struct drm_framebuffer *fb = plane_state->base.fb;
3379 const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
3380 const struct skl_plane_wm *p_wm =
3381 &crtc_state->wm.skl.optimal.planes[0];
3382 int pipe = intel_crtc->pipe; 3381 int pipe = intel_crtc->pipe;
3383 u32 plane_ctl; 3382 u32 plane_ctl;
3384 unsigned int rotation = plane_state->base.rotation; 3383 unsigned int rotation = plane_state->base.rotation;
@@ -3414,9 +3413,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
3414 intel_crtc->adjusted_x = src_x; 3413 intel_crtc->adjusted_x = src_x;
3415 intel_crtc->adjusted_y = src_y; 3414 intel_crtc->adjusted_y = src_y;
3416 3415
3417 if (wm->dirty_pipes & drm_crtc_mask(&intel_crtc->base))
3418 skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, 0);
3419
3420 I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl); 3416 I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
3421 I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x); 3417 I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x);
3422 I915_WRITE(PLANE_STRIDE(pipe, 0), stride); 3418 I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
@@ -3449,18 +3445,8 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
3449 struct drm_device *dev = crtc->dev; 3445 struct drm_device *dev = crtc->dev;
3450 struct drm_i915_private *dev_priv = to_i915(dev); 3446 struct drm_i915_private *dev_priv = to_i915(dev);
3451 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 3447 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3452 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
3453 const struct skl_plane_wm *p_wm = &cstate->wm.skl.optimal.planes[0];
3454 int pipe = intel_crtc->pipe; 3448 int pipe = intel_crtc->pipe;
3455 3449
3456 /*
3457 * We only populate skl_results on watermark updates, and if the
3458 * plane's visiblity isn't actually changing neither is its watermarks.
3459 */
3460 if (!crtc->primary->state->visible)
3461 skl_write_plane_wm(intel_crtc, p_wm,
3462 &dev_priv->wm.skl_results.ddb, 0);
3463
3464 I915_WRITE(PLANE_CTL(pipe, 0), 0); 3450 I915_WRITE(PLANE_CTL(pipe, 0), 0);
3465 I915_WRITE(PLANE_SURF(pipe, 0), 0); 3451 I915_WRITE(PLANE_SURF(pipe, 0), 0);
3466 POSTING_READ(PLANE_SURF(pipe, 0)); 3452 POSTING_READ(PLANE_SURF(pipe, 0));
@@ -3510,7 +3496,7 @@ __intel_display_resume(struct drm_device *dev,
3510 int i, ret; 3496 int i, ret;
3511 3497
3512 intel_modeset_setup_hw_state(dev); 3498 intel_modeset_setup_hw_state(dev);
3513 i915_redisable_vga(dev); 3499 i915_redisable_vga(to_i915(dev));
3514 3500
3515 if (!state) 3501 if (!state)
3516 return 0; 3502 return 0;
@@ -3687,8 +3673,7 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
3687static void intel_update_pipe_config(struct intel_crtc *crtc, 3673static void intel_update_pipe_config(struct intel_crtc *crtc,
3688 struct intel_crtc_state *old_crtc_state) 3674 struct intel_crtc_state *old_crtc_state)
3689{ 3675{
3690 struct drm_device *dev = crtc->base.dev; 3676 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3691 struct drm_i915_private *dev_priv = to_i915(dev);
3692 struct intel_crtc_state *pipe_config = 3677 struct intel_crtc_state *pipe_config =
3693 to_intel_crtc_state(crtc->base.state); 3678 to_intel_crtc_state(crtc->base.state);
3694 3679
@@ -3713,7 +3698,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc,
3713 (pipe_config->pipe_src_h - 1)); 3698 (pipe_config->pipe_src_h - 1));
3714 3699
3715 /* on skylake this is done by detaching scalers */ 3700 /* on skylake this is done by detaching scalers */
3716 if (INTEL_INFO(dev)->gen >= 9) { 3701 if (INTEL_GEN(dev_priv) >= 9) {
3717 skl_detach_scalers(crtc); 3702 skl_detach_scalers(crtc);
3718 3703
3719 if (pipe_config->pch_pfit.enabled) 3704 if (pipe_config->pch_pfit.enabled)
@@ -4734,13 +4719,8 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4734 */ 4719 */
4735int skl_update_scaler_crtc(struct intel_crtc_state *state) 4720int skl_update_scaler_crtc(struct intel_crtc_state *state)
4736{ 4721{
4737 struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
4738 const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; 4722 const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4739 4723
4740 DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n",
4741 intel_crtc->base.base.id, intel_crtc->base.name,
4742 intel_crtc->pipe, SKL_CRTC_INDEX);
4743
4744 return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, 4724 return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4745 &state->scaler_state.scaler_id, DRM_ROTATE_0, 4725 &state->scaler_state.scaler_id, DRM_ROTATE_0,
4746 state->pipe_src_w, state->pipe_src_h, 4726 state->pipe_src_w, state->pipe_src_h,
@@ -4761,7 +4741,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4761 struct intel_plane_state *plane_state) 4741 struct intel_plane_state *plane_state)
4762{ 4742{
4763 4743
4764 struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4765 struct intel_plane *intel_plane = 4744 struct intel_plane *intel_plane =
4766 to_intel_plane(plane_state->base.plane); 4745 to_intel_plane(plane_state->base.plane);
4767 struct drm_framebuffer *fb = plane_state->base.fb; 4746 struct drm_framebuffer *fb = plane_state->base.fb;
@@ -4769,10 +4748,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4769 4748
4770 bool force_detach = !fb || !plane_state->base.visible; 4749 bool force_detach = !fb || !plane_state->base.visible;
4771 4750
4772 DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n",
4773 intel_plane->base.base.id, intel_plane->base.name,
4774 intel_crtc->pipe, drm_plane_index(&intel_plane->base));
4775
4776 ret = skl_update_scaler(crtc_state, force_detach, 4751 ret = skl_update_scaler(crtc_state, force_detach,
4777 drm_plane_index(&intel_plane->base), 4752 drm_plane_index(&intel_plane->base),
4778 &plane_state->scaler_id, 4753 &plane_state->scaler_id,
@@ -5096,6 +5071,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
5096 struct drm_plane_state *old_pri_state = 5071 struct drm_plane_state *old_pri_state =
5097 drm_atomic_get_existing_plane_state(old_state, primary); 5072 drm_atomic_get_existing_plane_state(old_state, primary);
5098 bool modeset = needs_modeset(&pipe_config->base); 5073 bool modeset = needs_modeset(&pipe_config->base);
5074 struct intel_atomic_state *old_intel_state =
5075 to_intel_atomic_state(old_state);
5099 5076
5100 if (old_pri_state) { 5077 if (old_pri_state) {
5101 struct intel_plane_state *primary_state = 5078 struct intel_plane_state *primary_state =
@@ -5163,7 +5140,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
5163 * us to. 5140 * us to.
5164 */ 5141 */
5165 if (dev_priv->display.initial_watermarks != NULL) 5142 if (dev_priv->display.initial_watermarks != NULL)
5166 dev_priv->display.initial_watermarks(pipe_config); 5143 dev_priv->display.initial_watermarks(old_intel_state,
5144 pipe_config);
5167 else if (pipe_config->update_wm_pre) 5145 else if (pipe_config->update_wm_pre)
5168 intel_update_watermarks(crtc); 5146 intel_update_watermarks(crtc);
5169} 5147}
@@ -5319,6 +5297,8 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5319 struct drm_i915_private *dev_priv = to_i915(dev); 5297 struct drm_i915_private *dev_priv = to_i915(dev);
5320 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5298 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5321 int pipe = intel_crtc->pipe; 5299 int pipe = intel_crtc->pipe;
5300 struct intel_atomic_state *old_intel_state =
5301 to_intel_atomic_state(old_state);
5322 5302
5323 if (WARN_ON(intel_crtc->active)) 5303 if (WARN_ON(intel_crtc->active))
5324 return; 5304 return;
@@ -5377,7 +5357,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
5377 intel_color_load_luts(&pipe_config->base); 5357 intel_color_load_luts(&pipe_config->base);
5378 5358
5379 if (dev_priv->display.initial_watermarks != NULL) 5359 if (dev_priv->display.initial_watermarks != NULL)
5380 dev_priv->display.initial_watermarks(intel_crtc->config); 5360 dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config);
5381 intel_enable_pipe(intel_crtc); 5361 intel_enable_pipe(intel_crtc);
5382 5362
5383 if (intel_crtc->config->has_pch_encoder) 5363 if (intel_crtc->config->has_pch_encoder)
@@ -5408,11 +5388,12 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5408 struct drm_atomic_state *old_state) 5388 struct drm_atomic_state *old_state)
5409{ 5389{
5410 struct drm_crtc *crtc = pipe_config->base.crtc; 5390 struct drm_crtc *crtc = pipe_config->base.crtc;
5411 struct drm_device *dev = crtc->dev; 5391 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5412 struct drm_i915_private *dev_priv = to_i915(dev);
5413 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5392 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5414 int pipe = intel_crtc->pipe, hsw_workaround_pipe; 5393 int pipe = intel_crtc->pipe, hsw_workaround_pipe;
5415 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; 5394 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5395 struct intel_atomic_state *old_intel_state =
5396 to_intel_atomic_state(old_state);
5416 5397
5417 if (WARN_ON(intel_crtc->active)) 5398 if (WARN_ON(intel_crtc->active))
5418 return; 5399 return;
@@ -5467,7 +5448,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5467 if (!transcoder_is_dsi(cpu_transcoder)) 5448 if (!transcoder_is_dsi(cpu_transcoder))
5468 intel_ddi_enable_pipe_clock(intel_crtc); 5449 intel_ddi_enable_pipe_clock(intel_crtc);
5469 5450
5470 if (INTEL_INFO(dev)->gen >= 9) 5451 if (INTEL_GEN(dev_priv) >= 9)
5471 skylake_pfit_enable(intel_crtc); 5452 skylake_pfit_enable(intel_crtc);
5472 else 5453 else
5473 ironlake_pfit_enable(intel_crtc); 5454 ironlake_pfit_enable(intel_crtc);
@@ -5483,7 +5464,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5483 intel_ddi_enable_transcoder_func(crtc); 5464 intel_ddi_enable_transcoder_func(crtc);
5484 5465
5485 if (dev_priv->display.initial_watermarks != NULL) 5466 if (dev_priv->display.initial_watermarks != NULL)
5486 dev_priv->display.initial_watermarks(pipe_config); 5467 dev_priv->display.initial_watermarks(old_intel_state,
5468 pipe_config);
5487 else 5469 else
5488 intel_update_watermarks(intel_crtc); 5470 intel_update_watermarks(intel_crtc);
5489 5471
@@ -5494,7 +5476,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
5494 if (intel_crtc->config->has_pch_encoder) 5476 if (intel_crtc->config->has_pch_encoder)
5495 lpt_pch_enable(crtc); 5477 lpt_pch_enable(crtc);
5496 5478
5497 if (intel_crtc->config->dp_encoder_is_mst) 5479 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
5498 intel_ddi_set_vc_payload_alloc(crtc, true); 5480 intel_ddi_set_vc_payload_alloc(crtc, true);
5499 5481
5500 assert_vblank_disabled(crtc); 5482 assert_vblank_disabled(crtc);
@@ -5599,8 +5581,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5599 struct drm_atomic_state *old_state) 5581 struct drm_atomic_state *old_state)
5600{ 5582{
5601 struct drm_crtc *crtc = old_crtc_state->base.crtc; 5583 struct drm_crtc *crtc = old_crtc_state->base.crtc;
5602 struct drm_device *dev = crtc->dev; 5584 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5603 struct drm_i915_private *dev_priv = to_i915(dev);
5604 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5585 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5605 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; 5586 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5606 5587
@@ -5617,13 +5598,13 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
5617 if (!transcoder_is_dsi(cpu_transcoder)) 5598 if (!transcoder_is_dsi(cpu_transcoder))
5618 intel_disable_pipe(intel_crtc); 5599 intel_disable_pipe(intel_crtc);
5619 5600
5620 if (intel_crtc->config->dp_encoder_is_mst) 5601 if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
5621 intel_ddi_set_vc_payload_alloc(crtc, false); 5602 intel_ddi_set_vc_payload_alloc(crtc, false);
5622 5603
5623 if (!transcoder_is_dsi(cpu_transcoder)) 5604 if (!transcoder_is_dsi(cpu_transcoder))
5624 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder); 5605 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
5625 5606
5626 if (INTEL_INFO(dev)->gen >= 9) 5607 if (INTEL_GEN(dev_priv) >= 9)
5627 skylake_scaler_disable(intel_crtc); 5608 skylake_scaler_disable(intel_crtc);
5628 else 5609 else
5629 ironlake_pfit_disable(intel_crtc, false); 5610 ironlake_pfit_disable(intel_crtc, false);
@@ -7051,7 +7032,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
7051 } 7032 }
7052 } 7033 }
7053 7034
7054 if (INTEL_INFO(dev)->num_pipes == 2) 7035 if (INTEL_INFO(dev_priv)->num_pipes == 2)
7055 return 0; 7036 return 0;
7056 7037
7057 /* Ivybridge 3 pipe is really complicated */ 7038 /* Ivybridge 3 pipe is really complicated */
@@ -7192,7 +7173,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
7192 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; 7173 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
7193 int clock_limit = dev_priv->max_dotclk_freq; 7174 int clock_limit = dev_priv->max_dotclk_freq;
7194 7175
7195 if (INTEL_INFO(dev)->gen < 4) { 7176 if (INTEL_GEN(dev_priv) < 4) {
7196 clock_limit = dev_priv->max_cdclk_freq * 9 / 10; 7177 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
7197 7178
7198 /* 7179 /*
@@ -7786,12 +7767,11 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
7786 struct intel_link_m_n *m_n, 7767 struct intel_link_m_n *m_n,
7787 struct intel_link_m_n *m2_n2) 7768 struct intel_link_m_n *m2_n2)
7788{ 7769{
7789 struct drm_device *dev = crtc->base.dev; 7770 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
7790 struct drm_i915_private *dev_priv = to_i915(dev);
7791 int pipe = crtc->pipe; 7771 int pipe = crtc->pipe;
7792 enum transcoder transcoder = crtc->config->cpu_transcoder; 7772 enum transcoder transcoder = crtc->config->cpu_transcoder;
7793 7773
7794 if (INTEL_INFO(dev)->gen >= 5) { 7774 if (INTEL_GEN(dev_priv) >= 5) {
7795 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m); 7775 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7796 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n); 7776 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7797 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m); 7777 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
@@ -8245,8 +8225,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
8245 8225
8246static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) 8226static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
8247{ 8227{
8248 struct drm_device *dev = intel_crtc->base.dev; 8228 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
8249 struct drm_i915_private *dev_priv = to_i915(dev);
8250 enum pipe pipe = intel_crtc->pipe; 8229 enum pipe pipe = intel_crtc->pipe;
8251 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; 8230 enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8252 const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; 8231 const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
@@ -8272,7 +8251,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
8272 vsyncshift += adjusted_mode->crtc_htotal; 8251 vsyncshift += adjusted_mode->crtc_htotal;
8273 } 8252 }
8274 8253
8275 if (INTEL_INFO(dev)->gen > 3) 8254 if (INTEL_GEN(dev_priv) > 3)
8276 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift); 8255 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
8277 8256
8278 I915_WRITE(HTOTAL(cpu_transcoder), 8257 I915_WRITE(HTOTAL(cpu_transcoder),
@@ -8395,8 +8374,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
8395 8374
8396static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) 8375static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
8397{ 8376{
8398 struct drm_device *dev = intel_crtc->base.dev; 8377 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
8399 struct drm_i915_private *dev_priv = to_i915(dev);
8400 uint32_t pipeconf; 8378 uint32_t pipeconf;
8401 8379
8402 pipeconf = 0; 8380 pipeconf = 0;
@@ -8432,7 +8410,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
8432 } 8410 }
8433 } 8411 }
8434 8412
8435 if (HAS_PIPE_CXSR(dev)) { 8413 if (HAS_PIPE_CXSR(dev_priv)) {
8436 if (intel_crtc->lowfreq_avail) { 8414 if (intel_crtc->lowfreq_avail) {
8437 DRM_DEBUG_KMS("enabling CxSR downclocking\n"); 8415 DRM_DEBUG_KMS("enabling CxSR downclocking\n");
8438 pipeconf |= PIPECONF_CXSR_DOWNCLOCK; 8416 pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
@@ -8442,7 +8420,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
8442 } 8420 }
8443 8421
8444 if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { 8422 if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
8445 if (INTEL_INFO(dev)->gen < 4 || 8423 if (INTEL_GEN(dev_priv) < 4 ||
8446 intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) 8424 intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
8447 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; 8425 pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
8448 else 8426 else
@@ -8650,8 +8628,7 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8650static void i9xx_get_pfit_config(struct intel_crtc *crtc, 8628static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8651 struct intel_crtc_state *pipe_config) 8629 struct intel_crtc_state *pipe_config)
8652{ 8630{
8653 struct drm_device *dev = crtc->base.dev; 8631 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8654 struct drm_i915_private *dev_priv = to_i915(dev);
8655 uint32_t tmp; 8632 uint32_t tmp;
8656 8633
8657 if (INTEL_GEN(dev_priv) <= 3 && 8634 if (INTEL_GEN(dev_priv) <= 3 &&
@@ -8663,7 +8640,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8663 return; 8640 return;
8664 8641
8665 /* Check whether the pfit is attached to our pipe. */ 8642 /* Check whether the pfit is attached to our pipe. */
8666 if (INTEL_INFO(dev)->gen < 4) { 8643 if (INTEL_GEN(dev_priv) < 4) {
8667 if (crtc->pipe != PIPE_B) 8644 if (crtc->pipe != PIPE_B)
8668 return; 8645 return;
8669 } else { 8646 } else {
@@ -8727,7 +8704,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8727 8704
8728 fb = &intel_fb->base; 8705 fb = &intel_fb->base;
8729 8706
8730 if (INTEL_INFO(dev)->gen >= 4) { 8707 if (INTEL_GEN(dev_priv) >= 4) {
8731 if (val & DISPPLANE_TILED) { 8708 if (val & DISPPLANE_TILED) {
8732 plane_config->tiling = I915_TILING_X; 8709 plane_config->tiling = I915_TILING_X;
8733 fb->modifier[0] = I915_FORMAT_MOD_X_TILED; 8710 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
@@ -8739,7 +8716,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8739 fb->pixel_format = fourcc; 8716 fb->pixel_format = fourcc;
8740 fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8; 8717 fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
8741 8718
8742 if (INTEL_INFO(dev)->gen >= 4) { 8719 if (INTEL_GEN(dev_priv) >= 4) {
8743 if (plane_config->tiling) 8720 if (plane_config->tiling)
8744 offset = I915_READ(DSPTILEOFF(plane)); 8721 offset = I915_READ(DSPTILEOFF(plane));
8745 else 8722 else
@@ -8808,8 +8785,7 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc,
8808static bool i9xx_get_pipe_config(struct intel_crtc *crtc, 8785static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8809 struct intel_crtc_state *pipe_config) 8786 struct intel_crtc_state *pipe_config)
8810{ 8787{
8811 struct drm_device *dev = crtc->base.dev; 8788 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
8812 struct drm_i915_private *dev_priv = to_i915(dev);
8813 enum intel_display_power_domain power_domain; 8789 enum intel_display_power_domain power_domain;
8814 uint32_t tmp; 8790 uint32_t tmp;
8815 bool ret; 8791 bool ret;
@@ -8848,7 +8824,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8848 (tmp & PIPECONF_COLOR_RANGE_SELECT)) 8824 (tmp & PIPECONF_COLOR_RANGE_SELECT))
8849 pipe_config->limited_color_range = true; 8825 pipe_config->limited_color_range = true;
8850 8826
8851 if (INTEL_INFO(dev)->gen < 4) 8827 if (INTEL_GEN(dev_priv) < 4)
8852 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE; 8828 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8853 8829
8854 intel_get_pipe_timings(crtc, pipe_config); 8830 intel_get_pipe_timings(crtc, pipe_config);
@@ -8856,7 +8832,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8856 8832
8857 i9xx_get_pfit_config(crtc, pipe_config); 8833 i9xx_get_pfit_config(crtc, pipe_config);
8858 8834
8859 if (INTEL_INFO(dev)->gen >= 4) { 8835 if (INTEL_GEN(dev_priv) >= 4) {
8860 /* No way to read it out on pipes B and C */ 8836 /* No way to read it out on pipes B and C */
8861 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A) 8837 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
8862 tmp = dev_priv->chv_dpll_md[crtc->pipe]; 8838 tmp = dev_priv->chv_dpll_md[crtc->pipe];
@@ -9653,11 +9629,10 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9653 struct intel_link_m_n *m_n, 9629 struct intel_link_m_n *m_n,
9654 struct intel_link_m_n *m2_n2) 9630 struct intel_link_m_n *m2_n2)
9655{ 9631{
9656 struct drm_device *dev = crtc->base.dev; 9632 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
9657 struct drm_i915_private *dev_priv = to_i915(dev);
9658 enum pipe pipe = crtc->pipe; 9633 enum pipe pipe = crtc->pipe;
9659 9634
9660 if (INTEL_INFO(dev)->gen >= 5) { 9635 if (INTEL_GEN(dev_priv) >= 5) {
9661 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder)); 9636 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9662 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder)); 9637 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9663 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder)) 9638 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
@@ -9669,7 +9644,7 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9669 * gen < 8) and if DRRS is supported (to make sure the 9644 * gen < 8) and if DRRS is supported (to make sure the
9670 * registers are not unnecessarily read). 9645 * registers are not unnecessarily read).
9671 */ 9646 */
9672 if (m2_n2 && INTEL_INFO(dev)->gen < 8 && 9647 if (m2_n2 && INTEL_GEN(dev_priv) < 8 &&
9673 crtc->config->has_drrs) { 9648 crtc->config->has_drrs) {
9674 m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder)); 9649 m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9675 m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder)); 9650 m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
@@ -9871,7 +9846,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
9871 9846
9872 fb = &intel_fb->base; 9847 fb = &intel_fb->base;
9873 9848
9874 if (INTEL_INFO(dev)->gen >= 4) { 9849 if (INTEL_GEN(dev_priv) >= 4) {
9875 if (val & DISPPLANE_TILED) { 9850 if (val & DISPPLANE_TILED) {
9876 plane_config->tiling = I915_TILING_X; 9851 plane_config->tiling = I915_TILING_X;
9877 fb->modifier[0] = I915_FORMAT_MOD_X_TILED; 9852 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
@@ -10661,8 +10636,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10661static void haswell_get_ddi_port_state(struct intel_crtc *crtc, 10636static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10662 struct intel_crtc_state *pipe_config) 10637 struct intel_crtc_state *pipe_config)
10663{ 10638{
10664 struct drm_device *dev = crtc->base.dev; 10639 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10665 struct drm_i915_private *dev_priv = to_i915(dev);
10666 struct intel_shared_dpll *pll; 10640 struct intel_shared_dpll *pll;
10667 enum port port; 10641 enum port port;
10668 uint32_t tmp; 10642 uint32_t tmp;
@@ -10689,7 +10663,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10689 * DDI E. So just check whether this pipe is wired to DDI E and whether 10663 * DDI E. So just check whether this pipe is wired to DDI E and whether
10690 * the PCH transcoder is on. 10664 * the PCH transcoder is on.
10691 */ 10665 */
10692 if (INTEL_INFO(dev)->gen < 9 && 10666 if (INTEL_GEN(dev_priv) < 9 &&
10693 (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) { 10667 (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10694 pipe_config->has_pch_encoder = true; 10668 pipe_config->has_pch_encoder = true;
10695 10669
@@ -10704,8 +10678,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10704static bool haswell_get_pipe_config(struct intel_crtc *crtc, 10678static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10705 struct intel_crtc_state *pipe_config) 10679 struct intel_crtc_state *pipe_config)
10706{ 10680{
10707 struct drm_device *dev = crtc->base.dev; 10681 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10708 struct drm_i915_private *dev_priv = to_i915(dev);
10709 enum intel_display_power_domain power_domain; 10682 enum intel_display_power_domain power_domain;
10710 unsigned long power_domain_mask; 10683 unsigned long power_domain_mask;
10711 bool active; 10684 bool active;
@@ -10738,7 +10711,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10738 pipe_config->gamma_mode = 10711 pipe_config->gamma_mode =
10739 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK; 10712 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
10740 10713
10741 if (INTEL_INFO(dev)->gen >= 9) { 10714 if (INTEL_GEN(dev_priv) >= 9) {
10742 skl_init_scalers(dev_priv, crtc, pipe_config); 10715 skl_init_scalers(dev_priv, crtc, pipe_config);
10743 10716
10744 pipe_config->scaler_state.scaler_id = -1; 10717 pipe_config->scaler_state.scaler_id = -1;
@@ -10748,7 +10721,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10748 power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); 10721 power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10749 if (intel_display_power_get_if_enabled(dev_priv, power_domain)) { 10722 if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
10750 power_domain_mask |= BIT(power_domain); 10723 power_domain_mask |= BIT(power_domain);
10751 if (INTEL_INFO(dev)->gen >= 9) 10724 if (INTEL_GEN(dev_priv) >= 9)
10752 skylake_get_pfit_config(crtc, pipe_config); 10725 skylake_get_pfit_config(crtc, pipe_config);
10753 else 10726 else
10754 ironlake_get_pfit_config(crtc, pipe_config); 10727 ironlake_get_pfit_config(crtc, pipe_config);
@@ -10842,16 +10815,9 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
10842 struct drm_device *dev = crtc->dev; 10815 struct drm_device *dev = crtc->dev;
10843 struct drm_i915_private *dev_priv = to_i915(dev); 10816 struct drm_i915_private *dev_priv = to_i915(dev);
10844 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 10817 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10845 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
10846 const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
10847 const struct skl_plane_wm *p_wm =
10848 &cstate->wm.skl.optimal.planes[PLANE_CURSOR];
10849 int pipe = intel_crtc->pipe; 10818 int pipe = intel_crtc->pipe;
10850 uint32_t cntl = 0; 10819 uint32_t cntl = 0;
10851 10820
10852 if (INTEL_GEN(dev_priv) >= 9 && wm->dirty_pipes & drm_crtc_mask(crtc))
10853 skl_write_cursor_wm(intel_crtc, p_wm, &wm->ddb);
10854
10855 if (plane_state && plane_state->base.visible) { 10821 if (plane_state && plane_state->base.visible) {
10856 cntl = MCURSOR_GAMMA_ENABLE; 10822 cntl = MCURSOR_GAMMA_ENABLE;
10857 switch (plane_state->base.crtc_w) { 10823 switch (plane_state->base.crtc_w) {
@@ -10873,7 +10839,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
10873 if (HAS_DDI(dev_priv)) 10839 if (HAS_DDI(dev_priv))
10874 cntl |= CURSOR_PIPE_CSC_ENABLE; 10840 cntl |= CURSOR_PIPE_CSC_ENABLE;
10875 10841
10876 if (plane_state->base.rotation == DRM_ROTATE_180) 10842 if (plane_state->base.rotation & DRM_ROTATE_180)
10877 cntl |= CURSOR_ROTATE_180; 10843 cntl |= CURSOR_ROTATE_180;
10878 } 10844 }
10879 10845
@@ -10919,7 +10885,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
10919 10885
10920 /* ILK+ do this automagically */ 10886 /* ILK+ do this automagically */
10921 if (HAS_GMCH_DISPLAY(dev_priv) && 10887 if (HAS_GMCH_DISPLAY(dev_priv) &&
10922 plane_state->base.rotation == DRM_ROTATE_180) { 10888 plane_state->base.rotation & DRM_ROTATE_180) {
10923 base += (plane_state->base.crtc_h * 10889 base += (plane_state->base.crtc_h *
10924 plane_state->base.crtc_w - 1) * 4; 10890 plane_state->base.crtc_w - 1) * 4;
10925 } 10891 }
@@ -12062,6 +12028,7 @@ static void intel_mmio_flip_work_func(struct work_struct *w)
12062 to_intel_framebuffer(crtc->base.primary->fb); 12028 to_intel_framebuffer(crtc->base.primary->fb);
12063 struct drm_i915_gem_object *obj = intel_fb->obj; 12029 struct drm_i915_gem_object *obj = intel_fb->obj;
12064 12030
12031 i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
12065 WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0); 12032 WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0);
12066 12033
12067 intel_pipe_update_start(crtc); 12034 intel_pipe_update_start(crtc);
@@ -12186,7 +12153,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
12186 * TILEOFF/LINOFF registers can't be changed via MI display flips. 12153 * TILEOFF/LINOFF registers can't be changed via MI display flips.
12187 * Note that pitch changes could also affect these register. 12154 * Note that pitch changes could also affect these register.
12188 */ 12155 */
12189 if (INTEL_INFO(dev)->gen > 3 && 12156 if (INTEL_GEN(dev_priv) > 3 &&
12190 (fb->offsets[0] != crtc->primary->fb->offsets[0] || 12157 (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
12191 fb->pitches[0] != crtc->primary->fb->pitches[0])) 12158 fb->pitches[0] != crtc->primary->fb->pitches[0]))
12192 return -EINVAL; 12159 return -EINVAL;
@@ -12261,7 +12228,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
12261 engine = NULL; 12228 engine = NULL;
12262 } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { 12229 } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
12263 engine = dev_priv->engine[BCS]; 12230 engine = dev_priv->engine[BCS];
12264 } else if (INTEL_INFO(dev)->gen >= 7) { 12231 } else if (INTEL_GEN(dev_priv) >= 7) {
12265 engine = i915_gem_object_last_write_engine(obj); 12232 engine = i915_gem_object_last_write_engine(obj);
12266 if (engine == NULL || engine->id != RCS) 12233 if (engine == NULL || engine->id != RCS)
12267 engine = dev_priv->engine[BCS]; 12234 engine = dev_priv->engine[BCS];
@@ -12518,7 +12485,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
12518 12485
12519 /* Pre-gen9 platforms need two-step watermark updates */ 12486 /* Pre-gen9 platforms need two-step watermark updates */
12520 if ((pipe_config->update_wm_pre || pipe_config->update_wm_post) && 12487 if ((pipe_config->update_wm_pre || pipe_config->update_wm_post) &&
12521 INTEL_INFO(dev)->gen < 9 && dev_priv->display.optimize_watermarks) 12488 INTEL_GEN(dev_priv) < 9 && dev_priv->display.optimize_watermarks)
12522 to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true; 12489 to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true;
12523 12490
12524 if (visible || was_visible) 12491 if (visible || was_visible)
@@ -12623,7 +12590,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
12623 * old state and the new state. We can program these 12590 * old state and the new state. We can program these
12624 * immediately. 12591 * immediately.
12625 */ 12592 */
12626 ret = dev_priv->display.compute_intermediate_wm(crtc->dev, 12593 ret = dev_priv->display.compute_intermediate_wm(dev,
12627 intel_crtc, 12594 intel_crtc,
12628 pipe_config); 12595 pipe_config);
12629 if (ret) { 12596 if (ret) {
@@ -12635,7 +12602,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
12635 pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal; 12602 pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
12636 } 12603 }
12637 12604
12638 if (INTEL_INFO(dev)->gen >= 9) { 12605 if (INTEL_GEN(dev_priv) >= 9) {
12639 if (mode_changed) 12606 if (mode_changed)
12640 ret = skl_update_scaler_crtc(pipe_config); 12607 ret = skl_update_scaler_crtc(pipe_config);
12641 12608
@@ -12748,6 +12715,16 @@ static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12748 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags); 12715 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
12749} 12716}
12750 12717
12718static inline void
12719intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
12720 unsigned int lane_count, struct intel_link_m_n *m_n)
12721{
12722 DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12723 id, lane_count,
12724 m_n->gmch_m, m_n->gmch_n,
12725 m_n->link_m, m_n->link_n, m_n->tu);
12726}
12727
12751static void intel_dump_pipe_config(struct intel_crtc *crtc, 12728static void intel_dump_pipe_config(struct intel_crtc *crtc,
12752 struct intel_crtc_state *pipe_config, 12729 struct intel_crtc_state *pipe_config,
12753 const char *context) 12730 const char *context)
@@ -12759,61 +12736,58 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
12759 struct intel_plane_state *state; 12736 struct intel_plane_state *state;
12760 struct drm_framebuffer *fb; 12737 struct drm_framebuffer *fb;
12761 12738
12762 DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n", 12739 DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
12763 crtc->base.base.id, crtc->base.name, 12740 crtc->base.base.id, crtc->base.name, context);
12764 context, pipe_config, pipe_name(crtc->pipe));
12765 12741
12766 DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder)); 12742 DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
12767 DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n", 12743 transcoder_name(pipe_config->cpu_transcoder),
12768 pipe_config->pipe_bpp, pipe_config->dither); 12744 pipe_config->pipe_bpp, pipe_config->dither);
12769 DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", 12745
12770 pipe_config->has_pch_encoder, 12746 if (pipe_config->has_pch_encoder)
12771 pipe_config->fdi_lanes, 12747 intel_dump_m_n_config(pipe_config, "fdi",
12772 pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n, 12748 pipe_config->fdi_lanes,
12773 pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n, 12749 &pipe_config->fdi_m_n);
12774 pipe_config->fdi_m_n.tu); 12750
12775 DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", 12751 if (intel_crtc_has_dp_encoder(pipe_config)) {
12776 intel_crtc_has_dp_encoder(pipe_config), 12752 intel_dump_m_n_config(pipe_config, "dp m_n",
12777 pipe_config->lane_count, 12753 pipe_config->lane_count, &pipe_config->dp_m_n);
12778 pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n, 12754 if (pipe_config->has_drrs)
12779 pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n, 12755 intel_dump_m_n_config(pipe_config, "dp m2_n2",
12780 pipe_config->dp_m_n.tu); 12756 pipe_config->lane_count,
12781 12757 &pipe_config->dp_m2_n2);
12782 DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", 12758 }
12783 intel_crtc_has_dp_encoder(pipe_config),
12784 pipe_config->lane_count,
12785 pipe_config->dp_m2_n2.gmch_m,
12786 pipe_config->dp_m2_n2.gmch_n,
12787 pipe_config->dp_m2_n2.link_m,
12788 pipe_config->dp_m2_n2.link_n,
12789 pipe_config->dp_m2_n2.tu);
12790 12759
12791 DRM_DEBUG_KMS("audio: %i, infoframes: %i\n", 12760 DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
12792 pipe_config->has_audio, 12761 pipe_config->has_audio, pipe_config->has_infoframe);
12793 pipe_config->has_infoframe);
12794 12762
12795 DRM_DEBUG_KMS("requested mode:\n"); 12763 DRM_DEBUG_KMS("requested mode:\n");
12796 drm_mode_debug_printmodeline(&pipe_config->base.mode); 12764 drm_mode_debug_printmodeline(&pipe_config->base.mode);
12797 DRM_DEBUG_KMS("adjusted mode:\n"); 12765 DRM_DEBUG_KMS("adjusted mode:\n");
12798 drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode); 12766 drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12799 intel_dump_crtc_timings(&pipe_config->base.adjusted_mode); 12767 intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12800 DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock); 12768 DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d\n",
12801 DRM_DEBUG_KMS("pipe src size: %dx%d\n", 12769 pipe_config->port_clock,
12802 pipe_config->pipe_src_w, pipe_config->pipe_src_h); 12770 pipe_config->pipe_src_w, pipe_config->pipe_src_h);
12803 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n", 12771
12804 crtc->num_scalers, 12772 if (INTEL_GEN(dev_priv) >= 9)
12805 pipe_config->scaler_state.scaler_users, 12773 DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12806 pipe_config->scaler_state.scaler_id); 12774 crtc->num_scalers,
12807 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n", 12775 pipe_config->scaler_state.scaler_users,
12808 pipe_config->gmch_pfit.control, 12776 pipe_config->scaler_state.scaler_id);
12809 pipe_config->gmch_pfit.pgm_ratios, 12777
12810 pipe_config->gmch_pfit.lvds_border_bits); 12778 if (HAS_GMCH_DISPLAY(dev_priv))
12811 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n", 12779 DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12812 pipe_config->pch_pfit.pos, 12780 pipe_config->gmch_pfit.control,
12813 pipe_config->pch_pfit.size, 12781 pipe_config->gmch_pfit.pgm_ratios,
12814 pipe_config->pch_pfit.enabled ? "enabled" : "disabled"); 12782 pipe_config->gmch_pfit.lvds_border_bits);
12815 DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled); 12783 else
12816 DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide); 12784 DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
12785 pipe_config->pch_pfit.pos,
12786 pipe_config->pch_pfit.size,
12787 enableddisabled(pipe_config->pch_pfit.enabled));
12788
12789 DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
12790 pipe_config->ips_enabled, pipe_config->double_wide);
12817 12791
12818 if (IS_BROXTON(dev_priv)) { 12792 if (IS_BROXTON(dev_priv)) {
12819 DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x," 12793 DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
@@ -12864,20 +12838,20 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
12864 continue; 12838 continue;
12865 } 12839 }
12866 12840
12867 DRM_DEBUG_KMS("[PLANE:%d:%s] enabled", 12841 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
12868 plane->base.id, plane->name); 12842 plane->base.id, plane->name,
12869 DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s",
12870 fb->base.id, fb->width, fb->height, 12843 fb->base.id, fb->width, fb->height,
12871 drm_get_format_name(fb->pixel_format, &format_name)); 12844 drm_get_format_name(fb->pixel_format, &format_name));
12872 DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n", 12845 if (INTEL_GEN(dev_priv) >= 9)
12873 state->scaler_id, 12846 DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
12874 state->base.src.x1 >> 16, 12847 state->scaler_id,
12875 state->base.src.y1 >> 16, 12848 state->base.src.x1 >> 16,
12876 drm_rect_width(&state->base.src) >> 16, 12849 state->base.src.y1 >> 16,
12877 drm_rect_height(&state->base.src) >> 16, 12850 drm_rect_width(&state->base.src) >> 16,
12878 state->base.dst.x1, state->base.dst.y1, 12851 drm_rect_height(&state->base.src) >> 16,
12879 drm_rect_width(&state->base.dst), 12852 state->base.dst.x1, state->base.dst.y1,
12880 drm_rect_height(&state->base.dst)); 12853 drm_rect_width(&state->base.dst),
12854 drm_rect_height(&state->base.dst));
12881 } 12855 }
12882} 12856}
12883 12857
@@ -13192,12 +13166,11 @@ intel_compare_link_m_n(const struct intel_link_m_n *m_n,
13192} 13166}
13193 13167
13194static bool 13168static bool
13195intel_pipe_config_compare(struct drm_device *dev, 13169intel_pipe_config_compare(struct drm_i915_private *dev_priv,
13196 struct intel_crtc_state *current_config, 13170 struct intel_crtc_state *current_config,
13197 struct intel_crtc_state *pipe_config, 13171 struct intel_crtc_state *pipe_config,
13198 bool adjust) 13172 bool adjust)
13199{ 13173{
13200 struct drm_i915_private *dev_priv = to_i915(dev);
13201 bool ret = true; 13174 bool ret = true;
13202 13175
13203#define INTEL_ERR_OR_DBG_KMS(fmt, ...) \ 13176#define INTEL_ERR_OR_DBG_KMS(fmt, ...) \
@@ -13317,7 +13290,7 @@ intel_pipe_config_compare(struct drm_device *dev,
13317 PIPE_CONF_CHECK_I(lane_count); 13290 PIPE_CONF_CHECK_I(lane_count);
13318 PIPE_CONF_CHECK_X(lane_lat_optim_mask); 13291 PIPE_CONF_CHECK_X(lane_lat_optim_mask);
13319 13292
13320 if (INTEL_INFO(dev)->gen < 8) { 13293 if (INTEL_GEN(dev_priv) < 8) {
13321 PIPE_CONF_CHECK_M_N(dp_m_n); 13294 PIPE_CONF_CHECK_M_N(dp_m_n);
13322 13295
13323 if (current_config->has_drrs) 13296 if (current_config->has_drrs)
@@ -13366,7 +13339,7 @@ intel_pipe_config_compare(struct drm_device *dev,
13366 13339
13367 PIPE_CONF_CHECK_X(gmch_pfit.control); 13340 PIPE_CONF_CHECK_X(gmch_pfit.control);
13368 /* pfit ratios are autocomputed by the hw on gen4+ */ 13341 /* pfit ratios are autocomputed by the hw on gen4+ */
13369 if (INTEL_INFO(dev)->gen < 4) 13342 if (INTEL_GEN(dev_priv) < 4)
13370 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios); 13343 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
13371 PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits); 13344 PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
13372 13345
@@ -13441,8 +13414,7 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
13441static void verify_wm_state(struct drm_crtc *crtc, 13414static void verify_wm_state(struct drm_crtc *crtc,
13442 struct drm_crtc_state *new_state) 13415 struct drm_crtc_state *new_state)
13443{ 13416{
13444 struct drm_device *dev = crtc->dev; 13417 struct drm_i915_private *dev_priv = to_i915(crtc->dev);
13445 struct drm_i915_private *dev_priv = to_i915(dev);
13446 struct skl_ddb_allocation hw_ddb, *sw_ddb; 13418 struct skl_ddb_allocation hw_ddb, *sw_ddb;
13447 struct skl_pipe_wm hw_wm, *sw_wm; 13419 struct skl_pipe_wm hw_wm, *sw_wm;
13448 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm; 13420 struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
@@ -13451,7 +13423,7 @@ static void verify_wm_state(struct drm_crtc *crtc,
13451 const enum pipe pipe = intel_crtc->pipe; 13423 const enum pipe pipe = intel_crtc->pipe;
13452 int plane, level, max_level = ilk_wm_max_level(dev_priv); 13424 int plane, level, max_level = ilk_wm_max_level(dev_priv);
13453 13425
13454 if (INTEL_INFO(dev)->gen < 9 || !new_state->active) 13426 if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
13455 return; 13427 return;
13456 13428
13457 skl_pipe_wm_get_hw_state(crtc, &hw_wm); 13429 skl_pipe_wm_get_hw_state(crtc, &hw_wm);
@@ -13557,11 +13529,15 @@ static void verify_wm_state(struct drm_crtc *crtc,
13557} 13529}
13558 13530
13559static void 13531static void
13560verify_connector_state(struct drm_device *dev, struct drm_crtc *crtc) 13532verify_connector_state(struct drm_device *dev,
13533 struct drm_atomic_state *state,
13534 struct drm_crtc *crtc)
13561{ 13535{
13562 struct drm_connector *connector; 13536 struct drm_connector *connector;
13537 struct drm_connector_state *old_conn_state;
13538 int i;
13563 13539
13564 drm_for_each_connector(connector, dev) { 13540 for_each_connector_in_state(state, connector, old_conn_state, i) {
13565 struct drm_encoder *encoder = connector->encoder; 13541 struct drm_encoder *encoder = connector->encoder;
13566 struct drm_connector_state *state = connector->state; 13542 struct drm_connector_state *state = connector->state;
13567 13543
@@ -13676,7 +13652,7 @@ verify_crtc_state(struct drm_crtc *crtc,
13676 intel_pipe_config_sanity_check(dev_priv, pipe_config); 13652 intel_pipe_config_sanity_check(dev_priv, pipe_config);
13677 13653
13678 sw_config = to_intel_crtc_state(crtc->state); 13654 sw_config = to_intel_crtc_state(crtc->state);
13679 if (!intel_pipe_config_compare(dev, sw_config, 13655 if (!intel_pipe_config_compare(dev_priv, sw_config,
13680 pipe_config, false)) { 13656 pipe_config, false)) {
13681 I915_STATE_WARN(1, "pipe state doesn't match!\n"); 13657 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13682 intel_dump_pipe_config(intel_crtc, pipe_config, 13658 intel_dump_pipe_config(intel_crtc, pipe_config,
@@ -13769,15 +13745,16 @@ verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
13769 13745
13770static void 13746static void
13771intel_modeset_verify_crtc(struct drm_crtc *crtc, 13747intel_modeset_verify_crtc(struct drm_crtc *crtc,
13772 struct drm_crtc_state *old_state, 13748 struct drm_atomic_state *state,
13773 struct drm_crtc_state *new_state) 13749 struct drm_crtc_state *old_state,
13750 struct drm_crtc_state *new_state)
13774{ 13751{
13775 if (!needs_modeset(new_state) && 13752 if (!needs_modeset(new_state) &&
13776 !to_intel_crtc_state(new_state)->update_pipe) 13753 !to_intel_crtc_state(new_state)->update_pipe)
13777 return; 13754 return;
13778 13755
13779 verify_wm_state(crtc, new_state); 13756 verify_wm_state(crtc, new_state);
13780 verify_connector_state(crtc->dev, crtc); 13757 verify_connector_state(crtc->dev, state, crtc);
13781 verify_crtc_state(crtc, old_state, new_state); 13758 verify_crtc_state(crtc, old_state, new_state);
13782 verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state); 13759 verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
13783} 13760}
@@ -13793,10 +13770,11 @@ verify_disabled_dpll_state(struct drm_device *dev)
13793} 13770}
13794 13771
13795static void 13772static void
13796intel_modeset_verify_disabled(struct drm_device *dev) 13773intel_modeset_verify_disabled(struct drm_device *dev,
13774 struct drm_atomic_state *state)
13797{ 13775{
13798 verify_encoder_state(dev); 13776 verify_encoder_state(dev);
13799 verify_connector_state(dev, NULL); 13777 verify_connector_state(dev, state, NULL);
13800 verify_disabled_dpll_state(dev); 13778 verify_disabled_dpll_state(dev);
13801} 13779}
13802 13780
@@ -14094,7 +14072,7 @@ static int intel_atomic_check(struct drm_device *dev,
14094 } 14072 }
14095 14073
14096 if (i915.fastboot && 14074 if (i915.fastboot &&
14097 intel_pipe_config_compare(dev, 14075 intel_pipe_config_compare(dev_priv,
14098 to_intel_crtc_state(crtc->state), 14076 to_intel_crtc_state(crtc->state),
14099 pipe_config, true)) { 14077 pipe_config, true)) {
14100 crtc_state->mode_changed = false; 14078 crtc_state->mode_changed = false;
@@ -14294,6 +14272,14 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
14294 unsigned int updated = 0; 14272 unsigned int updated = 0;
14295 bool progress; 14273 bool progress;
14296 enum pipe pipe; 14274 enum pipe pipe;
14275 int i;
14276
14277 const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {};
14278
14279 for_each_crtc_in_state(state, crtc, old_crtc_state, i)
14280 /* ignore allocations for crtc's that have been turned off. */
14281 if (crtc->state->active)
14282 entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
14297 14283
14298 /* 14284 /*
14299 * Whenever the number of active pipes changes, we need to make sure we 14285 * Whenever the number of active pipes changes, we need to make sure we
@@ -14302,7 +14288,6 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
14302 * cause pipe underruns and other bad stuff. 14288 * cause pipe underruns and other bad stuff.
14303 */ 14289 */
14304 do { 14290 do {
14305 int i;
14306 progress = false; 14291 progress = false;
14307 14292
14308 for_each_crtc_in_state(state, crtc, old_crtc_state, i) { 14293 for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
@@ -14313,12 +14298,14 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
14313 cstate = to_intel_crtc_state(crtc->state); 14298 cstate = to_intel_crtc_state(crtc->state);
14314 pipe = intel_crtc->pipe; 14299 pipe = intel_crtc->pipe;
14315 14300
14316 if (updated & cmask || !crtc->state->active) 14301 if (updated & cmask || !cstate->base.active)
14317 continue; 14302 continue;
14318 if (skl_ddb_allocation_overlaps(state, intel_crtc)) 14303
14304 if (skl_ddb_allocation_overlaps(entries, &cstate->wm.skl.ddb, i))
14319 continue; 14305 continue;
14320 14306
14321 updated |= cmask; 14307 updated |= cmask;
14308 entries[i] = &cstate->wm.skl.ddb;
14322 14309
14323 /* 14310 /*
14324 * If this is an already active pipe, it's DDB changed, 14311 * If this is an already active pipe, it's DDB changed,
@@ -14327,7 +14314,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
14327 * new ddb allocation to take effect. 14314 * new ddb allocation to take effect.
14328 */ 14315 */
14329 if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb, 14316 if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
14330 &intel_crtc->hw_ddb) && 14317 &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
14331 !crtc->state->active_changed && 14318 !crtc->state->active_changed &&
14332 intel_state->wm_results.dirty_pipes != updated) 14319 intel_state->wm_results.dirty_pipes != updated)
14333 vbl_wait = true; 14320 vbl_wait = true;
@@ -14358,14 +14345,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
14358 14345
14359 drm_atomic_helper_wait_for_dependencies(state); 14346 drm_atomic_helper_wait_for_dependencies(state);
14360 14347
14361 if (intel_state->modeset) { 14348 if (intel_state->modeset)
14362 memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
14363 sizeof(intel_state->min_pixclk));
14364 dev_priv->active_crtcs = intel_state->active_crtcs;
14365 dev_priv->atomic_cdclk_freq = intel_state->cdclk;
14366
14367 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET); 14349 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
14368 }
14369 14350
14370 for_each_crtc_in_state(state, crtc, old_crtc_state, i) { 14351 for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
14371 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 14352 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -14398,8 +14379,17 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
14398 intel_check_cpu_fifo_underruns(dev_priv); 14379 intel_check_cpu_fifo_underruns(dev_priv);
14399 intel_check_pch_fifo_underruns(dev_priv); 14380 intel_check_pch_fifo_underruns(dev_priv);
14400 14381
14401 if (!crtc->state->active) 14382 if (!crtc->state->active) {
14402 intel_update_watermarks(intel_crtc); 14383 /*
14384 * Make sure we don't call initial_watermarks
14385 * for ILK-style watermark updates.
14386 */
14387 if (dev_priv->display.atomic_update_watermarks)
14388 dev_priv->display.initial_watermarks(intel_state,
14389 to_intel_crtc_state(crtc->state));
14390 else
14391 intel_update_watermarks(intel_crtc);
14392 }
14403 } 14393 }
14404 } 14394 }
14405 14395
@@ -14422,7 +14412,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
14422 if (!intel_can_enable_sagv(state)) 14412 if (!intel_can_enable_sagv(state))
14423 intel_disable_sagv(dev_priv); 14413 intel_disable_sagv(dev_priv);
14424 14414
14425 intel_modeset_verify_disabled(dev); 14415 intel_modeset_verify_disabled(dev, state);
14426 } 14416 }
14427 14417
14428 /* Complete the events for pipes that have now been disabled */ 14418 /* Complete the events for pipes that have now been disabled */
@@ -14465,7 +14455,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
14465 intel_cstate = to_intel_crtc_state(crtc->state); 14455 intel_cstate = to_intel_crtc_state(crtc->state);
14466 14456
14467 if (dev_priv->display.optimize_watermarks) 14457 if (dev_priv->display.optimize_watermarks)
14468 dev_priv->display.optimize_watermarks(intel_cstate); 14458 dev_priv->display.optimize_watermarks(intel_state,
14459 intel_cstate);
14469 } 14460 }
14470 14461
14471 for_each_crtc_in_state(state, crtc, old_crtc_state, i) { 14462 for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
@@ -14474,7 +14465,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
14474 if (put_domains[i]) 14465 if (put_domains[i])
14475 modeset_put_power_domains(dev_priv, put_domains[i]); 14466 modeset_put_power_domains(dev_priv, put_domains[i]);
14476 14467
14477 intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state); 14468 intel_modeset_verify_crtc(crtc, state, old_crtc_state, crtc->state);
14478 } 14469 }
14479 14470
14480 if (intel_state->modeset && intel_can_enable_sagv(state)) 14471 if (intel_state->modeset && intel_can_enable_sagv(state))
@@ -14557,10 +14548,6 @@ static void intel_atomic_track_fbs(struct drm_atomic_state *state)
14557 * This function commits a top-level state object that has been validated 14548 * This function commits a top-level state object that has been validated
14558 * with drm_atomic_helper_check(). 14549 * with drm_atomic_helper_check().
14559 * 14550 *
14560 * FIXME: Atomic modeset support for i915 is not yet complete. At the moment
14561 * nonblocking commits are only safe for pure plane updates. Everything else
14562 * should work though.
14563 *
14564 * RETURNS 14551 * RETURNS
14565 * Zero for success or -errno. 14552 * Zero for success or -errno.
14566 */ 14553 */
@@ -14572,11 +14559,6 @@ static int intel_atomic_commit(struct drm_device *dev,
14572 struct drm_i915_private *dev_priv = to_i915(dev); 14559 struct drm_i915_private *dev_priv = to_i915(dev);
14573 int ret = 0; 14560 int ret = 0;
14574 14561
14575 if (intel_state->modeset && nonblock) {
14576 DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n");
14577 return -EINVAL;
14578 }
14579
14580 ret = drm_atomic_helper_setup_commit(state, nonblock); 14562 ret = drm_atomic_helper_setup_commit(state, nonblock);
14581 if (ret) 14563 if (ret)
14582 return ret; 14564 return ret;
@@ -14594,10 +14576,16 @@ static int intel_atomic_commit(struct drm_device *dev,
14594 14576
14595 drm_atomic_helper_swap_state(state, true); 14577 drm_atomic_helper_swap_state(state, true);
14596 dev_priv->wm.distrust_bios_wm = false; 14578 dev_priv->wm.distrust_bios_wm = false;
14597 dev_priv->wm.skl_results = intel_state->wm_results;
14598 intel_shared_dpll_commit(state); 14579 intel_shared_dpll_commit(state);
14599 intel_atomic_track_fbs(state); 14580 intel_atomic_track_fbs(state);
14600 14581
14582 if (intel_state->modeset) {
14583 memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
14584 sizeof(intel_state->min_pixclk));
14585 dev_priv->active_crtcs = intel_state->active_crtcs;
14586 dev_priv->atomic_cdclk_freq = intel_state->cdclk;
14587 }
14588
14601 drm_atomic_state_get(state); 14589 drm_atomic_state_get(state);
14602 INIT_WORK(&state->commit_work, 14590 INIT_WORK(&state->commit_work,
14603 nonblock ? intel_atomic_commit_work : NULL); 14591 nonblock ? intel_atomic_commit_work : NULL);
@@ -14720,8 +14708,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
14720{ 14708{
14721 struct intel_atomic_state *intel_state = 14709 struct intel_atomic_state *intel_state =
14722 to_intel_atomic_state(new_state->state); 14710 to_intel_atomic_state(new_state->state);
14723 struct drm_device *dev = plane->dev; 14711 struct drm_i915_private *dev_priv = to_i915(plane->dev);
14724 struct drm_i915_private *dev_priv = to_i915(dev);
14725 struct drm_framebuffer *fb = new_state->fb; 14712 struct drm_framebuffer *fb = new_state->fb;
14726 struct drm_i915_gem_object *obj = intel_fb_obj(fb); 14713 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14727 struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); 14714 struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
@@ -14775,10 +14762,12 @@ intel_prepare_plane_fb(struct drm_plane *plane,
14775 GFP_KERNEL); 14762 GFP_KERNEL);
14776 if (ret < 0) 14763 if (ret < 0)
14777 return ret; 14764 return ret;
14765
14766 i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
14778 } 14767 }
14779 14768
14780 if (plane->type == DRM_PLANE_TYPE_CURSOR && 14769 if (plane->type == DRM_PLANE_TYPE_CURSOR &&
14781 INTEL_INFO(dev)->cursor_needs_physical) { 14770 INTEL_INFO(dev_priv)->cursor_needs_physical) {
14782 int align = IS_I830(dev_priv) ? 16 * 1024 : 256; 14771 int align = IS_I830(dev_priv) ? 16 * 1024 : 256;
14783 ret = i915_gem_object_attach_phys(obj, align); 14772 ret = i915_gem_object_attach_phys(obj, align);
14784 if (ret) { 14773 if (ret) {
@@ -14811,7 +14800,7 @@ void
14811intel_cleanup_plane_fb(struct drm_plane *plane, 14800intel_cleanup_plane_fb(struct drm_plane *plane,
14812 struct drm_plane_state *old_state) 14801 struct drm_plane_state *old_state)
14813{ 14802{
14814 struct drm_device *dev = plane->dev; 14803 struct drm_i915_private *dev_priv = to_i915(plane->dev);
14815 struct intel_plane_state *old_intel_state; 14804 struct intel_plane_state *old_intel_state;
14816 struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); 14805 struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
14817 struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); 14806 struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
@@ -14822,7 +14811,7 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
14822 return; 14811 return;
14823 14812
14824 if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR || 14813 if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR ||
14825 !INTEL_INFO(dev)->cursor_needs_physical)) 14814 !INTEL_INFO(dev_priv)->cursor_needs_physical))
14826 intel_unpin_fb_obj(old_state->fb, old_state->rotation); 14815 intel_unpin_fb_obj(old_state->fb, old_state->rotation);
14827} 14816}
14828 14817
@@ -14900,30 +14889,32 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
14900 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 14889 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
14901 struct intel_crtc_state *intel_cstate = 14890 struct intel_crtc_state *intel_cstate =
14902 to_intel_crtc_state(crtc->state); 14891 to_intel_crtc_state(crtc->state);
14903 struct intel_crtc_state *old_intel_state = 14892 struct intel_crtc_state *old_intel_cstate =
14904 to_intel_crtc_state(old_crtc_state); 14893 to_intel_crtc_state(old_crtc_state);
14894 struct intel_atomic_state *old_intel_state =
14895 to_intel_atomic_state(old_crtc_state->state);
14905 bool modeset = needs_modeset(crtc->state); 14896 bool modeset = needs_modeset(crtc->state);
14906 enum pipe pipe = intel_crtc->pipe;
14907 14897
14908 /* Perform vblank evasion around commit operation */ 14898 /* Perform vblank evasion around commit operation */
14909 intel_pipe_update_start(intel_crtc); 14899 intel_pipe_update_start(intel_crtc);
14910 14900
14911 if (modeset) 14901 if (modeset)
14912 return; 14902 goto out;
14913 14903
14914 if (crtc->state->color_mgmt_changed || to_intel_crtc_state(crtc->state)->update_pipe) { 14904 if (crtc->state->color_mgmt_changed || to_intel_crtc_state(crtc->state)->update_pipe) {
14915 intel_color_set_csc(crtc->state); 14905 intel_color_set_csc(crtc->state);
14916 intel_color_load_luts(crtc->state); 14906 intel_color_load_luts(crtc->state);
14917 } 14907 }
14918 14908
14919 if (intel_cstate->update_pipe) { 14909 if (intel_cstate->update_pipe)
14920 intel_update_pipe_config(intel_crtc, old_intel_state); 14910 intel_update_pipe_config(intel_crtc, old_intel_cstate);
14921 } else if (INTEL_GEN(dev_priv) >= 9) { 14911 else if (INTEL_GEN(dev_priv) >= 9)
14922 skl_detach_scalers(intel_crtc); 14912 skl_detach_scalers(intel_crtc);
14923 14913
14924 I915_WRITE(PIPE_WM_LINETIME(pipe), 14914out:
14925 intel_cstate->wm.skl.optimal.linetime); 14915 if (dev_priv->display.atomic_update_watermarks)
14926 } 14916 dev_priv->display.atomic_update_watermarks(old_intel_state,
14917 intel_cstate);
14927} 14918}
14928 14919
14929static void intel_finish_crtc_commit(struct drm_crtc *crtc, 14920static void intel_finish_crtc_commit(struct drm_crtc *crtc,
@@ -14989,11 +14980,16 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
14989 state->scaler_id = -1; 14980 state->scaler_id = -1;
14990 } 14981 }
14991 primary->pipe = pipe; 14982 primary->pipe = pipe;
14992 primary->plane = pipe; 14983 /*
14984 * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
14985 * port is hooked to pipe B. Hence we want plane A feeding pipe B.
14986 */
14987 if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14988 primary->plane = (enum plane) !pipe;
14989 else
14990 primary->plane = (enum plane) pipe;
14993 primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe); 14991 primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
14994 primary->check_plane = intel_check_primary_plane; 14992 primary->check_plane = intel_check_primary_plane;
14995 if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
14996 primary->plane = !pipe;
14997 14993
14998 if (INTEL_GEN(dev_priv) >= 9) { 14994 if (INTEL_GEN(dev_priv) >= 9) {
14999 intel_primary_formats = skl_primary_formats; 14995 intel_primary_formats = skl_primary_formats;
@@ -15046,6 +15042,10 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
15046 supported_rotations = 15042 supported_rotations =
15047 DRM_ROTATE_0 | DRM_ROTATE_90 | 15043 DRM_ROTATE_0 | DRM_ROTATE_90 |
15048 DRM_ROTATE_180 | DRM_ROTATE_270; 15044 DRM_ROTATE_180 | DRM_ROTATE_270;
15045 } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
15046 supported_rotations =
15047 DRM_ROTATE_0 | DRM_ROTATE_180 |
15048 DRM_REFLECT_X;
15049 } else if (INTEL_GEN(dev_priv) >= 4) { 15049 } else if (INTEL_GEN(dev_priv) >= 4) {
15050 supported_rotations = 15050 supported_rotations =
15051 DRM_ROTATE_0 | DRM_ROTATE_180; 15051 DRM_ROTATE_0 | DRM_ROTATE_180;
@@ -15147,13 +15147,13 @@ intel_update_cursor_plane(struct drm_plane *plane,
15147{ 15147{
15148 struct drm_crtc *crtc = crtc_state->base.crtc; 15148 struct drm_crtc *crtc = crtc_state->base.crtc;
15149 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 15149 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
15150 struct drm_device *dev = plane->dev; 15150 struct drm_i915_private *dev_priv = to_i915(plane->dev);
15151 struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb); 15151 struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
15152 uint32_t addr; 15152 uint32_t addr;
15153 15153
15154 if (!obj) 15154 if (!obj)
15155 addr = 0; 15155 addr = 0;
15156 else if (!INTEL_INFO(dev)->cursor_needs_physical) 15156 else if (!INTEL_INFO(dev_priv)->cursor_needs_physical)
15157 addr = i915_gem_object_ggtt_offset(obj, NULL); 15157 addr = i915_gem_object_ggtt_offset(obj, NULL);
15158 else 15158 else
15159 addr = obj->phys_handle->busaddr; 15159 addr = obj->phys_handle->busaddr;
@@ -15280,14 +15280,14 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15280 struct intel_plane *plane; 15280 struct intel_plane *plane;
15281 15281
15282 plane = intel_sprite_plane_create(dev_priv, pipe, sprite); 15282 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
15283 if (!plane) { 15283 if (IS_ERR(plane)) {
15284 ret = PTR_ERR(plane); 15284 ret = PTR_ERR(plane);
15285 goto fail; 15285 goto fail;
15286 } 15286 }
15287 } 15287 }
15288 15288
15289 cursor = intel_cursor_plane_create(dev_priv, pipe); 15289 cursor = intel_cursor_plane_create(dev_priv, pipe);
15290 if (!cursor) { 15290 if (IS_ERR(cursor)) {
15291 ret = PTR_ERR(cursor); 15291 ret = PTR_ERR(cursor);
15292 goto fail; 15292 goto fail;
15293 } 15293 }
@@ -15299,16 +15299,8 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
15299 if (ret) 15299 if (ret)
15300 goto fail; 15300 goto fail;
15301 15301
15302 /*
15303 * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
15304 * is hooked to pipe B. Hence we want plane A feeding pipe B.
15305 */
15306 intel_crtc->pipe = pipe; 15302 intel_crtc->pipe = pipe;
15307 intel_crtc->plane = (enum plane) pipe; 15303 intel_crtc->plane = primary->plane;
15308 if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) {
15309 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
15310 intel_crtc->plane = !pipe;
15311 }
15312 15304
15313 intel_crtc->cursor_base = ~0; 15305 intel_crtc->cursor_base = ~0;
15314 intel_crtc->cursor_cntl = ~0; 15306 intel_crtc->cursor_cntl = ~0;
@@ -15401,11 +15393,9 @@ static bool has_edp_a(struct drm_i915_private *dev_priv)
15401 return true; 15393 return true;
15402} 15394}
15403 15395
15404static bool intel_crt_present(struct drm_device *dev) 15396static bool intel_crt_present(struct drm_i915_private *dev_priv)
15405{ 15397{
15406 struct drm_i915_private *dev_priv = to_i915(dev); 15398 if (INTEL_GEN(dev_priv) >= 9)
15407
15408 if (INTEL_INFO(dev)->gen >= 9)
15409 return false; 15399 return false;
15410 15400
15411 if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv)) 15401 if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
@@ -15479,7 +15469,7 @@ static void intel_setup_outputs(struct drm_device *dev)
15479 */ 15469 */
15480 intel_lvds_init(dev); 15470 intel_lvds_init(dev);
15481 15471
15482 if (intel_crt_present(dev)) 15472 if (intel_crt_present(dev_priv))
15483 intel_crt_init(dev); 15473 intel_crt_init(dev);
15484 15474
15485 if (IS_BROXTON(dev_priv)) { 15475 if (IS_BROXTON(dev_priv)) {
@@ -15527,7 +15517,7 @@ static void intel_setup_outputs(struct drm_device *dev)
15527 15517
15528 } else if (HAS_PCH_SPLIT(dev_priv)) { 15518 } else if (HAS_PCH_SPLIT(dev_priv)) {
15529 int found; 15519 int found;
15530 dpd_is_edp = intel_dp_is_edp(dev, PORT_D); 15520 dpd_is_edp = intel_dp_is_edp(dev_priv, PORT_D);
15531 15521
15532 if (has_edp_a(dev_priv)) 15522 if (has_edp_a(dev_priv))
15533 intel_dp_init(dev, DP_A, PORT_A); 15523 intel_dp_init(dev, DP_A, PORT_A);
@@ -15570,14 +15560,14 @@ static void intel_setup_outputs(struct drm_device *dev)
15570 * trust the port type the VBT declares as we've seen at least 15560 * trust the port type the VBT declares as we've seen at least
15571 * HDMI ports that the VBT claim are DP or eDP. 15561 * HDMI ports that the VBT claim are DP or eDP.
15572 */ 15562 */
15573 has_edp = intel_dp_is_edp(dev, PORT_B); 15563 has_edp = intel_dp_is_edp(dev_priv, PORT_B);
15574 has_port = intel_bios_is_port_present(dev_priv, PORT_B); 15564 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
15575 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port) 15565 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
15576 has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B); 15566 has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B);
15577 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp) 15567 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
15578 intel_hdmi_init(dev, VLV_HDMIB, PORT_B); 15568 intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
15579 15569
15580 has_edp = intel_dp_is_edp(dev, PORT_C); 15570 has_edp = intel_dp_is_edp(dev_priv, PORT_C);
15581 has_port = intel_bios_is_port_present(dev_priv, PORT_C); 15571 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
15582 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port) 15572 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
15583 has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C); 15573 has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C);
@@ -15634,7 +15624,7 @@ static void intel_setup_outputs(struct drm_device *dev)
15634 } else if (IS_GEN2(dev_priv)) 15624 } else if (IS_GEN2(dev_priv))
15635 intel_dvo_init(dev); 15625 intel_dvo_init(dev);
15636 15626
15637 if (SUPPORTS_TV(dev)) 15627 if (SUPPORTS_TV(dev_priv))
15638 intel_tv_init(dev); 15628 intel_tv_init(dev);
15639 15629
15640 intel_psr_init(dev); 15630 intel_psr_init(dev);
@@ -15689,6 +15679,8 @@ static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
15689 struct drm_i915_gem_object *obj = intel_fb->obj; 15679 struct drm_i915_gem_object *obj = intel_fb->obj;
15690 15680
15691 mutex_lock(&dev->struct_mutex); 15681 mutex_lock(&dev->struct_mutex);
15682 if (obj->pin_display && obj->cache_dirty)
15683 i915_gem_clflush_object(obj, true);
15692 intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB); 15684 intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
15693 mutex_unlock(&dev->struct_mutex); 15685 mutex_unlock(&dev->struct_mutex);
15694 15686
@@ -15769,7 +15761,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
15769 switch (mode_cmd->modifier[0]) { 15761 switch (mode_cmd->modifier[0]) {
15770 case I915_FORMAT_MOD_Y_TILED: 15762 case I915_FORMAT_MOD_Y_TILED:
15771 case I915_FORMAT_MOD_Yf_TILED: 15763 case I915_FORMAT_MOD_Yf_TILED:
15772 if (INTEL_INFO(dev)->gen < 9) { 15764 if (INTEL_GEN(dev_priv) < 9) {
15773 DRM_DEBUG("Unsupported tiling 0x%llx!\n", 15765 DRM_DEBUG("Unsupported tiling 0x%llx!\n",
15774 mode_cmd->modifier[0]); 15766 mode_cmd->modifier[0]);
15775 return -EINVAL; 15767 return -EINVAL;
@@ -15832,7 +15824,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
15832 case DRM_FORMAT_ARGB8888: 15824 case DRM_FORMAT_ARGB8888:
15833 break; 15825 break;
15834 case DRM_FORMAT_XRGB1555: 15826 case DRM_FORMAT_XRGB1555:
15835 if (INTEL_INFO(dev)->gen > 3) { 15827 if (INTEL_GEN(dev_priv) > 3) {
15836 DRM_DEBUG("unsupported pixel format: %s\n", 15828 DRM_DEBUG("unsupported pixel format: %s\n",
15837 drm_get_format_name(mode_cmd->pixel_format, &format_name)); 15829 drm_get_format_name(mode_cmd->pixel_format, &format_name));
15838 return -EINVAL; 15830 return -EINVAL;
@@ -15840,7 +15832,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
15840 break; 15832 break;
15841 case DRM_FORMAT_ABGR8888: 15833 case DRM_FORMAT_ABGR8888:
15842 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && 15834 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
15843 INTEL_INFO(dev)->gen < 9) { 15835 INTEL_GEN(dev_priv) < 9) {
15844 DRM_DEBUG("unsupported pixel format: %s\n", 15836 DRM_DEBUG("unsupported pixel format: %s\n",
15845 drm_get_format_name(mode_cmd->pixel_format, &format_name)); 15837 drm_get_format_name(mode_cmd->pixel_format, &format_name));
15846 return -EINVAL; 15838 return -EINVAL;
@@ -15849,7 +15841,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
15849 case DRM_FORMAT_XBGR8888: 15841 case DRM_FORMAT_XBGR8888:
15850 case DRM_FORMAT_XRGB2101010: 15842 case DRM_FORMAT_XRGB2101010:
15851 case DRM_FORMAT_XBGR2101010: 15843 case DRM_FORMAT_XBGR2101010:
15852 if (INTEL_INFO(dev)->gen < 4) { 15844 if (INTEL_GEN(dev_priv) < 4) {
15853 DRM_DEBUG("unsupported pixel format: %s\n", 15845 DRM_DEBUG("unsupported pixel format: %s\n",
15854 drm_get_format_name(mode_cmd->pixel_format, &format_name)); 15846 drm_get_format_name(mode_cmd->pixel_format, &format_name));
15855 return -EINVAL; 15847 return -EINVAL;
@@ -15866,7 +15858,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
15866 case DRM_FORMAT_UYVY: 15858 case DRM_FORMAT_UYVY:
15867 case DRM_FORMAT_YVYU: 15859 case DRM_FORMAT_YVYU:
15868 case DRM_FORMAT_VYUY: 15860 case DRM_FORMAT_VYUY:
15869 if (INTEL_INFO(dev)->gen < 5) { 15861 if (INTEL_GEN(dev_priv) < 5) {
15870 DRM_DEBUG("unsupported pixel format: %s\n", 15862 DRM_DEBUG("unsupported pixel format: %s\n",
15871 drm_get_format_name(mode_cmd->pixel_format, &format_name)); 15863 drm_get_format_name(mode_cmd->pixel_format, &format_name));
15872 return -EINVAL; 15864 return -EINVAL;
@@ -16295,9 +16287,8 @@ static void intel_init_quirks(struct drm_device *dev)
16295} 16287}
16296 16288
16297/* Disable the VGA plane that we never use */ 16289/* Disable the VGA plane that we never use */
16298static void i915_disable_vga(struct drm_device *dev) 16290static void i915_disable_vga(struct drm_i915_private *dev_priv)
16299{ 16291{
16300 struct drm_i915_private *dev_priv = to_i915(dev);
16301 struct pci_dev *pdev = dev_priv->drm.pdev; 16292 struct pci_dev *pdev = dev_priv->drm.pdev;
16302 u8 sr1; 16293 u8 sr1;
16303 i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); 16294 i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
@@ -16339,6 +16330,7 @@ static void sanitize_watermarks(struct drm_device *dev)
16339{ 16330{
16340 struct drm_i915_private *dev_priv = to_i915(dev); 16331 struct drm_i915_private *dev_priv = to_i915(dev);
16341 struct drm_atomic_state *state; 16332 struct drm_atomic_state *state;
16333 struct intel_atomic_state *intel_state;
16342 struct drm_crtc *crtc; 16334 struct drm_crtc *crtc;
16343 struct drm_crtc_state *cstate; 16335 struct drm_crtc_state *cstate;
16344 struct drm_modeset_acquire_ctx ctx; 16336 struct drm_modeset_acquire_ctx ctx;
@@ -16367,12 +16359,14 @@ retry:
16367 if (WARN_ON(IS_ERR(state))) 16359 if (WARN_ON(IS_ERR(state)))
16368 goto fail; 16360 goto fail;
16369 16361
16362 intel_state = to_intel_atomic_state(state);
16363
16370 /* 16364 /*
16371 * Hardware readout is the only time we don't want to calculate 16365 * Hardware readout is the only time we don't want to calculate
16372 * intermediate watermarks (since we don't trust the current 16366 * intermediate watermarks (since we don't trust the current
16373 * watermarks). 16367 * watermarks).
16374 */ 16368 */
16375 to_intel_atomic_state(state)->skip_intermediate_wm = true; 16369 intel_state->skip_intermediate_wm = true;
16376 16370
16377 ret = intel_atomic_check(dev, state); 16371 ret = intel_atomic_check(dev, state);
16378 if (ret) { 16372 if (ret) {
@@ -16396,7 +16390,7 @@ retry:
16396 struct intel_crtc_state *cs = to_intel_crtc_state(cstate); 16390 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
16397 16391
16398 cs->wm.need_postvbl_update = true; 16392 cs->wm.need_postvbl_update = true;
16399 dev_priv->display.optimize_watermarks(cs); 16393 dev_priv->display.optimize_watermarks(intel_state, cs);
16400 } 16394 }
16401 16395
16402put_state: 16396put_state:
@@ -16429,7 +16423,7 @@ int intel_modeset_init(struct drm_device *dev)
16429 16423
16430 intel_init_pm(dev_priv); 16424 intel_init_pm(dev_priv);
16431 16425
16432 if (INTEL_INFO(dev)->num_pipes == 0) 16426 if (INTEL_INFO(dev_priv)->num_pipes == 0)
16433 return 0; 16427 return 0;
16434 16428
16435 /* 16429 /*
@@ -16475,8 +16469,8 @@ int intel_modeset_init(struct drm_device *dev)
16475 dev->mode_config.fb_base = ggtt->mappable_base; 16469 dev->mode_config.fb_base = ggtt->mappable_base;
16476 16470
16477 DRM_DEBUG_KMS("%d display pipe%s available.\n", 16471 DRM_DEBUG_KMS("%d display pipe%s available.\n",
16478 INTEL_INFO(dev)->num_pipes, 16472 INTEL_INFO(dev_priv)->num_pipes,
16479 INTEL_INFO(dev)->num_pipes > 1 ? "s" : ""); 16473 INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
16480 16474
16481 for_each_pipe(dev_priv, pipe) { 16475 for_each_pipe(dev_priv, pipe) {
16482 int ret; 16476 int ret;
@@ -16497,7 +16491,7 @@ int intel_modeset_init(struct drm_device *dev)
16497 intel_update_max_cdclk(dev_priv); 16491 intel_update_max_cdclk(dev_priv);
16498 16492
16499 /* Just disable it once at startup */ 16493 /* Just disable it once at startup */
16500 i915_disable_vga(dev); 16494 i915_disable_vga(dev_priv);
16501 intel_setup_outputs(dev); 16495 intel_setup_outputs(dev);
16502 16496
16503 drm_modeset_lock_all(dev); 16497 drm_modeset_lock_all(dev);
@@ -16564,11 +16558,10 @@ static void intel_enable_pipe_a(struct drm_device *dev)
16564static bool 16558static bool
16565intel_check_plane_mapping(struct intel_crtc *crtc) 16559intel_check_plane_mapping(struct intel_crtc *crtc)
16566{ 16560{
16567 struct drm_device *dev = crtc->base.dev; 16561 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
16568 struct drm_i915_private *dev_priv = to_i915(dev);
16569 u32 val; 16562 u32 val;
16570 16563
16571 if (INTEL_INFO(dev)->num_pipes == 1) 16564 if (INTEL_INFO(dev_priv)->num_pipes == 1)
16572 return true; 16565 return true;
16573 16566
16574 val = I915_READ(DSPCNTR(!crtc->plane)); 16567 val = I915_READ(DSPCNTR(!crtc->plane));
@@ -16642,7 +16635,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
16642 /* We need to sanitize the plane -> pipe mapping first because this will 16635 /* We need to sanitize the plane -> pipe mapping first because this will
16643 * disable the crtc (and hence change the state) if it is wrong. Note 16636 * disable the crtc (and hence change the state) if it is wrong. Note
16644 * that gen4+ has a fixed plane -> pipe mapping. */ 16637 * that gen4+ has a fixed plane -> pipe mapping. */
16645 if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) { 16638 if (INTEL_GEN(dev_priv) < 4 && !intel_check_plane_mapping(crtc)) {
16646 bool plane; 16639 bool plane;
16647 16640
16648 DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n", 16641 DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n",
@@ -16744,21 +16737,18 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
16744 * the crtc fixup. */ 16737 * the crtc fixup. */
16745} 16738}
16746 16739
16747void i915_redisable_vga_power_on(struct drm_device *dev) 16740void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
16748{ 16741{
16749 struct drm_i915_private *dev_priv = to_i915(dev);
16750 i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv); 16742 i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
16751 16743
16752 if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { 16744 if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
16753 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n"); 16745 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
16754 i915_disable_vga(dev); 16746 i915_disable_vga(dev_priv);
16755 } 16747 }
16756} 16748}
16757 16749
16758void i915_redisable_vga(struct drm_device *dev) 16750void i915_redisable_vga(struct drm_i915_private *dev_priv)
16759{ 16751{
16760 struct drm_i915_private *dev_priv = to_i915(dev);
16761
16762 /* This function can be called both from intel_modeset_setup_hw_state or 16752 /* This function can be called both from intel_modeset_setup_hw_state or
16763 * at a very early point in our resume sequence, where the power well 16753 * at a very early point in our resume sequence, where the power well
16764 * structures are not yet restored. Since this function is at a very 16754 * structures are not yet restored. Since this function is at a very
@@ -16769,7 +16759,7 @@ void i915_redisable_vga(struct drm_device *dev)
16769 if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA)) 16759 if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
16770 return; 16760 return;
16771 16761
16772 i915_redisable_vga_power_on(dev); 16762 i915_redisable_vga_power_on(dev_priv);
16773 16763
16774 intel_display_power_put(dev_priv, POWER_DOMAIN_VGA); 16764 intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
16775} 16765}
@@ -16841,7 +16831,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
16841 16831
16842 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n", 16832 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16843 crtc->base.base.id, crtc->base.name, 16833 crtc->base.base.id, crtc->base.name,
16844 crtc->active ? "enabled" : "disabled"); 16834 enableddisabled(crtc->active));
16845 } 16835 }
16846 16836
16847 for (i = 0; i < dev_priv->num_shared_dpll; i++) { 16837 for (i = 0; i < dev_priv->num_shared_dpll; i++) {
@@ -16874,9 +16864,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
16874 } 16864 }
16875 16865
16876 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n", 16866 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16877 encoder->base.base.id, 16867 encoder->base.base.id, encoder->base.name,
16878 encoder->base.name, 16868 enableddisabled(encoder->base.crtc),
16879 encoder->base.crtc ? "enabled" : "disabled",
16880 pipe_name(pipe)); 16869 pipe_name(pipe));
16881 } 16870 }
16882 16871
@@ -16905,9 +16894,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
16905 connector->base.encoder = NULL; 16894 connector->base.encoder = NULL;
16906 } 16895 }
16907 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n", 16896 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16908 connector->base.base.id, 16897 connector->base.base.id, connector->base.name,
16909 connector->base.name, 16898 enableddisabled(connector->base.encoder));
16910 connector->base.encoder ? "enabled" : "disabled");
16911 } 16899 }
16912 16900
16913 for_each_intel_crtc(dev, crtc) { 16901 for_each_intel_crtc(dev, crtc) {
@@ -17155,10 +17143,9 @@ void intel_connector_attach_encoder(struct intel_connector *connector,
17155/* 17143/*
17156 * set vga decode state - true == enable VGA decode 17144 * set vga decode state - true == enable VGA decode
17157 */ 17145 */
17158int intel_modeset_vga_set_state(struct drm_device *dev, bool state) 17146int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
17159{ 17147{
17160 struct drm_i915_private *dev_priv = to_i915(dev); 17148 unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17161 unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
17162 u16 gmch_ctrl; 17149 u16 gmch_ctrl;
17163 17150
17164 if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) { 17151 if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
@@ -17312,16 +17299,15 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv)
17312 17299
17313void 17300void
17314intel_display_print_error_state(struct drm_i915_error_state_buf *m, 17301intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17315 struct drm_device *dev, 17302 struct drm_i915_private *dev_priv,
17316 struct intel_display_error_state *error) 17303 struct intel_display_error_state *error)
17317{ 17304{
17318 struct drm_i915_private *dev_priv = to_i915(dev);
17319 int i; 17305 int i;
17320 17306
17321 if (!error) 17307 if (!error)
17322 return; 17308 return;
17323 17309
17324 err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes); 17310 err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
17325 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 17311 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
17326 err_printf(m, "PWR_WELL_CTL2: %08x\n", 17312 err_printf(m, "PWR_WELL_CTL2: %08x\n",
17327 error->power_well_driver); 17313 error->power_well_driver);
@@ -17335,13 +17321,13 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
17335 err_printf(m, "Plane [%d]:\n", i); 17321 err_printf(m, "Plane [%d]:\n", i);
17336 err_printf(m, " CNTR: %08x\n", error->plane[i].control); 17322 err_printf(m, " CNTR: %08x\n", error->plane[i].control);
17337 err_printf(m, " STRIDE: %08x\n", error->plane[i].stride); 17323 err_printf(m, " STRIDE: %08x\n", error->plane[i].stride);
17338 if (INTEL_INFO(dev)->gen <= 3) { 17324 if (INTEL_GEN(dev_priv) <= 3) {
17339 err_printf(m, " SIZE: %08x\n", error->plane[i].size); 17325 err_printf(m, " SIZE: %08x\n", error->plane[i].size);
17340 err_printf(m, " POS: %08x\n", error->plane[i].pos); 17326 err_printf(m, " POS: %08x\n", error->plane[i].pos);
17341 } 17327 }
17342 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv)) 17328 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
17343 err_printf(m, " ADDR: %08x\n", error->plane[i].addr); 17329 err_printf(m, " ADDR: %08x\n", error->plane[i].addr);
17344 if (INTEL_INFO(dev)->gen >= 4) { 17330 if (INTEL_GEN(dev_priv) >= 4) {
17345 err_printf(m, " SURF: %08x\n", error->plane[i].surface); 17331 err_printf(m, " SURF: %08x\n", error->plane[i].surface);
17346 err_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset); 17332 err_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset);
17347 } 17333 }
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 9df331b3305b..90283edcafba 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -942,14 +942,14 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
942 uint8_t *recv, int recv_size) 942 uint8_t *recv, int recv_size)
943{ 943{
944 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); 944 struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
945 struct drm_device *dev = intel_dig_port->base.base.dev; 945 struct drm_i915_private *dev_priv =
946 struct drm_i915_private *dev_priv = to_i915(dev); 946 to_i915(intel_dig_port->base.base.dev);
947 i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; 947 i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
948 uint32_t aux_clock_divider; 948 uint32_t aux_clock_divider;
949 int i, ret, recv_bytes; 949 int i, ret, recv_bytes;
950 uint32_t status; 950 uint32_t status;
951 int try, clock = 0; 951 int try, clock = 0;
952 bool has_aux_irq = HAS_AUX_IRQ(dev); 952 bool has_aux_irq = HAS_AUX_IRQ(dev_priv);
953 bool vdd; 953 bool vdd;
954 954
955 pps_lock(intel_dp); 955 pps_lock(intel_dp);
@@ -1542,8 +1542,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
1542 struct intel_crtc_state *pipe_config, 1542 struct intel_crtc_state *pipe_config,
1543 struct drm_connector_state *conn_state) 1543 struct drm_connector_state *conn_state)
1544{ 1544{
1545 struct drm_device *dev = encoder->base.dev; 1545 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1546 struct drm_i915_private *dev_priv = to_i915(dev);
1547 struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; 1546 struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
1548 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 1547 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
1549 enum port port = dp_to_dig_port(intel_dp)->port; 1548 enum port port = dp_to_dig_port(intel_dp)->port;
@@ -1578,7 +1577,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
1578 intel_fixed_panel_mode(intel_connector->panel.fixed_mode, 1577 intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
1579 adjusted_mode); 1578 adjusted_mode);
1580 1579
1581 if (INTEL_INFO(dev)->gen >= 9) { 1580 if (INTEL_GEN(dev_priv) >= 9) {
1582 int ret; 1581 int ret;
1583 ret = skl_update_scaler_crtc(pipe_config); 1582 ret = skl_update_scaler_crtc(pipe_config);
1584 if (ret) 1583 if (ret)
@@ -1791,9 +1790,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
1791 trans_dp &= ~TRANS_DP_ENH_FRAMING; 1790 trans_dp &= ~TRANS_DP_ENH_FRAMING;
1792 I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp); 1791 I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
1793 } else { 1792 } else {
1794 if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) && 1793 if (IS_G4X(dev_priv) && pipe_config->limited_color_range)
1795 !IS_CHERRYVIEW(dev_priv) &&
1796 pipe_config->limited_color_range)
1797 intel_dp->DP |= DP_COLOR_RANGE_16_235; 1794 intel_dp->DP |= DP_COLOR_RANGE_16_235;
1798 1795
1799 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) 1796 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -2515,8 +2512,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
2515 2512
2516 pipe_config->base.adjusted_mode.flags |= flags; 2513 pipe_config->base.adjusted_mode.flags |= flags;
2517 2514
2518 if (!HAS_PCH_SPLIT(dev_priv) && !IS_VALLEYVIEW(dev_priv) && 2515 if (IS_G4X(dev_priv) && tmp & DP_COLOR_RANGE_16_235)
2519 !IS_CHERRYVIEW(dev_priv) && tmp & DP_COLOR_RANGE_16_235)
2520 pipe_config->limited_color_range = true; 2516 pipe_config->limited_color_range = true;
2521 2517
2522 pipe_config->lane_count = 2518 pipe_config->lane_count =
@@ -2735,7 +2731,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp,
2735} 2731}
2736 2732
2737static void intel_enable_dp(struct intel_encoder *encoder, 2733static void intel_enable_dp(struct intel_encoder *encoder,
2738 struct intel_crtc_state *pipe_config) 2734 struct intel_crtc_state *pipe_config,
2735 struct drm_connector_state *conn_state)
2739{ 2736{
2740 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 2737 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
2741 struct drm_device *dev = encoder->base.dev; 2738 struct drm_device *dev = encoder->base.dev;
@@ -2777,7 +2774,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
2777 if (pipe_config->has_audio) { 2774 if (pipe_config->has_audio) {
2778 DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", 2775 DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
2779 pipe_name(pipe)); 2776 pipe_name(pipe));
2780 intel_audio_codec_enable(encoder); 2777 intel_audio_codec_enable(encoder, pipe_config, conn_state);
2781 } 2778 }
2782} 2779}
2783 2780
@@ -2787,7 +2784,7 @@ static void g4x_enable_dp(struct intel_encoder *encoder,
2787{ 2784{
2788 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); 2785 struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
2789 2786
2790 intel_enable_dp(encoder, pipe_config); 2787 intel_enable_dp(encoder, pipe_config, conn_state);
2791 intel_edp_backlight_on(intel_dp); 2788 intel_edp_backlight_on(intel_dp);
2792} 2789}
2793 2790
@@ -2924,7 +2921,7 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder,
2924{ 2921{
2925 vlv_phy_pre_encoder_enable(encoder); 2922 vlv_phy_pre_encoder_enable(encoder);
2926 2923
2927 intel_enable_dp(encoder, pipe_config); 2924 intel_enable_dp(encoder, pipe_config, conn_state);
2928} 2925}
2929 2926
2930static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder, 2927static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder,
@@ -2942,7 +2939,7 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder,
2942{ 2939{
2943 chv_phy_pre_encoder_enable(encoder); 2940 chv_phy_pre_encoder_enable(encoder);
2944 2941
2945 intel_enable_dp(encoder, pipe_config); 2942 intel_enable_dp(encoder, pipe_config, conn_state);
2946 2943
2947 /* Second common lane will stay alive on its own now */ 2944 /* Second common lane will stay alive on its own now */
2948 chv_phy_release_cl2_override(encoder); 2945 chv_phy_release_cl2_override(encoder);
@@ -2979,13 +2976,12 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_
2979uint8_t 2976uint8_t
2980intel_dp_voltage_max(struct intel_dp *intel_dp) 2977intel_dp_voltage_max(struct intel_dp *intel_dp)
2981{ 2978{
2982 struct drm_device *dev = intel_dp_to_dev(intel_dp); 2979 struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
2983 struct drm_i915_private *dev_priv = to_i915(dev);
2984 enum port port = dp_to_dig_port(intel_dp)->port; 2980 enum port port = dp_to_dig_port(intel_dp)->port;
2985 2981
2986 if (IS_BROXTON(dev_priv)) 2982 if (IS_BROXTON(dev_priv))
2987 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; 2983 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
2988 else if (INTEL_INFO(dev)->gen >= 9) { 2984 else if (INTEL_GEN(dev_priv) >= 9) {
2989 if (dev_priv->vbt.edp.low_vswing && port == PORT_A) 2985 if (dev_priv->vbt.edp.low_vswing && port == PORT_A)
2990 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3; 2986 return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
2991 return DP_TRAIN_VOLTAGE_SWING_LEVEL_2; 2987 return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
@@ -4873,15 +4869,13 @@ put_power:
4873} 4869}
4874 4870
4875/* check the VBT to see whether the eDP is on another port */ 4871/* check the VBT to see whether the eDP is on another port */
4876bool intel_dp_is_edp(struct drm_device *dev, enum port port) 4872bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port)
4877{ 4873{
4878 struct drm_i915_private *dev_priv = to_i915(dev);
4879
4880 /* 4874 /*
4881 * eDP not supported on g4x. so bail out early just 4875 * eDP not supported on g4x. so bail out early just
4882 * for a bit extra safety in case the VBT is bonkers. 4876 * for a bit extra safety in case the VBT is bonkers.
4883 */ 4877 */
4884 if (INTEL_INFO(dev)->gen < 5) 4878 if (INTEL_GEN(dev_priv) < 5)
4885 return false; 4879 return false;
4886 4880
4887 if (port == PORT_A) 4881 if (port == PORT_A)
@@ -5483,7 +5477,7 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
5483 INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work); 5477 INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
5484 mutex_init(&dev_priv->drrs.mutex); 5478 mutex_init(&dev_priv->drrs.mutex);
5485 5479
5486 if (INTEL_INFO(dev)->gen <= 6) { 5480 if (INTEL_GEN(dev_priv) <= 6) {
5487 DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n"); 5481 DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n");
5488 return NULL; 5482 return NULL;
5489 } 5483 }
@@ -5657,7 +5651,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
5657 intel_dp->pps_pipe = INVALID_PIPE; 5651 intel_dp->pps_pipe = INVALID_PIPE;
5658 5652
5659 /* intel_dp vfuncs */ 5653 /* intel_dp vfuncs */
5660 if (INTEL_INFO(dev)->gen >= 9) 5654 if (INTEL_GEN(dev_priv) >= 9)
5661 intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider; 5655 intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
5662 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 5656 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5663 intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider; 5657 intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
@@ -5666,7 +5660,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
5666 else 5660 else
5667 intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider; 5661 intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
5668 5662
5669 if (INTEL_INFO(dev)->gen >= 9) 5663 if (INTEL_GEN(dev_priv) >= 9)
5670 intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl; 5664 intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl;
5671 else 5665 else
5672 intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl; 5666 intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
@@ -5678,7 +5672,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
5678 intel_dp->DP = I915_READ(intel_dp->output_reg); 5672 intel_dp->DP = I915_READ(intel_dp->output_reg);
5679 intel_dp->attached_connector = intel_connector; 5673 intel_dp->attached_connector = intel_connector;
5680 5674
5681 if (intel_dp_is_edp(dev, port)) 5675 if (intel_dp_is_edp(dev_priv, port))
5682 type = DRM_MODE_CONNECTOR_eDP; 5676 type = DRM_MODE_CONNECTOR_eDP;
5683 else 5677 else
5684 type = DRM_MODE_CONNECTOR_DisplayPort; 5678 type = DRM_MODE_CONNECTOR_DisplayPort;
@@ -5742,7 +5736,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
5742 } 5736 }
5743 5737
5744 /* init MST on ports that can support it */ 5738 /* init MST on ports that can support it */
5745 if (HAS_DP_MST(dev) && !is_edp(intel_dp) && 5739 if (HAS_DP_MST(dev_priv) && !is_edp(intel_dp) &&
5746 (port == PORT_B || port == PORT_C || port == PORT_D)) 5740 (port == PORT_B || port == PORT_C || port == PORT_D))
5747 intel_dp_mst_encoder_init(intel_dig_port, 5741 intel_dp_mst_encoder_init(intel_dig_port,
5748 intel_connector->base.base.id); 5742 intel_connector->base.base.id);
@@ -5816,7 +5810,7 @@ bool intel_dp_init(struct drm_device *dev,
5816 } else { 5810 } else {
5817 intel_encoder->pre_enable = g4x_pre_enable_dp; 5811 intel_encoder->pre_enable = g4x_pre_enable_dp;
5818 intel_encoder->enable = g4x_enable_dp; 5812 intel_encoder->enable = g4x_enable_dp;
5819 if (INTEL_INFO(dev)->gen >= 5) 5813 if (INTEL_GEN(dev_priv) >= 5)
5820 intel_encoder->post_disable = ilk_post_disable_dp; 5814 intel_encoder->post_disable = ilk_post_disable_dp;
5821 } 5815 }
5822 5816
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 3ffbd69e4551..b029d1026a28 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -43,7 +43,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
43 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; 43 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
44 int mst_pbn; 44 int mst_pbn;
45 45
46 pipe_config->dp_encoder_is_mst = true;
47 pipe_config->has_pch_encoder = false; 46 pipe_config->has_pch_encoder = false;
48 bpp = 24; 47 bpp = 24;
49 /* 48 /*
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 21853a17b6d9..58a756f2f224 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -188,13 +188,12 @@ out:
188 188
189void intel_disable_shared_dpll(struct intel_crtc *crtc) 189void intel_disable_shared_dpll(struct intel_crtc *crtc)
190{ 190{
191 struct drm_device *dev = crtc->base.dev; 191 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
192 struct drm_i915_private *dev_priv = to_i915(dev);
193 struct intel_shared_dpll *pll = crtc->config->shared_dpll; 192 struct intel_shared_dpll *pll = crtc->config->shared_dpll;
194 unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base); 193 unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base);
195 194
196 /* PCH only available on ILK+ */ 195 /* PCH only available on ILK+ */
197 if (INTEL_INFO(dev)->gen < 5) 196 if (INTEL_GEN(dev_priv) < 5)
198 return; 197 return;
199 198
200 if (pll == NULL) 199 if (pll == NULL)
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 398195bf6dd1..cd132c216a67 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -294,6 +294,9 @@ struct intel_connector {
294 */ 294 */
295 struct intel_encoder *encoder; 295 struct intel_encoder *encoder;
296 296
297 /* ACPI device id for ACPI and driver cooperation */
298 u32 acpi_device_id;
299
297 /* Reads out the current hw, returning true if the connector is enabled 300 /* Reads out the current hw, returning true if the connector is enabled
298 * and active (i.e. dpms ON state). */ 301 * and active (i.e. dpms ON state). */
299 bool (*get_hw_state)(struct intel_connector *); 302 bool (*get_hw_state)(struct intel_connector *);
@@ -652,7 +655,6 @@ struct intel_crtc_state {
652 655
653 bool double_wide; 656 bool double_wide;
654 657
655 bool dp_encoder_is_mst;
656 int pbn; 658 int pbn;
657 659
658 struct intel_crtc_scaler_state scaler_state; 660 struct intel_crtc_scaler_state scaler_state;
@@ -728,9 +730,6 @@ struct intel_crtc {
728 bool cxsr_allowed; 730 bool cxsr_allowed;
729 } wm; 731 } wm;
730 732
731 /* gen9+: ddb allocation currently being used */
732 struct skl_ddb_entry hw_ddb;
733
734 int scanline_offset; 733 int scanline_offset;
735 734
736 struct { 735 struct {
@@ -1187,7 +1186,9 @@ u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv,
1187 1186
1188/* intel_audio.c */ 1187/* intel_audio.c */
1189void intel_init_audio_hooks(struct drm_i915_private *dev_priv); 1188void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
1190void intel_audio_codec_enable(struct intel_encoder *encoder); 1189void intel_audio_codec_enable(struct intel_encoder *encoder,
1190 const struct intel_crtc_state *crtc_state,
1191 const struct drm_connector_state *conn_state);
1191void intel_audio_codec_disable(struct intel_encoder *encoder); 1192void intel_audio_codec_disable(struct intel_encoder *encoder);
1192void i915_audio_component_init(struct drm_i915_private *dev_priv); 1193void i915_audio_component_init(struct drm_i915_private *dev_priv);
1193void i915_audio_component_cleanup(struct drm_i915_private *dev_priv); 1194void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
@@ -1392,7 +1393,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
1392bool intel_dp_compute_config(struct intel_encoder *encoder, 1393bool intel_dp_compute_config(struct intel_encoder *encoder,
1393 struct intel_crtc_state *pipe_config, 1394 struct intel_crtc_state *pipe_config,
1394 struct drm_connector_state *conn_state); 1395 struct drm_connector_state *conn_state);
1395bool intel_dp_is_edp(struct drm_device *dev, enum port port); 1396bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port);
1396enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, 1397enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
1397 bool long_hpd); 1398 bool long_hpd);
1398void intel_edp_backlight_on(struct intel_dp *intel_dp); 1399void intel_edp_backlight_on(struct intel_dp *intel_dp);
@@ -1738,18 +1739,9 @@ int intel_enable_sagv(struct drm_i915_private *dev_priv);
1738int intel_disable_sagv(struct drm_i915_private *dev_priv); 1739int intel_disable_sagv(struct drm_i915_private *dev_priv);
1739bool skl_wm_level_equals(const struct skl_wm_level *l1, 1740bool skl_wm_level_equals(const struct skl_wm_level *l1,
1740 const struct skl_wm_level *l2); 1741 const struct skl_wm_level *l2);
1741bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old, 1742bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
1742 const struct skl_ddb_allocation *new, 1743 const struct skl_ddb_entry *ddb,
1743 enum pipe pipe); 1744 int ignore);
1744bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
1745 struct intel_crtc *intel_crtc);
1746void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
1747 const struct skl_plane_wm *wm,
1748 const struct skl_ddb_allocation *ddb);
1749void skl_write_plane_wm(struct intel_crtc *intel_crtc,
1750 const struct skl_plane_wm *wm,
1751 const struct skl_ddb_allocation *ddb,
1752 int plane);
1753uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config); 1745uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
1754bool ilk_disable_lp_wm(struct drm_device *dev); 1746bool ilk_disable_lp_wm(struct drm_device *dev);
1755int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6); 1747int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
index 9f279a3d0f74..0d8ff0034b88 100644
--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
@@ -774,9 +774,8 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
774 8); 774 8);
775 intel_dsi->clk_hs_to_lp_count += extra_byte_count; 775 intel_dsi->clk_hs_to_lp_count += extra_byte_count;
776 776
777 DRM_DEBUG_KMS("Eot %s\n", intel_dsi->eotp_pkt ? "enabled" : "disabled"); 777 DRM_DEBUG_KMS("Eot %s\n", enableddisabled(intel_dsi->eotp_pkt));
778 DRM_DEBUG_KMS("Clockstop %s\n", intel_dsi->clock_stop ? 778 DRM_DEBUG_KMS("Clockstop %s\n", enableddisabled(!intel_dsi->clock_stop));
779 "disabled" : "enabled");
780 DRM_DEBUG_KMS("Mode %s\n", intel_dsi->operation_mode ? "command" : "video"); 779 DRM_DEBUG_KMS("Mode %s\n", intel_dsi->operation_mode ? "command" : "video");
781 if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK) 780 if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK)
782 DRM_DEBUG_KMS("Dual link: DSI_DUAL_LINK_FRONT_BACK\n"); 781 DRM_DEBUG_KMS("Dual link: DSI_DUAL_LINK_FRONT_BACK\n");
@@ -795,8 +794,7 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
795 DRM_DEBUG_KMS("LP to HS Clock Count 0x%x\n", intel_dsi->clk_lp_to_hs_count); 794 DRM_DEBUG_KMS("LP to HS Clock Count 0x%x\n", intel_dsi->clk_lp_to_hs_count);
796 DRM_DEBUG_KMS("HS to LP Clock Count 0x%x\n", intel_dsi->clk_hs_to_lp_count); 795 DRM_DEBUG_KMS("HS to LP Clock Count 0x%x\n", intel_dsi->clk_hs_to_lp_count);
797 DRM_DEBUG_KMS("BTA %s\n", 796 DRM_DEBUG_KMS("BTA %s\n",
798 intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA ? 797 enableddisabled(!(intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA)));
799 "disabled" : "enabled");
800 798
801 /* delays in VBT are in unit of 100us, so need to convert 799 /* delays in VBT are in unit of 100us, so need to convert
802 * here in ms 800 * here in ms
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 841f8d1e1410..3da4d466e332 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -102,6 +102,9 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
102 engine->mmio_base = info->mmio_base; 102 engine->mmio_base = info->mmio_base;
103 engine->irq_shift = info->irq_shift; 103 engine->irq_shift = info->irq_shift;
104 104
105 /* Nothing to do here, execute in order of dependencies */
106 engine->schedule = NULL;
107
105 dev_priv->engine[id] = engine; 108 dev_priv->engine[id] = engine;
106 return 0; 109 return 0;
107} 110}
@@ -236,8 +239,8 @@ static void intel_engine_init_timeline(struct intel_engine_cs *engine)
236 */ 239 */
237void intel_engine_setup_common(struct intel_engine_cs *engine) 240void intel_engine_setup_common(struct intel_engine_cs *engine)
238{ 241{
239 INIT_LIST_HEAD(&engine->execlist_queue); 242 engine->execlist_queue = RB_ROOT;
240 spin_lock_init(&engine->execlist_lock); 243 engine->execlist_first = NULL;
241 244
242 intel_engine_init_timeline(engine); 245 intel_engine_init_timeline(engine);
243 intel_engine_init_hangcheck(engine); 246 intel_engine_init_hangcheck(engine);
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index e230d480c5e6..62f215b12eb5 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -48,17 +48,17 @@ static inline bool fbc_supported(struct drm_i915_private *dev_priv)
48 48
49static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv) 49static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv)
50{ 50{
51 return IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8; 51 return IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8;
52} 52}
53 53
54static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv) 54static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv)
55{ 55{
56 return INTEL_INFO(dev_priv)->gen < 4; 56 return INTEL_GEN(dev_priv) < 4;
57} 57}
58 58
59static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv) 59static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv)
60{ 60{
61 return INTEL_INFO(dev_priv)->gen <= 3; 61 return INTEL_GEN(dev_priv) <= 3;
62} 62}
63 63
64/* 64/*
@@ -351,7 +351,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
351 351
352static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv) 352static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)
353{ 353{
354 if (INTEL_INFO(dev_priv)->gen >= 5) 354 if (INTEL_GEN(dev_priv) >= 5)
355 return ilk_fbc_is_active(dev_priv); 355 return ilk_fbc_is_active(dev_priv);
356 else if (IS_GM45(dev_priv)) 356 else if (IS_GM45(dev_priv))
357 return g4x_fbc_is_active(dev_priv); 357 return g4x_fbc_is_active(dev_priv);
@@ -365,9 +365,9 @@ static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)
365 365
366 fbc->active = true; 366 fbc->active = true;
367 367
368 if (INTEL_INFO(dev_priv)->gen >= 7) 368 if (INTEL_GEN(dev_priv) >= 7)
369 gen7_fbc_activate(dev_priv); 369 gen7_fbc_activate(dev_priv);
370 else if (INTEL_INFO(dev_priv)->gen >= 5) 370 else if (INTEL_GEN(dev_priv) >= 5)
371 ilk_fbc_activate(dev_priv); 371 ilk_fbc_activate(dev_priv);
372 else if (IS_GM45(dev_priv)) 372 else if (IS_GM45(dev_priv))
373 g4x_fbc_activate(dev_priv); 373 g4x_fbc_activate(dev_priv);
@@ -381,7 +381,7 @@ static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)
381 381
382 fbc->active = false; 382 fbc->active = false;
383 383
384 if (INTEL_INFO(dev_priv)->gen >= 5) 384 if (INTEL_GEN(dev_priv) >= 5)
385 ilk_fbc_deactivate(dev_priv); 385 ilk_fbc_deactivate(dev_priv);
386 else if (IS_GM45(dev_priv)) 386 else if (IS_GM45(dev_priv))
387 g4x_fbc_deactivate(dev_priv); 387 g4x_fbc_deactivate(dev_priv);
@@ -561,7 +561,7 @@ again:
561 561
562 ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1, 562 ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1,
563 4096, 0, end); 563 4096, 0, end);
564 if (ret && INTEL_INFO(dev_priv)->gen <= 4) { 564 if (ret && INTEL_GEN(dev_priv) <= 4) {
565 return 0; 565 return 0;
566 } else if (ret) { 566 } else if (ret) {
567 compression_threshold <<= 1; 567 compression_threshold <<= 1;
@@ -594,7 +594,7 @@ static int intel_fbc_alloc_cfb(struct intel_crtc *crtc)
594 594
595 fbc->threshold = ret; 595 fbc->threshold = ret;
596 596
597 if (INTEL_INFO(dev_priv)->gen >= 5) 597 if (INTEL_GEN(dev_priv) >= 5)
598 I915_WRITE(ILK_DPFC_CB_BASE, fbc->compressed_fb.start); 598 I915_WRITE(ILK_DPFC_CB_BASE, fbc->compressed_fb.start);
599 else if (IS_GM45(dev_priv)) { 599 else if (IS_GM45(dev_priv)) {
600 I915_WRITE(DPFC_CB_BASE, fbc->compressed_fb.start); 600 I915_WRITE(DPFC_CB_BASE, fbc->compressed_fb.start);
@@ -708,10 +708,10 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
708 struct intel_fbc *fbc = &dev_priv->fbc; 708 struct intel_fbc *fbc = &dev_priv->fbc;
709 unsigned int effective_w, effective_h, max_w, max_h; 709 unsigned int effective_w, effective_h, max_w, max_h;
710 710
711 if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) { 711 if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {
712 max_w = 4096; 712 max_w = 4096;
713 max_h = 4096; 713 max_h = 4096;
714 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) { 714 } else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
715 max_w = 4096; 715 max_w = 4096;
716 max_h = 2048; 716 max_h = 2048;
717 } else { 717 } else {
@@ -812,7 +812,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
812 fbc->no_fbc_reason = "framebuffer not tiled or fenced"; 812 fbc->no_fbc_reason = "framebuffer not tiled or fenced";
813 return false; 813 return false;
814 } 814 }
815 if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) && 815 if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) &&
816 cache->plane.rotation != DRM_ROTATE_0) { 816 cache->plane.rotation != DRM_ROTATE_0) {
817 fbc->no_fbc_reason = "rotation unsupported"; 817 fbc->no_fbc_reason = "rotation unsupported";
818 return false; 818 return false;
@@ -854,9 +854,8 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
854 return true; 854 return true;
855} 855}
856 856
857static bool intel_fbc_can_choose(struct intel_crtc *crtc) 857static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)
858{ 858{
859 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
860 struct intel_fbc *fbc = &dev_priv->fbc; 859 struct intel_fbc *fbc = &dev_priv->fbc;
861 860
862 if (intel_vgpu_active(dev_priv)) { 861 if (intel_vgpu_active(dev_priv)) {
@@ -874,16 +873,6 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc)
874 return false; 873 return false;
875 } 874 }
876 875
877 if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) {
878 fbc->no_fbc_reason = "no enabled pipes can have FBC";
879 return false;
880 }
881
882 if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A) {
883 fbc->no_fbc_reason = "no enabled planes can have FBC";
884 return false;
885 }
886
887 return true; 876 return true;
888} 877}
889 878
@@ -1066,23 +1055,19 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
1066 struct drm_atomic_state *state) 1055 struct drm_atomic_state *state)
1067{ 1056{
1068 struct intel_fbc *fbc = &dev_priv->fbc; 1057 struct intel_fbc *fbc = &dev_priv->fbc;
1069 struct drm_crtc *crtc;
1070 struct drm_crtc_state *crtc_state;
1071 struct drm_plane *plane; 1058 struct drm_plane *plane;
1072 struct drm_plane_state *plane_state; 1059 struct drm_plane_state *plane_state;
1073 bool fbc_crtc_present = false; 1060 bool crtc_chosen = false;
1074 int i, j; 1061 int i;
1075 1062
1076 mutex_lock(&fbc->lock); 1063 mutex_lock(&fbc->lock);
1077 1064
1078 for_each_crtc_in_state(state, crtc, crtc_state, i) { 1065 /* Does this atomic commit involve the CRTC currently tied to FBC? */
1079 if (fbc->crtc == to_intel_crtc(crtc)) { 1066 if (fbc->crtc &&
1080 fbc_crtc_present = true; 1067 !drm_atomic_get_existing_crtc_state(state, &fbc->crtc->base))
1081 break; 1068 goto out;
1082 } 1069
1083 } 1070 if (!intel_fbc_can_enable(dev_priv))
1084 /* This atomic commit doesn't involve the CRTC currently tied to FBC. */
1085 if (!fbc_crtc_present && fbc->crtc != NULL)
1086 goto out; 1071 goto out;
1087 1072
1088 /* Simply choose the first CRTC that is compatible and has a visible 1073 /* Simply choose the first CRTC that is compatible and has a visible
@@ -1092,25 +1077,29 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
1092 for_each_plane_in_state(state, plane, plane_state, i) { 1077 for_each_plane_in_state(state, plane, plane_state, i) {
1093 struct intel_plane_state *intel_plane_state = 1078 struct intel_plane_state *intel_plane_state =
1094 to_intel_plane_state(plane_state); 1079 to_intel_plane_state(plane_state);
1080 struct intel_crtc_state *intel_crtc_state;
1081 struct intel_crtc *crtc = to_intel_crtc(plane_state->crtc);
1095 1082
1096 if (!intel_plane_state->base.visible) 1083 if (!intel_plane_state->base.visible)
1097 continue; 1084 continue;
1098 1085
1099 for_each_crtc_in_state(state, crtc, crtc_state, j) { 1086 if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A)
1100 struct intel_crtc_state *intel_crtc_state = 1087 continue;
1101 to_intel_crtc_state(crtc_state);
1102 1088
1103 if (plane_state->crtc != crtc) 1089 if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A)
1104 continue; 1090 continue;
1105 1091
1106 if (!intel_fbc_can_choose(to_intel_crtc(crtc))) 1092 intel_crtc_state = to_intel_crtc_state(
1107 break; 1093 drm_atomic_get_existing_crtc_state(state, &crtc->base));
1108 1094
1109 intel_crtc_state->enable_fbc = true; 1095 intel_crtc_state->enable_fbc = true;
1110 goto out; 1096 crtc_chosen = true;
1111 } 1097 break;
1112 } 1098 }
1113 1099
1100 if (!crtc_chosen)
1101 fbc->no_fbc_reason = "no suitable CRTC for FBC";
1102
1114out: 1103out:
1115 mutex_unlock(&fbc->lock); 1104 mutex_unlock(&fbc->lock);
1116} 1105}
@@ -1386,7 +1375,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
1386 } 1375 }
1387 1376
1388 /* This value was pulled out of someone's hat */ 1377 /* This value was pulled out of someone's hat */
1389 if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_GM45(dev_priv)) 1378 if (INTEL_GEN(dev_priv) <= 4 && !IS_GM45(dev_priv))
1390 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT); 1379 I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
1391 1380
1392 /* We still don't have any sort of hardware state readout for FBC, so 1381 /* We still don't have any sort of hardware state readout for FBC, so
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index e87221bba475..fc958d5ed0dc 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -356,7 +356,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
356 struct drm_fb_offset *offsets, 356 struct drm_fb_offset *offsets,
357 bool *enabled, int width, int height) 357 bool *enabled, int width, int height)
358{ 358{
359 struct drm_device *dev = fb_helper->dev; 359 struct drm_i915_private *dev_priv = to_i915(fb_helper->dev);
360 unsigned long conn_configured, mask; 360 unsigned long conn_configured, mask;
361 unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG); 361 unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
362 int i, j; 362 int i, j;
@@ -509,7 +509,7 @@ retry:
509 * fbdev helper library. 509 * fbdev helper library.
510 */ 510 */
511 if (num_connectors_enabled != num_connectors_detected && 511 if (num_connectors_enabled != num_connectors_detected &&
512 num_connectors_enabled < INTEL_INFO(dev)->num_pipes) { 512 num_connectors_enabled < INTEL_INFO(dev_priv)->num_pipes) {
513 DRM_DEBUG_KMS("fallback: Not all outputs enabled\n"); 513 DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
514 DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled, 514 DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
515 num_connectors_detected); 515 num_connectors_detected);
@@ -697,11 +697,11 @@ static void intel_fbdev_suspend_worker(struct work_struct *work)
697 697
698int intel_fbdev_init(struct drm_device *dev) 698int intel_fbdev_init(struct drm_device *dev)
699{ 699{
700 struct intel_fbdev *ifbdev;
701 struct drm_i915_private *dev_priv = to_i915(dev); 700 struct drm_i915_private *dev_priv = to_i915(dev);
701 struct intel_fbdev *ifbdev;
702 int ret; 702 int ret;
703 703
704 if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0)) 704 if (WARN_ON(INTEL_INFO(dev_priv)->num_pipes == 0))
705 return -ENODEV; 705 return -ENODEV;
706 706
707 ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL); 707 ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL);
@@ -714,7 +714,7 @@ int intel_fbdev_init(struct drm_device *dev)
714 ifbdev->preferred_bpp = 32; 714 ifbdev->preferred_bpp = 32;
715 715
716 ret = drm_fb_helper_init(dev, &ifbdev->helper, 716 ret = drm_fb_helper_init(dev, &ifbdev->helper,
717 INTEL_INFO(dev)->num_pipes, 4); 717 INTEL_INFO(dev_priv)->num_pipes, 4);
718 if (ret) { 718 if (ret) {
719 kfree(ifbdev); 719 kfree(ifbdev);
720 return ret; 720 return ret;
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.h b/drivers/gpu/drm/i915/intel_frontbuffer.h
index 76ceb539f9f0..7bab41218cf7 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.h
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.h
@@ -53,16 +53,17 @@ void __intel_fb_obj_flush(struct drm_i915_gem_object *obj,
53 * until the rendering completes or a flip on this frontbuffer plane is 53 * until the rendering completes or a flip on this frontbuffer plane is
54 * scheduled. 54 * scheduled.
55 */ 55 */
56static inline void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj, 56static inline bool intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
57 enum fb_op_origin origin) 57 enum fb_op_origin origin)
58{ 58{
59 unsigned int frontbuffer_bits; 59 unsigned int frontbuffer_bits;
60 60
61 frontbuffer_bits = atomic_read(&obj->frontbuffer_bits); 61 frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
62 if (!frontbuffer_bits) 62 if (!frontbuffer_bits)
63 return; 63 return false;
64 64
65 __intel_fb_obj_invalidate(obj, origin, frontbuffer_bits); 65 __intel_fb_obj_invalidate(obj, origin, frontbuffer_bits);
66 return true;
66} 67}
67 68
68/** 69/**
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 1aa85236b788..34d6ad2cf7c1 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -566,7 +566,7 @@ fail:
566 ret = 0; 566 ret = 0;
567 } 567 }
568 568
569 if (err == 0 && !HAS_GUC_UCODE(dev)) 569 if (err == 0 && !HAS_GUC_UCODE(dev_priv))
570 ; /* Don't mention the GuC! */ 570 ; /* Don't mention the GuC! */
571 else if (err == 0) 571 else if (err == 0)
572 DRM_INFO("GuC firmware load skipped\n"); 572 DRM_INFO("GuC firmware load skipped\n");
@@ -725,18 +725,18 @@ void intel_guc_init(struct drm_device *dev)
725 struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; 725 struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
726 const char *fw_path; 726 const char *fw_path;
727 727
728 if (!HAS_GUC(dev)) { 728 if (!HAS_GUC(dev_priv)) {
729 i915.enable_guc_loading = 0; 729 i915.enable_guc_loading = 0;
730 i915.enable_guc_submission = 0; 730 i915.enable_guc_submission = 0;
731 } else { 731 } else {
732 /* A negative value means "use platform default" */ 732 /* A negative value means "use platform default" */
733 if (i915.enable_guc_loading < 0) 733 if (i915.enable_guc_loading < 0)
734 i915.enable_guc_loading = HAS_GUC_UCODE(dev); 734 i915.enable_guc_loading = HAS_GUC_UCODE(dev_priv);
735 if (i915.enable_guc_submission < 0) 735 if (i915.enable_guc_submission < 0)
736 i915.enable_guc_submission = HAS_GUC_SCHED(dev); 736 i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
737 } 737 }
738 738
739 if (!HAS_GUC_UCODE(dev)) { 739 if (!HAS_GUC_UCODE(dev_priv)) {
740 fw_path = NULL; 740 fw_path = NULL;
741 } else if (IS_SKYLAKE(dev_priv)) { 741 } else if (IS_SKYLAKE(dev_priv)) {
742 fw_path = I915_SKL_GUC_UCODE; 742 fw_path = I915_SKL_GUC_UCODE;
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 35ada4e1c6cf..fb88e32e25a3 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -975,14 +975,16 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
975 pipe_config->lane_count = 4; 975 pipe_config->lane_count = 4;
976} 976}
977 977
978static void intel_enable_hdmi_audio(struct intel_encoder *encoder) 978static void intel_enable_hdmi_audio(struct intel_encoder *encoder,
979 struct intel_crtc_state *pipe_config,
980 struct drm_connector_state *conn_state)
979{ 981{
980 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); 982 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
981 983
982 WARN_ON(!crtc->config->has_hdmi_sink); 984 WARN_ON(!crtc->config->has_hdmi_sink);
983 DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", 985 DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n",
984 pipe_name(crtc->pipe)); 986 pipe_name(crtc->pipe));
985 intel_audio_codec_enable(encoder); 987 intel_audio_codec_enable(encoder, pipe_config, conn_state);
986} 988}
987 989
988static void g4x_enable_hdmi(struct intel_encoder *encoder, 990static void g4x_enable_hdmi(struct intel_encoder *encoder,
@@ -991,21 +993,20 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder,
991{ 993{
992 struct drm_device *dev = encoder->base.dev; 994 struct drm_device *dev = encoder->base.dev;
993 struct drm_i915_private *dev_priv = to_i915(dev); 995 struct drm_i915_private *dev_priv = to_i915(dev);
994 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
995 struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); 996 struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
996 u32 temp; 997 u32 temp;
997 998
998 temp = I915_READ(intel_hdmi->hdmi_reg); 999 temp = I915_READ(intel_hdmi->hdmi_reg);
999 1000
1000 temp |= SDVO_ENABLE; 1001 temp |= SDVO_ENABLE;
1001 if (crtc->config->has_audio) 1002 if (pipe_config->has_audio)
1002 temp |= SDVO_AUDIO_ENABLE; 1003 temp |= SDVO_AUDIO_ENABLE;
1003 1004
1004 I915_WRITE(intel_hdmi->hdmi_reg, temp); 1005 I915_WRITE(intel_hdmi->hdmi_reg, temp);
1005 POSTING_READ(intel_hdmi->hdmi_reg); 1006 POSTING_READ(intel_hdmi->hdmi_reg);
1006 1007
1007 if (crtc->config->has_audio) 1008 if (pipe_config->has_audio)
1008 intel_enable_hdmi_audio(encoder); 1009 intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
1009} 1010}
1010 1011
1011static void ibx_enable_hdmi(struct intel_encoder *encoder, 1012static void ibx_enable_hdmi(struct intel_encoder *encoder,
@@ -1040,8 +1041,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder,
1040 * FIXME: BSpec says this should be done at the end of 1041 * FIXME: BSpec says this should be done at the end of
1041 * of the modeset sequence, so not sure if this isn't too soon. 1042 * of the modeset sequence, so not sure if this isn't too soon.
1042 */ 1043 */
1043 if (crtc->config->pipe_bpp > 24 && 1044 if (pipe_config->pipe_bpp > 24 &&
1044 crtc->config->pixel_multiplier > 1) { 1045 pipe_config->pixel_multiplier > 1) {
1045 I915_WRITE(intel_hdmi->hdmi_reg, temp & ~SDVO_ENABLE); 1046 I915_WRITE(intel_hdmi->hdmi_reg, temp & ~SDVO_ENABLE);
1046 POSTING_READ(intel_hdmi->hdmi_reg); 1047 POSTING_READ(intel_hdmi->hdmi_reg);
1047 1048
@@ -1055,8 +1056,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder,
1055 POSTING_READ(intel_hdmi->hdmi_reg); 1056 POSTING_READ(intel_hdmi->hdmi_reg);
1056 } 1057 }
1057 1058
1058 if (crtc->config->has_audio) 1059 if (pipe_config->has_audio)
1059 intel_enable_hdmi_audio(encoder); 1060 intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
1060} 1061}
1061 1062
1062static void cpt_enable_hdmi(struct intel_encoder *encoder, 1063static void cpt_enable_hdmi(struct intel_encoder *encoder,
@@ -1073,7 +1074,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
1073 temp = I915_READ(intel_hdmi->hdmi_reg); 1074 temp = I915_READ(intel_hdmi->hdmi_reg);
1074 1075
1075 temp |= SDVO_ENABLE; 1076 temp |= SDVO_ENABLE;
1076 if (crtc->config->has_audio) 1077 if (pipe_config->has_audio)
1077 temp |= SDVO_AUDIO_ENABLE; 1078 temp |= SDVO_AUDIO_ENABLE;
1078 1079
1079 /* 1080 /*
@@ -1086,7 +1087,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
1086 * 4. enable HDMI clock gating 1087 * 4. enable HDMI clock gating
1087 */ 1088 */
1088 1089
1089 if (crtc->config->pipe_bpp > 24) { 1090 if (pipe_config->pipe_bpp > 24) {
1090 I915_WRITE(TRANS_CHICKEN1(pipe), 1091 I915_WRITE(TRANS_CHICKEN1(pipe),
1091 I915_READ(TRANS_CHICKEN1(pipe)) | 1092 I915_READ(TRANS_CHICKEN1(pipe)) |
1092 TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); 1093 TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
@@ -1098,7 +1099,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
1098 I915_WRITE(intel_hdmi->hdmi_reg, temp); 1099 I915_WRITE(intel_hdmi->hdmi_reg, temp);
1099 POSTING_READ(intel_hdmi->hdmi_reg); 1100 POSTING_READ(intel_hdmi->hdmi_reg);
1100 1101
1101 if (crtc->config->pipe_bpp > 24) { 1102 if (pipe_config->pipe_bpp > 24) {
1102 temp &= ~SDVO_COLOR_FORMAT_MASK; 1103 temp &= ~SDVO_COLOR_FORMAT_MASK;
1103 temp |= HDMI_COLOR_FORMAT_12bpc; 1104 temp |= HDMI_COLOR_FORMAT_12bpc;
1104 1105
@@ -1110,8 +1111,8 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
1110 ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE); 1111 ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
1111 } 1112 }
1112 1113
1113 if (crtc->config->has_audio) 1114 if (pipe_config->has_audio)
1114 intel_enable_hdmi_audio(encoder); 1115 intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
1115} 1116}
1116 1117
1117static void vlv_enable_hdmi(struct intel_encoder *encoder, 1118static void vlv_enable_hdmi(struct intel_encoder *encoder,
@@ -1178,9 +1179,7 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder,
1178 struct intel_crtc_state *old_crtc_state, 1179 struct intel_crtc_state *old_crtc_state,
1179 struct drm_connector_state *old_conn_state) 1180 struct drm_connector_state *old_conn_state)
1180{ 1181{
1181 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); 1182 if (old_crtc_state->has_audio)
1182
1183 if (crtc->config->has_audio)
1184 intel_audio_codec_disable(encoder); 1183 intel_audio_codec_disable(encoder);
1185 1184
1186 intel_disable_hdmi(encoder, old_crtc_state, old_conn_state); 1185 intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
@@ -1190,9 +1189,7 @@ static void pch_disable_hdmi(struct intel_encoder *encoder,
1190 struct intel_crtc_state *old_crtc_state, 1189 struct intel_crtc_state *old_crtc_state,
1191 struct drm_connector_state *old_conn_state) 1190 struct drm_connector_state *old_conn_state)
1192{ 1191{
1193 struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); 1192 if (old_crtc_state->has_audio)
1194
1195 if (crtc->config->has_audio)
1196 intel_audio_codec_disable(encoder); 1193 intel_audio_codec_disable(encoder);
1197} 1194}
1198 1195
@@ -1645,13 +1642,12 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder,
1645 struct drm_connector_state *conn_state) 1642 struct drm_connector_state *conn_state)
1646{ 1643{
1647 struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); 1644 struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
1648 struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); 1645 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
1649 const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
1650 1646
1651 intel_hdmi_prepare(encoder); 1647 intel_hdmi_prepare(encoder);
1652 1648
1653 intel_hdmi->set_infoframes(&encoder->base, 1649 intel_hdmi->set_infoframes(&encoder->base,
1654 intel_crtc->config->has_hdmi_sink, 1650 pipe_config->has_hdmi_sink,
1655 adjusted_mode); 1651 adjusted_mode);
1656} 1652}
1657 1653
@@ -1663,9 +1659,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
1663 struct intel_hdmi *intel_hdmi = &dport->hdmi; 1659 struct intel_hdmi *intel_hdmi = &dport->hdmi;
1664 struct drm_device *dev = encoder->base.dev; 1660 struct drm_device *dev = encoder->base.dev;
1665 struct drm_i915_private *dev_priv = to_i915(dev); 1661 struct drm_i915_private *dev_priv = to_i915(dev);
1666 struct intel_crtc *intel_crtc = 1662 const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
1667 to_intel_crtc(encoder->base.crtc);
1668 const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
1669 1663
1670 vlv_phy_pre_encoder_enable(encoder); 1664 vlv_phy_pre_encoder_enable(encoder);
1671 1665
@@ -1674,7 +1668,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
1674 0x2b247878); 1668 0x2b247878);
1675 1669
1676 intel_hdmi->set_infoframes(&encoder->base, 1670 intel_hdmi->set_infoframes(&encoder->base,
1677 intel_crtc->config->has_hdmi_sink, 1671 pipe_config->has_hdmi_sink,
1678 adjusted_mode); 1672 adjusted_mode);
1679 1673
1680 g4x_enable_hdmi(encoder, pipe_config, conn_state); 1674 g4x_enable_hdmi(encoder, pipe_config, conn_state);
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index 334d47b5811a..3d546c019de0 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -501,7 +501,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work)
501 if (intel_connector->mst_port) 501 if (intel_connector->mst_port)
502 continue; 502 continue;
503 503
504 if (!connector->polled && I915_HAS_HOTPLUG(dev) && 504 if (!connector->polled && I915_HAS_HOTPLUG(dev_priv) &&
505 intel_connector->encoder->hpd_pin > HPD_NONE) { 505 intel_connector->encoder->hpd_pin > HPD_NONE) {
506 connector->polled = enabled ? 506 connector->polled = enabled ?
507 DRM_CONNECTOR_POLL_CONNECT | 507 DRM_CONNECTOR_POLL_CONNECT |
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index dde04b7643b1..0a09024d6ca3 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -432,8 +432,10 @@ static bool can_merge_ctx(const struct i915_gem_context *prev,
432 432
433static void execlists_dequeue(struct intel_engine_cs *engine) 433static void execlists_dequeue(struct intel_engine_cs *engine)
434{ 434{
435 struct drm_i915_gem_request *cursor, *last; 435 struct drm_i915_gem_request *last;
436 struct execlist_port *port = engine->execlist_port; 436 struct execlist_port *port = engine->execlist_port;
437 unsigned long flags;
438 struct rb_node *rb;
437 bool submit = false; 439 bool submit = false;
438 440
439 last = port->request; 441 last = port->request;
@@ -469,8 +471,12 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
469 * and context switches) submission. 471 * and context switches) submission.
470 */ 472 */
471 473
472 spin_lock(&engine->execlist_lock); 474 spin_lock_irqsave(&engine->timeline->lock, flags);
473 list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) { 475 rb = engine->execlist_first;
476 while (rb) {
477 struct drm_i915_gem_request *cursor =
478 rb_entry(rb, typeof(*cursor), priotree.node);
479
474 /* Can we combine this request with the current port? It has to 480 /* Can we combine this request with the current port? It has to
475 * be the same context/ringbuffer and not have any exceptions 481 * be the same context/ringbuffer and not have any exceptions
476 * (e.g. GVT saying never to combine contexts). 482 * (e.g. GVT saying never to combine contexts).
@@ -493,7 +499,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
493 * context (even though a different request) to 499 * context (even though a different request) to
494 * the second port. 500 * the second port.
495 */ 501 */
496 if (ctx_single_port_submission(cursor->ctx)) 502 if (ctx_single_port_submission(last->ctx) ||
503 ctx_single_port_submission(cursor->ctx))
497 break; 504 break;
498 505
499 GEM_BUG_ON(last->ctx == cursor->ctx); 506 GEM_BUG_ON(last->ctx == cursor->ctx);
@@ -501,17 +508,30 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
501 i915_gem_request_assign(&port->request, last); 508 i915_gem_request_assign(&port->request, last);
502 port++; 509 port++;
503 } 510 }
511
512 rb = rb_next(rb);
513 rb_erase(&cursor->priotree.node, &engine->execlist_queue);
514 RB_CLEAR_NODE(&cursor->priotree.node);
515 cursor->priotree.priority = INT_MAX;
516
517 /* We keep the previous context alive until we retire the
518 * following request. This ensures that any the context object
519 * is still pinned for any residual writes the HW makes into it
520 * on the context switch into the next object following the
521 * breadcrumb. Otherwise, we may retire the context too early.
522 */
523 cursor->previous_context = engine->last_context;
524 engine->last_context = cursor->ctx;
525
526 __i915_gem_request_submit(cursor);
504 last = cursor; 527 last = cursor;
505 submit = true; 528 submit = true;
506 } 529 }
507 if (submit) { 530 if (submit) {
508 /* Decouple all the requests submitted from the queue */
509 engine->execlist_queue.next = &cursor->execlist_link;
510 cursor->execlist_link.prev = &engine->execlist_queue;
511
512 i915_gem_request_assign(&port->request, last); 531 i915_gem_request_assign(&port->request, last);
532 engine->execlist_first = rb;
513 } 533 }
514 spin_unlock(&engine->execlist_lock); 534 spin_unlock_irqrestore(&engine->timeline->lock, flags);
515 535
516 if (submit) 536 if (submit)
517 execlists_submit_ports(engine); 537 execlists_submit_ports(engine);
@@ -614,27 +634,147 @@ static void intel_lrc_irq_handler(unsigned long data)
614 intel_uncore_forcewake_put(dev_priv, engine->fw_domains); 634 intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
615} 635}
616 636
637static bool insert_request(struct i915_priotree *pt, struct rb_root *root)
638{
639 struct rb_node **p, *rb;
640 bool first = true;
641
642 /* most positive priority is scheduled first, equal priorities fifo */
643 rb = NULL;
644 p = &root->rb_node;
645 while (*p) {
646 struct i915_priotree *pos;
647
648 rb = *p;
649 pos = rb_entry(rb, typeof(*pos), node);
650 if (pt->priority > pos->priority) {
651 p = &rb->rb_left;
652 } else {
653 p = &rb->rb_right;
654 first = false;
655 }
656 }
657 rb_link_node(&pt->node, rb, p);
658 rb_insert_color(&pt->node, root);
659
660 return first;
661}
662
617static void execlists_submit_request(struct drm_i915_gem_request *request) 663static void execlists_submit_request(struct drm_i915_gem_request *request)
618{ 664{
619 struct intel_engine_cs *engine = request->engine; 665 struct intel_engine_cs *engine = request->engine;
620 unsigned long flags; 666 unsigned long flags;
621 667
622 spin_lock_irqsave(&engine->execlist_lock, flags); 668 /* Will be called from irq-context when using foreign fences. */
669 spin_lock_irqsave(&engine->timeline->lock, flags);
623 670
624 /* We keep the previous context alive until we retire the following 671 if (insert_request(&request->priotree, &engine->execlist_queue))
625 * request. This ensures that any the context object is still pinned 672 engine->execlist_first = &request->priotree.node;
626 * for any residual writes the HW makes into it on the context switch
627 * into the next object following the breadcrumb. Otherwise, we may
628 * retire the context too early.
629 */
630 request->previous_context = engine->last_context;
631 engine->last_context = request->ctx;
632
633 list_add_tail(&request->execlist_link, &engine->execlist_queue);
634 if (execlists_elsp_idle(engine)) 673 if (execlists_elsp_idle(engine))
635 tasklet_hi_schedule(&engine->irq_tasklet); 674 tasklet_hi_schedule(&engine->irq_tasklet);
636 675
637 spin_unlock_irqrestore(&engine->execlist_lock, flags); 676 spin_unlock_irqrestore(&engine->timeline->lock, flags);
677}
678
679static struct intel_engine_cs *
680pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
681{
682 struct intel_engine_cs *engine;
683
684 engine = container_of(pt,
685 struct drm_i915_gem_request,
686 priotree)->engine;
687 if (engine != locked) {
688 if (locked)
689 spin_unlock_irq(&locked->timeline->lock);
690 spin_lock_irq(&engine->timeline->lock);
691 }
692
693 return engine;
694}
695
696static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
697{
698 static DEFINE_MUTEX(lock);
699 struct intel_engine_cs *engine = NULL;
700 struct i915_dependency *dep, *p;
701 struct i915_dependency stack;
702 LIST_HEAD(dfs);
703
704 if (prio <= READ_ONCE(request->priotree.priority))
705 return;
706
707 /* Need global lock to use the temporary link inside i915_dependency */
708 mutex_lock(&lock);
709
710 stack.signaler = &request->priotree;
711 list_add(&stack.dfs_link, &dfs);
712
713 /* Recursively bump all dependent priorities to match the new request.
714 *
715 * A naive approach would be to use recursion:
716 * static void update_priorities(struct i915_priotree *pt, prio) {
717 * list_for_each_entry(dep, &pt->signalers_list, signal_link)
718 * update_priorities(dep->signal, prio)
719 * insert_request(pt);
720 * }
721 * but that may have unlimited recursion depth and so runs a very
722 * real risk of overunning the kernel stack. Instead, we build
723 * a flat list of all dependencies starting with the current request.
724 * As we walk the list of dependencies, we add all of its dependencies
725 * to the end of the list (this may include an already visited
726 * request) and continue to walk onwards onto the new dependencies. The
727 * end result is a topological list of requests in reverse order, the
728 * last element in the list is the request we must execute first.
729 */
730 list_for_each_entry_safe(dep, p, &dfs, dfs_link) {
731 struct i915_priotree *pt = dep->signaler;
732
733 list_for_each_entry(p, &pt->signalers_list, signal_link)
734 if (prio > READ_ONCE(p->signaler->priority))
735 list_move_tail(&p->dfs_link, &dfs);
736
737 p = list_next_entry(dep, dfs_link);
738 if (!RB_EMPTY_NODE(&pt->node))
739 continue;
740
741 engine = pt_lock_engine(pt, engine);
742
743 /* If it is not already in the rbtree, we can update the
744 * priority inplace and skip over it (and its dependencies)
745 * if it is referenced *again* as we descend the dfs.
746 */
747 if (prio > pt->priority && RB_EMPTY_NODE(&pt->node)) {
748 pt->priority = prio;
749 list_del_init(&dep->dfs_link);
750 }
751 }
752
753 /* Fifo and depth-first replacement ensure our deps execute before us */
754 list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
755 struct i915_priotree *pt = dep->signaler;
756
757 INIT_LIST_HEAD(&dep->dfs_link);
758
759 engine = pt_lock_engine(pt, engine);
760
761 if (prio <= pt->priority)
762 continue;
763
764 GEM_BUG_ON(RB_EMPTY_NODE(&pt->node));
765
766 pt->priority = prio;
767 rb_erase(&pt->node, &engine->execlist_queue);
768 if (insert_request(pt, &engine->execlist_queue))
769 engine->execlist_first = &pt->node;
770 }
771
772 if (engine)
773 spin_unlock_irq(&engine->timeline->lock);
774
775 mutex_unlock(&lock);
776
777 /* XXX Do we need to preempt to make room for us and our deps? */
638} 778}
639 779
640int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) 780int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
@@ -1673,8 +1813,10 @@ void intel_execlists_enable_submission(struct drm_i915_private *dev_priv)
1673 struct intel_engine_cs *engine; 1813 struct intel_engine_cs *engine;
1674 enum intel_engine_id id; 1814 enum intel_engine_id id;
1675 1815
1676 for_each_engine(engine, dev_priv, id) 1816 for_each_engine(engine, dev_priv, id) {
1677 engine->submit_request = execlists_submit_request; 1817 engine->submit_request = execlists_submit_request;
1818 engine->schedule = execlists_schedule;
1819 }
1678} 1820}
1679 1821
1680static void 1822static void
@@ -1687,6 +1829,7 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1687 engine->emit_breadcrumb = gen8_emit_breadcrumb; 1829 engine->emit_breadcrumb = gen8_emit_breadcrumb;
1688 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; 1830 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
1689 engine->submit_request = execlists_submit_request; 1831 engine->submit_request = execlists_submit_request;
1832 engine->schedule = execlists_schedule;
1690 1833
1691 engine->irq_enable = gen8_logical_ring_enable_irq; 1834 engine->irq_enable = gen8_logical_ring_enable_irq;
1692 engine->irq_disable = gen8_logical_ring_disable_irq; 1835 engine->irq_disable = gen8_logical_ring_disable_irq;
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index de7b3e6ed477..d12ef0047d49 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -122,8 +122,7 @@ out:
122static void intel_lvds_get_config(struct intel_encoder *encoder, 122static void intel_lvds_get_config(struct intel_encoder *encoder,
123 struct intel_crtc_state *pipe_config) 123 struct intel_crtc_state *pipe_config)
124{ 124{
125 struct drm_device *dev = encoder->base.dev; 125 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
126 struct drm_i915_private *dev_priv = to_i915(dev);
127 struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); 126 struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
128 u32 tmp, flags = 0; 127 u32 tmp, flags = 0;
129 128
@@ -139,12 +138,12 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
139 138
140 pipe_config->base.adjusted_mode.flags |= flags; 139 pipe_config->base.adjusted_mode.flags |= flags;
141 140
142 if (INTEL_INFO(dev)->gen < 5) 141 if (INTEL_GEN(dev_priv) < 5)
143 pipe_config->gmch_pfit.lvds_border_bits = 142 pipe_config->gmch_pfit.lvds_border_bits =
144 tmp & LVDS_BORDER_ENABLE; 143 tmp & LVDS_BORDER_ENABLE;
145 144
146 /* gen2/3 store dither state in pfit control, needs to match */ 145 /* gen2/3 store dither state in pfit control, needs to match */
147 if (INTEL_INFO(dev)->gen < 4) { 146 if (INTEL_GEN(dev_priv) < 4) {
148 tmp = I915_READ(PFIT_CONTROL); 147 tmp = I915_READ(PFIT_CONTROL);
149 148
150 pipe_config->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE; 149 pipe_config->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 7acbbbf97833..f4429f67a4e3 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -642,24 +642,6 @@ static struct notifier_block intel_opregion_notifier = {
642 * (version 3) 642 * (version 3)
643 */ 643 */
644 644
645static u32 get_did(struct intel_opregion *opregion, int i)
646{
647 u32 did;
648
649 if (i < ARRAY_SIZE(opregion->acpi->didl)) {
650 did = opregion->acpi->didl[i];
651 } else {
652 i -= ARRAY_SIZE(opregion->acpi->didl);
653
654 if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2)))
655 return 0;
656
657 did = opregion->acpi->did2[i];
658 }
659
660 return did;
661}
662
663static void set_did(struct intel_opregion *opregion, int i, u32 val) 645static void set_did(struct intel_opregion *opregion, int i, u32 val)
664{ 646{
665 if (i < ARRAY_SIZE(opregion->acpi->didl)) { 647 if (i < ARRAY_SIZE(opregion->acpi->didl)) {
@@ -674,11 +656,11 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val)
674 } 656 }
675} 657}
676 658
677static u32 acpi_display_type(struct drm_connector *connector) 659static u32 acpi_display_type(struct intel_connector *connector)
678{ 660{
679 u32 display_type; 661 u32 display_type;
680 662
681 switch (connector->connector_type) { 663 switch (connector->base.connector_type) {
682 case DRM_MODE_CONNECTOR_VGA: 664 case DRM_MODE_CONNECTOR_VGA:
683 case DRM_MODE_CONNECTOR_DVIA: 665 case DRM_MODE_CONNECTOR_DVIA:
684 display_type = ACPI_DISPLAY_TYPE_VGA; 666 display_type = ACPI_DISPLAY_TYPE_VGA;
@@ -707,7 +689,7 @@ static u32 acpi_display_type(struct drm_connector *connector)
707 display_type = ACPI_DISPLAY_TYPE_OTHER; 689 display_type = ACPI_DISPLAY_TYPE_OTHER;
708 break; 690 break;
709 default: 691 default:
710 MISSING_CASE(connector->connector_type); 692 MISSING_CASE(connector->base.connector_type);
711 display_type = ACPI_DISPLAY_TYPE_OTHER; 693 display_type = ACPI_DISPLAY_TYPE_OTHER;
712 break; 694 break;
713 } 695 }
@@ -718,34 +700,9 @@ static u32 acpi_display_type(struct drm_connector *connector)
718static void intel_didl_outputs(struct drm_i915_private *dev_priv) 700static void intel_didl_outputs(struct drm_i915_private *dev_priv)
719{ 701{
720 struct intel_opregion *opregion = &dev_priv->opregion; 702 struct intel_opregion *opregion = &dev_priv->opregion;
721 struct pci_dev *pdev = dev_priv->drm.pdev; 703 struct intel_connector *connector;
722 struct drm_connector *connector; 704 int i = 0, max_outputs;
723 acpi_handle handle; 705 int display_index[16] = {};
724 struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL;
725 unsigned long long device_id;
726 acpi_status status;
727 u32 temp, max_outputs;
728 int i = 0;
729
730 handle = ACPI_HANDLE(&pdev->dev);
731 if (!handle || acpi_bus_get_device(handle, &acpi_dev))
732 return;
733
734 if (acpi_is_video_device(handle))
735 acpi_video_bus = acpi_dev;
736 else {
737 list_for_each_entry(acpi_cdev, &acpi_dev->children, node) {
738 if (acpi_is_video_device(acpi_cdev->handle)) {
739 acpi_video_bus = acpi_cdev;
740 break;
741 }
742 }
743 }
744
745 if (!acpi_video_bus) {
746 DRM_DEBUG_KMS("No ACPI video bus found\n");
747 return;
748 }
749 706
750 /* 707 /*
751 * In theory, did2, the extended didl, gets added at opregion version 708 * In theory, did2, the extended didl, gets added at opregion version
@@ -757,64 +714,58 @@ static void intel_didl_outputs(struct drm_i915_private *dev_priv)
757 max_outputs = ARRAY_SIZE(opregion->acpi->didl) + 714 max_outputs = ARRAY_SIZE(opregion->acpi->didl) +
758 ARRAY_SIZE(opregion->acpi->did2); 715 ARRAY_SIZE(opregion->acpi->did2);
759 716
760 list_for_each_entry(acpi_cdev, &acpi_video_bus->children, node) { 717 for_each_intel_connector(&dev_priv->drm, connector) {
761 if (i >= max_outputs) { 718 u32 device_id, type;
762 DRM_DEBUG_KMS("More than %u outputs detected via ACPI\n", 719
763 max_outputs); 720 device_id = acpi_display_type(connector);
764 return; 721
765 } 722 /* Use display type specific display index. */
766 status = acpi_evaluate_integer(acpi_cdev->handle, "_ADR", 723 type = (device_id & ACPI_DISPLAY_TYPE_MASK)
767 NULL, &device_id); 724 >> ACPI_DISPLAY_TYPE_SHIFT;
768 if (ACPI_SUCCESS(status)) { 725 device_id |= display_index[type]++ << ACPI_DISPLAY_INDEX_SHIFT;
769 if (!device_id) 726
770 goto blind_set; 727 connector->acpi_device_id = device_id;
771 set_did(opregion, i++, (u32)(device_id & 0x0f0f)); 728 if (i < max_outputs)
772 } 729 set_did(opregion, i, device_id);
730 i++;
773 } 731 }
774 732
775end:
776 DRM_DEBUG_KMS("%d outputs detected\n", i); 733 DRM_DEBUG_KMS("%d outputs detected\n", i);
777 734
735 if (i > max_outputs)
736 DRM_ERROR("More than %d outputs in connector list\n",
737 max_outputs);
738
778 /* If fewer than max outputs, the list must be null terminated */ 739 /* If fewer than max outputs, the list must be null terminated */
779 if (i < max_outputs) 740 if (i < max_outputs)
780 set_did(opregion, i, 0); 741 set_did(opregion, i, 0);
781 return;
782
783blind_set:
784 i = 0;
785 list_for_each_entry(connector,
786 &dev_priv->drm.mode_config.connector_list, head) {
787 int display_type = acpi_display_type(connector);
788
789 if (i >= max_outputs) {
790 DRM_DEBUG_KMS("More than %u outputs in connector list\n",
791 max_outputs);
792 return;
793 }
794
795 temp = get_did(opregion, i);
796 set_did(opregion, i, temp | (1 << 31) | display_type | i);
797 i++;
798 }
799 goto end;
800} 742}
801 743
802static void intel_setup_cadls(struct drm_i915_private *dev_priv) 744static void intel_setup_cadls(struct drm_i915_private *dev_priv)
803{ 745{
804 struct intel_opregion *opregion = &dev_priv->opregion; 746 struct intel_opregion *opregion = &dev_priv->opregion;
747 struct intel_connector *connector;
805 int i = 0; 748 int i = 0;
806 u32 disp_id; 749
807 750 /*
808 /* Initialize the CADL field by duplicating the DIDL values. 751 * Initialize the CADL field from the connector device ids. This is
809 * Technically, this is not always correct as display outputs may exist, 752 * essentially the same as copying from the DIDL. Technically, this is
810 * but not active. This initialization is necessary for some Clevo 753 * not always correct as display outputs may exist, but not active. This
811 * laptops that check this field before processing the brightness and 754 * initialization is necessary for some Clevo laptops that check this
812 * display switching hotkeys. Just like DIDL, CADL is NULL-terminated if 755 * field before processing the brightness and display switching hotkeys.
813 * there are less than eight devices. */ 756 *
814 do { 757 * Note that internal panels should be at the front of the connector
815 disp_id = get_did(opregion, i); 758 * list already, ensuring they're not left out.
816 opregion->acpi->cadl[i] = disp_id; 759 */
817 } while (++i < 8 && disp_id != 0); 760 for_each_intel_connector(&dev_priv->drm, connector) {
761 if (i >= ARRAY_SIZE(opregion->acpi->cadl))
762 break;
763 opregion->acpi->cadl[i++] = connector->acpi_device_id;
764 }
765
766 /* If fewer than 8 active devices, the list must be null terminated */
767 if (i < ARRAY_SIZE(opregion->acpi->cadl))
768 opregion->acpi->cadl[i] = 0;
818} 769}
819 770
820void intel_opregion_register(struct drm_i915_private *dev_priv) 771void intel_opregion_register(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index be4b4d546fd9..08ab6d762ca4 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -304,7 +304,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
304 struct intel_crtc_state *pipe_config, 304 struct intel_crtc_state *pipe_config,
305 int fitting_mode) 305 int fitting_mode)
306{ 306{
307 struct drm_device *dev = intel_crtc->base.dev; 307 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
308 u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0; 308 u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
309 struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; 309 struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
310 310
@@ -325,7 +325,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
325 break; 325 break;
326 case DRM_MODE_SCALE_ASPECT: 326 case DRM_MODE_SCALE_ASPECT:
327 /* Scale but preserve the aspect ratio */ 327 /* Scale but preserve the aspect ratio */
328 if (INTEL_INFO(dev)->gen >= 4) 328 if (INTEL_GEN(dev_priv) >= 4)
329 i965_scale_aspect(pipe_config, &pfit_control); 329 i965_scale_aspect(pipe_config, &pfit_control);
330 else 330 else
331 i9xx_scale_aspect(pipe_config, &pfit_control, 331 i9xx_scale_aspect(pipe_config, &pfit_control,
@@ -339,7 +339,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
339 if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay || 339 if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay ||
340 pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { 340 pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) {
341 pfit_control |= PFIT_ENABLE; 341 pfit_control |= PFIT_ENABLE;
342 if (INTEL_INFO(dev)->gen >= 4) 342 if (INTEL_GEN(dev_priv) >= 4)
343 pfit_control |= PFIT_SCALING_AUTO; 343 pfit_control |= PFIT_SCALING_AUTO;
344 else 344 else
345 pfit_control |= (VERT_AUTO_SCALE | 345 pfit_control |= (VERT_AUTO_SCALE |
@@ -355,7 +355,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
355 355
356 /* 965+ wants fuzzy fitting */ 356 /* 965+ wants fuzzy fitting */
357 /* FIXME: handle multiple panels by failing gracefully */ 357 /* FIXME: handle multiple panels by failing gracefully */
358 if (INTEL_INFO(dev)->gen >= 4) 358 if (INTEL_GEN(dev_priv) >= 4)
359 pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) | 359 pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) |
360 PFIT_FILTER_FUZZY); 360 PFIT_FILTER_FUZZY);
361 361
@@ -366,7 +366,7 @@ out:
366 } 366 }
367 367
368 /* Make sure pre-965 set dither correctly for 18bpp panels. */ 368 /* Make sure pre-965 set dither correctly for 18bpp panels. */
369 if (INTEL_INFO(dev)->gen < 4 && pipe_config->pipe_bpp == 18) 369 if (INTEL_GEN(dev_priv) < 4 && pipe_config->pipe_bpp == 18)
370 pfit_control |= PANEL_8TO6_DITHER_ENABLE; 370 pfit_control |= PANEL_8TO6_DITHER_ENABLE;
371 371
372 pipe_config->gmch_pfit.control = pfit_control; 372 pipe_config->gmch_pfit.control = pfit_control;
@@ -1722,7 +1722,7 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe)
1722 1722
1723 DRM_DEBUG_KMS("Connector %s backlight initialized, %s, brightness %u/%u\n", 1723 DRM_DEBUG_KMS("Connector %s backlight initialized, %s, brightness %u/%u\n",
1724 connector->name, 1724 connector->name,
1725 panel->backlight.enabled ? "enabled" : "disabled", 1725 enableddisabled(panel->backlight.enabled),
1726 panel->backlight.level, panel->backlight.max); 1726 panel->backlight.level, panel->backlight.max);
1727 1727
1728 return 0; 1728 return 0;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index cc9e0c0f445f..e207dc69e8b3 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -347,8 +347,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
347 return; 347 return;
348 } 348 }
349 349
350 DRM_DEBUG_KMS("memory self-refresh is %s\n", 350 DRM_DEBUG_KMS("memory self-refresh is %s\n", enableddisabled(enable));
351 enable ? "enabled" : "disabled");
352} 351}
353 352
354 353
@@ -1061,7 +1060,8 @@ static void vlv_invert_wms(struct intel_crtc *crtc)
1061 1060
1062 for (level = 0; level < wm_state->num_levels; level++) { 1061 for (level = 0; level < wm_state->num_levels; level++) {
1063 struct drm_device *dev = crtc->base.dev; 1062 struct drm_device *dev = crtc->base.dev;
1064 const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; 1063 const int sr_fifo_size =
1064 INTEL_INFO(to_i915(dev))->num_pipes * 512 - 1;
1065 struct intel_plane *plane; 1065 struct intel_plane *plane;
1066 1066
1067 wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane; 1067 wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
@@ -1091,15 +1091,16 @@ static void vlv_invert_wms(struct intel_crtc *crtc)
1091static void vlv_compute_wm(struct intel_crtc *crtc) 1091static void vlv_compute_wm(struct intel_crtc *crtc)
1092{ 1092{
1093 struct drm_device *dev = crtc->base.dev; 1093 struct drm_device *dev = crtc->base.dev;
1094 struct drm_i915_private *dev_priv = to_i915(dev);
1094 struct vlv_wm_state *wm_state = &crtc->wm_state; 1095 struct vlv_wm_state *wm_state = &crtc->wm_state;
1095 struct intel_plane *plane; 1096 struct intel_plane *plane;
1096 int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1; 1097 int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1097 int level; 1098 int level;
1098 1099
1099 memset(wm_state, 0, sizeof(*wm_state)); 1100 memset(wm_state, 0, sizeof(*wm_state));
1100 1101
1101 wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed; 1102 wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
1102 wm_state->num_levels = to_i915(dev)->wm.max_level + 1; 1103 wm_state->num_levels = dev_priv->wm.max_level + 1;
1103 1104
1104 wm_state->num_active_planes = 0; 1105 wm_state->num_active_planes = 0;
1105 1106
@@ -1179,7 +1180,7 @@ static void vlv_compute_wm(struct intel_crtc *crtc)
1179 } 1180 }
1180 1181
1181 /* clear any (partially) filled invalid levels */ 1182 /* clear any (partially) filled invalid levels */
1182 for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) { 1183 for (level = wm_state->num_levels; level < dev_priv->wm.max_level + 1; level++) {
1183 memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level])); 1184 memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
1184 memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level])); 1185 memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
1185 } 1186 }
@@ -1861,23 +1862,25 @@ static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1861 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp); 1862 return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
1862} 1863}
1863 1864
1864static unsigned int ilk_display_fifo_size(const struct drm_device *dev) 1865static unsigned int
1866ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
1865{ 1867{
1866 if (INTEL_INFO(dev)->gen >= 8) 1868 if (INTEL_GEN(dev_priv) >= 8)
1867 return 3072; 1869 return 3072;
1868 else if (INTEL_INFO(dev)->gen >= 7) 1870 else if (INTEL_GEN(dev_priv) >= 7)
1869 return 768; 1871 return 768;
1870 else 1872 else
1871 return 512; 1873 return 512;
1872} 1874}
1873 1875
1874static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev, 1876static unsigned int
1875 int level, bool is_sprite) 1877ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
1878 int level, bool is_sprite)
1876{ 1879{
1877 if (INTEL_INFO(dev)->gen >= 8) 1880 if (INTEL_GEN(dev_priv) >= 8)
1878 /* BDW primary/sprite plane watermarks */ 1881 /* BDW primary/sprite plane watermarks */
1879 return level == 0 ? 255 : 2047; 1882 return level == 0 ? 255 : 2047;
1880 else if (INTEL_INFO(dev)->gen >= 7) 1883 else if (INTEL_GEN(dev_priv) >= 7)
1881 /* IVB/HSW primary/sprite plane watermarks */ 1884 /* IVB/HSW primary/sprite plane watermarks */
1882 return level == 0 ? 127 : 1023; 1885 return level == 0 ? 127 : 1023;
1883 else if (!is_sprite) 1886 else if (!is_sprite)
@@ -1888,18 +1891,18 @@ static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1888 return level == 0 ? 63 : 255; 1891 return level == 0 ? 63 : 255;
1889} 1892}
1890 1893
1891static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev, 1894static unsigned int
1892 int level) 1895ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
1893{ 1896{
1894 if (INTEL_INFO(dev)->gen >= 7) 1897 if (INTEL_GEN(dev_priv) >= 7)
1895 return level == 0 ? 63 : 255; 1898 return level == 0 ? 63 : 255;
1896 else 1899 else
1897 return level == 0 ? 31 : 63; 1900 return level == 0 ? 31 : 63;
1898} 1901}
1899 1902
1900static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev) 1903static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
1901{ 1904{
1902 if (INTEL_INFO(dev)->gen >= 8) 1905 if (INTEL_GEN(dev_priv) >= 8)
1903 return 31; 1906 return 31;
1904 else 1907 else
1905 return 15; 1908 return 15;
@@ -1912,7 +1915,8 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1912 enum intel_ddb_partitioning ddb_partitioning, 1915 enum intel_ddb_partitioning ddb_partitioning,
1913 bool is_sprite) 1916 bool is_sprite)
1914{ 1917{
1915 unsigned int fifo_size = ilk_display_fifo_size(dev); 1918 struct drm_i915_private *dev_priv = to_i915(dev);
1919 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
1916 1920
1917 /* if sprites aren't enabled, sprites get nothing */ 1921 /* if sprites aren't enabled, sprites get nothing */
1918 if (is_sprite && !config->sprites_enabled) 1922 if (is_sprite && !config->sprites_enabled)
@@ -1920,14 +1924,14 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1920 1924
1921 /* HSW allows LP1+ watermarks even with multiple pipes */ 1925 /* HSW allows LP1+ watermarks even with multiple pipes */
1922 if (level == 0 || config->num_pipes_active > 1) { 1926 if (level == 0 || config->num_pipes_active > 1) {
1923 fifo_size /= INTEL_INFO(dev)->num_pipes; 1927 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
1924 1928
1925 /* 1929 /*
1926 * For some reason the non self refresh 1930 * For some reason the non self refresh
1927 * FIFO size is only half of the self 1931 * FIFO size is only half of the self
1928 * refresh FIFO size on ILK/SNB. 1932 * refresh FIFO size on ILK/SNB.
1929 */ 1933 */
1930 if (INTEL_INFO(dev)->gen <= 6) 1934 if (INTEL_GEN(dev_priv) <= 6)
1931 fifo_size /= 2; 1935 fifo_size /= 2;
1932 } 1936 }
1933 1937
@@ -1943,7 +1947,7 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1943 } 1947 }
1944 1948
1945 /* clamp to max that the registers can hold */ 1949 /* clamp to max that the registers can hold */
1946 return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite)); 1950 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
1947} 1951}
1948 1952
1949/* Calculate the maximum cursor plane watermark */ 1953/* Calculate the maximum cursor plane watermark */
@@ -1956,7 +1960,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1956 return 64; 1960 return 64;
1957 1961
1958 /* otherwise just report max that registers can hold */ 1962 /* otherwise just report max that registers can hold */
1959 return ilk_cursor_wm_reg_max(dev, level); 1963 return ilk_cursor_wm_reg_max(to_i915(dev), level);
1960} 1964}
1961 1965
1962static void ilk_compute_wm_maximums(const struct drm_device *dev, 1966static void ilk_compute_wm_maximums(const struct drm_device *dev,
@@ -1968,17 +1972,17 @@ static void ilk_compute_wm_maximums(const struct drm_device *dev,
1968 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false); 1972 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1969 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true); 1973 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1970 max->cur = ilk_cursor_wm_max(dev, level, config); 1974 max->cur = ilk_cursor_wm_max(dev, level, config);
1971 max->fbc = ilk_fbc_wm_reg_max(dev); 1975 max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
1972} 1976}
1973 1977
1974static void ilk_compute_wm_reg_maximums(struct drm_device *dev, 1978static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
1975 int level, 1979 int level,
1976 struct ilk_wm_maximums *max) 1980 struct ilk_wm_maximums *max)
1977{ 1981{
1978 max->pri = ilk_plane_wm_reg_max(dev, level, false); 1982 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
1979 max->spr = ilk_plane_wm_reg_max(dev, level, true); 1983 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
1980 max->cur = ilk_cursor_wm_reg_max(dev, level); 1984 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
1981 max->fbc = ilk_fbc_wm_reg_max(dev); 1985 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
1982} 1986}
1983 1987
1984static bool ilk_validate_wm_level(int level, 1988static bool ilk_validate_wm_level(int level,
@@ -2382,7 +2386,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2382 usable_level = max_level; 2386 usable_level = max_level;
2383 2387
2384 /* ILK/SNB: LP2+ watermarks only w/o sprites */ 2388 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2385 if (INTEL_INFO(dev)->gen <= 6 && pipe_wm->sprites_enabled) 2389 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
2386 usable_level = 1; 2390 usable_level = 1;
2387 2391
2388 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ 2392 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
@@ -2401,7 +2405,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
2401 if (!ilk_validate_pipe_wm(dev, pipe_wm)) 2405 if (!ilk_validate_pipe_wm(dev, pipe_wm))
2402 return -EINVAL; 2406 return -EINVAL;
2403 2407
2404 ilk_compute_wm_reg_maximums(dev, 1, &max); 2408 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2405 2409
2406 for (level = 1; level <= max_level; level++) { 2410 for (level = 1; level <= max_level; level++) {
2407 struct intel_wm_level *wm = &pipe_wm->raw_wm[level]; 2411 struct intel_wm_level *wm = &pipe_wm->raw_wm[level];
@@ -2530,7 +2534,7 @@ static void ilk_wm_merge(struct drm_device *dev,
2530 last_enabled_level = 0; 2534 last_enabled_level = 0;
2531 2535
2532 /* ILK: FBC WM must be disabled always */ 2536 /* ILK: FBC WM must be disabled always */
2533 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; 2537 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
2534 2538
2535 /* merge each WM1+ level */ 2539 /* merge each WM1+ level */
2536 for (level = 1; level <= max_level; level++) { 2540 for (level = 1; level <= max_level; level++) {
@@ -2593,6 +2597,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
2593 enum intel_ddb_partitioning partitioning, 2597 enum intel_ddb_partitioning partitioning,
2594 struct ilk_wm_values *results) 2598 struct ilk_wm_values *results)
2595{ 2599{
2600 struct drm_i915_private *dev_priv = to_i915(dev);
2596 struct intel_crtc *intel_crtc; 2601 struct intel_crtc *intel_crtc;
2597 int level, wm_lp; 2602 int level, wm_lp;
2598 2603
@@ -2619,7 +2624,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
2619 if (r->enable) 2624 if (r->enable)
2620 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN; 2625 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2621 2626
2622 if (INTEL_INFO(dev)->gen >= 8) 2627 if (INTEL_GEN(dev_priv) >= 8)
2623 results->wm_lp[wm_lp - 1] |= 2628 results->wm_lp[wm_lp - 1] |=
2624 r->fbc_val << WM1_LP_FBC_SHIFT_BDW; 2629 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2625 else 2630 else
@@ -2630,7 +2635,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
2630 * Always set WM1S_LP_EN when spr_val != 0, even if the 2635 * Always set WM1S_LP_EN when spr_val != 0, even if the
2631 * level is disabled. Doing otherwise could cause underruns. 2636 * level is disabled. Doing otherwise could cause underruns.
2632 */ 2637 */
2633 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) { 2638 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
2634 WARN_ON(wm_lp != 1); 2639 WARN_ON(wm_lp != 1);
2635 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val; 2640 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2636 } else 2641 } else
@@ -2780,7 +2785,6 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2780static void ilk_write_wm_values(struct drm_i915_private *dev_priv, 2785static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2781 struct ilk_wm_values *results) 2786 struct ilk_wm_values *results)
2782{ 2787{
2783 struct drm_device *dev = &dev_priv->drm;
2784 struct ilk_wm_values *previous = &dev_priv->wm.hw; 2788 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2785 unsigned int dirty; 2789 unsigned int dirty;
2786 uint32_t val; 2790 uint32_t val;
@@ -2836,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2836 previous->wm_lp_spr[0] != results->wm_lp_spr[0]) 2840 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2837 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]); 2841 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2838 2842
2839 if (INTEL_INFO(dev)->gen >= 7) { 2843 if (INTEL_GEN(dev_priv) >= 7) {
2840 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1]) 2844 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2841 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]); 2845 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2842 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2]) 2846 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
@@ -3118,7 +3122,11 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3118 * we currently hold. 3122 * we currently hold.
3119 */ 3123 */
3120 if (!intel_state->active_pipe_changes) { 3124 if (!intel_state->active_pipe_changes) {
3121 *alloc = to_intel_crtc(for_crtc)->hw_ddb; 3125 /*
3126 * alloc may be cleared by clear_intel_crtc_state,
3127 * copy from old state to be sure
3128 */
3129 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3122 return; 3130 return;
3123 } 3131 }
3124 3132
@@ -3624,6 +3632,9 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3624 y_min_scanlines = 4; 3632 y_min_scanlines = 4;
3625 } 3633 }
3626 3634
3635 if (apply_memory_bw_wa)
3636 y_min_scanlines *= 2;
3637
3627 plane_bytes_per_line = width * cpp; 3638 plane_bytes_per_line = width * cpp;
3628 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || 3639 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
3629 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { 3640 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
@@ -3644,8 +3655,6 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
3644 plane_blocks_per_line); 3655 plane_blocks_per_line);
3645 3656
3646 y_tile_minimum = plane_blocks_per_line * y_min_scanlines; 3657 y_tile_minimum = plane_blocks_per_line * y_min_scanlines;
3647 if (apply_memory_bw_wa)
3648 y_tile_minimum *= 2;
3649 3658
3650 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED || 3659 if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
3651 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { 3660 fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
@@ -3906,25 +3915,16 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
3906 return a->start < b->end && b->start < a->end; 3915 return a->start < b->end && b->start < a->end;
3907} 3916}
3908 3917
3909bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state, 3918bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
3910 struct intel_crtc *intel_crtc) 3919 const struct skl_ddb_entry *ddb,
3920 int ignore)
3911{ 3921{
3912 struct drm_crtc *other_crtc;
3913 struct drm_crtc_state *other_cstate;
3914 struct intel_crtc *other_intel_crtc;
3915 const struct skl_ddb_entry *ddb =
3916 &to_intel_crtc_state(intel_crtc->base.state)->wm.skl.ddb;
3917 int i; 3922 int i;
3918 3923
3919 for_each_crtc_in_state(state, other_crtc, other_cstate, i) { 3924 for (i = 0; i < I915_MAX_PIPES; i++)
3920 other_intel_crtc = to_intel_crtc(other_crtc); 3925 if (i != ignore && entries[i] &&
3921 3926 skl_ddb_entries_overlap(ddb, entries[i]))
3922 if (other_intel_crtc == intel_crtc)
3923 continue;
3924
3925 if (skl_ddb_entries_overlap(ddb, &other_intel_crtc->hw_ddb))
3926 return true; 3927 return true;
3927 }
3928 3928
3929 return false; 3929 return false;
3930} 3930}
@@ -4193,14 +4193,35 @@ skl_compute_wm(struct drm_atomic_state *state)
4193 return 0; 4193 return 0;
4194} 4194}
4195 4195
4196static void skl_update_wm(struct intel_crtc *intel_crtc) 4196static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
4197 struct intel_crtc_state *cstate)
4198{
4199 struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
4200 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4201 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
4202 const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
4203 enum pipe pipe = crtc->pipe;
4204 int plane;
4205
4206 if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
4207 return;
4208
4209 I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
4210
4211 for_each_universal_plane(dev_priv, pipe, plane)
4212 skl_write_plane_wm(crtc, &pipe_wm->planes[plane], ddb, plane);
4213
4214 skl_write_cursor_wm(crtc, &pipe_wm->planes[PLANE_CURSOR], ddb);
4215}
4216
4217static void skl_initial_wm(struct intel_atomic_state *state,
4218 struct intel_crtc_state *cstate)
4197{ 4219{
4220 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4198 struct drm_device *dev = intel_crtc->base.dev; 4221 struct drm_device *dev = intel_crtc->base.dev;
4199 struct drm_i915_private *dev_priv = to_i915(dev); 4222 struct drm_i915_private *dev_priv = to_i915(dev);
4200 struct skl_wm_values *results = &dev_priv->wm.skl_results; 4223 struct skl_wm_values *results = &state->wm_results;
4201 struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw; 4224 struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
4202 struct intel_crtc_state *cstate = to_intel_crtc_state(intel_crtc->base.state);
4203 struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
4204 enum pipe pipe = intel_crtc->pipe; 4225 enum pipe pipe = intel_crtc->pipe;
4205 4226
4206 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0) 4227 if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
@@ -4208,27 +4229,11 @@ static void skl_update_wm(struct intel_crtc *intel_crtc)
4208 4229
4209 mutex_lock(&dev_priv->wm.wm_mutex); 4230 mutex_lock(&dev_priv->wm.wm_mutex);
4210 4231
4211 /* 4232 if (cstate->base.active_changed)
4212 * If this pipe isn't active already, we're going to be enabling it 4233 skl_atomic_update_crtc_wm(state, cstate);
4213 * very soon. Since it's safe to update a pipe's ddb allocation while
4214 * the pipe's shut off, just do so here. Already active pipes will have
4215 * their watermarks updated once we update their planes.
4216 */
4217 if (intel_crtc->base.state->active_changed) {
4218 int plane;
4219
4220 for_each_universal_plane(dev_priv, pipe, plane)
4221 skl_write_plane_wm(intel_crtc, &pipe_wm->planes[plane],
4222 &results->ddb, plane);
4223
4224 skl_write_cursor_wm(intel_crtc, &pipe_wm->planes[PLANE_CURSOR],
4225 &results->ddb);
4226 }
4227 4234
4228 skl_copy_wm_for_pipe(hw_vals, results, pipe); 4235 skl_copy_wm_for_pipe(hw_vals, results, pipe);
4229 4236
4230 intel_crtc->hw_ddb = cstate->wm.skl.ddb;
4231
4232 mutex_unlock(&dev_priv->wm.wm_mutex); 4237 mutex_unlock(&dev_priv->wm.wm_mutex);
4233} 4238}
4234 4239
@@ -4265,7 +4270,7 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
4265 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2); 4270 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
4266 4271
4267 /* 5/6 split only in single pipe config on IVB+ */ 4272 /* 5/6 split only in single pipe config on IVB+ */
4268 if (INTEL_INFO(dev)->gen >= 7 && 4273 if (INTEL_GEN(dev_priv) >= 7 &&
4269 config.num_pipes_active == 1 && config.sprites_enabled) { 4274 config.num_pipes_active == 1 && config.sprites_enabled) {
4270 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max); 4275 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
4271 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6); 4276 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
@@ -4283,7 +4288,8 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
4283 ilk_write_wm_values(dev_priv, &results); 4288 ilk_write_wm_values(dev_priv, &results);
4284} 4289}
4285 4290
4286static void ilk_initial_watermarks(struct intel_crtc_state *cstate) 4291static void ilk_initial_watermarks(struct intel_atomic_state *state,
4292 struct intel_crtc_state *cstate)
4287{ 4293{
4288 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); 4294 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4289 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 4295 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
@@ -4294,7 +4300,8 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate)
4294 mutex_unlock(&dev_priv->wm.wm_mutex); 4300 mutex_unlock(&dev_priv->wm.wm_mutex);
4295} 4301}
4296 4302
4297static void ilk_optimize_watermarks(struct intel_crtc_state *cstate) 4303static void ilk_optimize_watermarks(struct intel_atomic_state *state,
4304 struct intel_crtc_state *cstate)
4298{ 4305{
4299 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); 4306 struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
4300 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); 4307 struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
@@ -4605,7 +4612,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
4605 hw->wm_lp[2] = I915_READ(WM3_LP_ILK); 4612 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
4606 4613
4607 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK); 4614 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
4608 if (INTEL_INFO(dev)->gen >= 7) { 4615 if (INTEL_GEN(dev_priv) >= 7) {
4609 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB); 4616 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
4610 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB); 4617 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
4611 } 4618 }
@@ -7690,7 +7697,8 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
7690 /* For FIFO watermark updates */ 7697 /* For FIFO watermark updates */
7691 if (INTEL_GEN(dev_priv) >= 9) { 7698 if (INTEL_GEN(dev_priv) >= 9) {
7692 skl_setup_wm_latency(dev_priv); 7699 skl_setup_wm_latency(dev_priv);
7693 dev_priv->display.update_wm = skl_update_wm; 7700 dev_priv->display.initial_watermarks = skl_initial_wm;
7701 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
7694 dev_priv->display.compute_global_watermarks = skl_compute_wm; 7702 dev_priv->display.compute_global_watermarks = skl_compute_wm;
7695 } else if (HAS_PCH_SPLIT(dev_priv)) { 7703 } else if (HAS_PCH_SPLIT(dev_priv)) {
7696 ilk_setup_wm_latency(dev_priv); 7704 ilk_setup_wm_latency(dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 271a3e29ff23..7b488e2793d9 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -427,7 +427,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
427 struct drm_i915_private *dev_priv = to_i915(dev); 427 struct drm_i915_private *dev_priv = to_i915(dev);
428 struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); 428 struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
429 429
430 if (!HAS_PSR(dev)) { 430 if (!HAS_PSR(dev_priv)) {
431 DRM_DEBUG_KMS("PSR not supported on this platform\n"); 431 DRM_DEBUG_KMS("PSR not supported on this platform\n");
432 return; 432 return;
433 } 433 }
@@ -472,7 +472,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
472 /* Enable PSR on the panel */ 472 /* Enable PSR on the panel */
473 hsw_psr_enable_sink(intel_dp); 473 hsw_psr_enable_sink(intel_dp);
474 474
475 if (INTEL_INFO(dev)->gen >= 9) 475 if (INTEL_GEN(dev_priv) >= 9)
476 intel_psr_activate(intel_dp); 476 intel_psr_activate(intel_dp);
477 } else { 477 } else {
478 vlv_psr_setup_vsc(intel_dp); 478 vlv_psr_setup_vsc(intel_dp);
@@ -498,7 +498,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
498 * - On HSW/BDW we get a recoverable frozen screen until next 498 * - On HSW/BDW we get a recoverable frozen screen until next
499 * exit-activate sequence. 499 * exit-activate sequence.
500 */ 500 */
501 if (INTEL_INFO(dev)->gen < 9) 501 if (INTEL_GEN(dev_priv) < 9)
502 schedule_delayed_work(&dev_priv->psr.work, 502 schedule_delayed_work(&dev_priv->psr.work,
503 msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5)); 503 msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5));
504 504
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 700e93d80616..aeb637dc1fdf 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1294,6 +1294,8 @@ static void i9xx_submit_request(struct drm_i915_gem_request *request)
1294{ 1294{
1295 struct drm_i915_private *dev_priv = request->i915; 1295 struct drm_i915_private *dev_priv = request->i915;
1296 1296
1297 i915_gem_request_submit(request);
1298
1297 I915_WRITE_TAIL(request->engine, request->tail); 1299 I915_WRITE_TAIL(request->engine, request->tail);
1298} 1300}
1299 1301
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 642b54692d0d..3466b4e77e7c 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -267,6 +267,15 @@ struct intel_engine_cs {
267 */ 267 */
268 void (*submit_request)(struct drm_i915_gem_request *req); 268 void (*submit_request)(struct drm_i915_gem_request *req);
269 269
270 /* Call when the priority on a request has changed and it and its
271 * dependencies may need rescheduling. Note the request itself may
272 * not be ready to run!
273 *
274 * Called under the struct_mutex.
275 */
276 void (*schedule)(struct drm_i915_gem_request *request,
277 int priority);
278
270 /* Some chipsets are not quite as coherent as advertised and need 279 /* Some chipsets are not quite as coherent as advertised and need
271 * an expensive kick to force a true read of the up-to-date seqno. 280 * an expensive kick to force a true read of the up-to-date seqno.
272 * However, the up-to-date seqno is not always required and the last 281 * However, the up-to-date seqno is not always required and the last
@@ -335,12 +344,12 @@ struct intel_engine_cs {
335 344
336 /* Execlists */ 345 /* Execlists */
337 struct tasklet_struct irq_tasklet; 346 struct tasklet_struct irq_tasklet;
338 spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */
339 struct execlist_port { 347 struct execlist_port {
340 struct drm_i915_gem_request *request; 348 struct drm_i915_gem_request *request;
341 unsigned int count; 349 unsigned int count;
342 } execlist_port[2]; 350 } execlist_port[2];
343 struct list_head execlist_queue; 351 struct rb_root execlist_queue;
352 struct rb_node *execlist_first;
344 unsigned int fw_domains; 353 unsigned int fw_domains;
345 bool disable_lite_restore_wa; 354 bool disable_lite_restore_wa;
346 bool preempt_wa; 355 bool preempt_wa;
@@ -578,7 +587,6 @@ static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine)
578 587
579void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); 588void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
580void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); 589void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
581unsigned int intel_kick_waiters(struct drm_i915_private *i915); 590unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915);
582unsigned int intel_kick_signalers(struct drm_i915_private *i915);
583 591
584#endif /* _INTEL_RINGBUFFER_H_ */ 592#endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 05994083e161..356c662ad453 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -1066,7 +1066,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1066 * 1066 *
1067 * CHV DPLL B/C have some issues if VGA mode is enabled. 1067 * CHV DPLL B/C have some issues if VGA mode is enabled.
1068 */ 1068 */
1069 for_each_pipe(&dev_priv->drm, pipe) { 1069 for_each_pipe(dev_priv, pipe) {
1070 u32 val = I915_READ(DPLL(pipe)); 1070 u32 val = I915_READ(DPLL(pipe));
1071 1071
1072 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; 1072 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
@@ -1097,7 +1097,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1097 intel_crt_reset(&encoder->base); 1097 intel_crt_reset(&encoder->base);
1098 } 1098 }
1099 1099
1100 i915_redisable_vga_power_on(&dev_priv->drm); 1100 i915_redisable_vga_power_on(dev_priv);
1101 1101
1102 intel_pps_unlock_regs_wa(dev_priv); 1102 intel_pps_unlock_regs_wa(dev_priv);
1103} 1103}
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 3990c805a5b5..27808e91cb5a 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1195,8 +1195,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
1195 struct intel_crtc_state *crtc_state, 1195 struct intel_crtc_state *crtc_state,
1196 struct drm_connector_state *conn_state) 1196 struct drm_connector_state *conn_state)
1197{ 1197{
1198 struct drm_device *dev = intel_encoder->base.dev; 1198 struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
1199 struct drm_i915_private *dev_priv = to_i915(dev);
1200 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); 1199 struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1201 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode; 1200 const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
1202 struct drm_display_mode *mode = &crtc_state->base.mode; 1201 struct drm_display_mode *mode = &crtc_state->base.mode;
@@ -1269,13 +1268,13 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
1269 return; 1268 return;
1270 1269
1271 /* Set the SDVO control regs. */ 1270 /* Set the SDVO control regs. */
1272 if (INTEL_INFO(dev)->gen >= 4) { 1271 if (INTEL_GEN(dev_priv) >= 4) {
1273 /* The real mode polarity is set by the SDVO commands, using 1272 /* The real mode polarity is set by the SDVO commands, using
1274 * struct intel_sdvo_dtd. */ 1273 * struct intel_sdvo_dtd. */
1275 sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH; 1274 sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH;
1276 if (!HAS_PCH_SPLIT(dev_priv) && crtc_state->limited_color_range) 1275 if (!HAS_PCH_SPLIT(dev_priv) && crtc_state->limited_color_range)
1277 sdvox |= HDMI_COLOR_RANGE_16_235; 1276 sdvox |= HDMI_COLOR_RANGE_16_235;
1278 if (INTEL_INFO(dev)->gen < 5) 1277 if (INTEL_GEN(dev_priv) < 5)
1279 sdvox |= SDVO_BORDER_ENABLE; 1278 sdvox |= SDVO_BORDER_ENABLE;
1280 } else { 1279 } else {
1281 sdvox = I915_READ(intel_sdvo->sdvo_reg); 1280 sdvox = I915_READ(intel_sdvo->sdvo_reg);
@@ -1294,7 +1293,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
1294 if (intel_sdvo->has_hdmi_audio) 1293 if (intel_sdvo->has_hdmi_audio)
1295 sdvox |= SDVO_AUDIO_ENABLE; 1294 sdvox |= SDVO_AUDIO_ENABLE;
1296 1295
1297 if (INTEL_INFO(dev)->gen >= 4) { 1296 if (INTEL_GEN(dev_priv) >= 4) {
1298 /* done in crtc_mode_set as the dpll_md reg must be written early */ 1297 /* done in crtc_mode_set as the dpll_md reg must be written early */
1299 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || 1298 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
1300 IS_G33(dev_priv)) { 1299 IS_G33(dev_priv)) {
@@ -1305,7 +1304,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
1305 } 1304 }
1306 1305
1307 if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL && 1306 if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL &&
1308 INTEL_INFO(dev)->gen < 5) 1307 INTEL_GEN(dev_priv) < 5)
1309 sdvox |= SDVO_STALL_SELECT; 1308 sdvox |= SDVO_STALL_SELECT;
1310 intel_sdvo_write_sdvox(intel_sdvo, sdvox); 1309 intel_sdvo_write_sdvox(intel_sdvo, sdvox);
1311} 1310}
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 0e9aac2e3eae..c2b629c922e7 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -203,13 +203,8 @@ skl_update_plane(struct drm_plane *drm_plane,
203 struct drm_i915_private *dev_priv = to_i915(dev); 203 struct drm_i915_private *dev_priv = to_i915(dev);
204 struct intel_plane *intel_plane = to_intel_plane(drm_plane); 204 struct intel_plane *intel_plane = to_intel_plane(drm_plane);
205 struct drm_framebuffer *fb = plane_state->base.fb; 205 struct drm_framebuffer *fb = plane_state->base.fb;
206 const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
207 struct drm_crtc *crtc = crtc_state->base.crtc;
208 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
209 const int pipe = intel_plane->pipe; 206 const int pipe = intel_plane->pipe;
210 const int plane = intel_plane->plane + 1; 207 const int plane = intel_plane->plane + 1;
211 const struct skl_plane_wm *p_wm =
212 &crtc_state->wm.skl.optimal.planes[plane];
213 u32 plane_ctl; 208 u32 plane_ctl;
214 const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; 209 const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
215 u32 surf_addr = plane_state->main.offset; 210 u32 surf_addr = plane_state->main.offset;
@@ -233,9 +228,6 @@ skl_update_plane(struct drm_plane *drm_plane,
233 228
234 plane_ctl |= skl_plane_ctl_rotation(rotation); 229 plane_ctl |= skl_plane_ctl_rotation(rotation);
235 230
236 if (wm->dirty_pipes & drm_crtc_mask(crtc))
237 skl_write_plane_wm(intel_crtc, p_wm, &wm->ddb, plane);
238
239 if (key->flags) { 231 if (key->flags) {
240 I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value); 232 I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value);
241 I915_WRITE(PLANE_KEYMAX(pipe, plane), key->max_value); 233 I915_WRITE(PLANE_KEYMAX(pipe, plane), key->max_value);
@@ -291,19 +283,9 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
291 struct drm_device *dev = dplane->dev; 283 struct drm_device *dev = dplane->dev;
292 struct drm_i915_private *dev_priv = to_i915(dev); 284 struct drm_i915_private *dev_priv = to_i915(dev);
293 struct intel_plane *intel_plane = to_intel_plane(dplane); 285 struct intel_plane *intel_plane = to_intel_plane(dplane);
294 struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
295 const int pipe = intel_plane->pipe; 286 const int pipe = intel_plane->pipe;
296 const int plane = intel_plane->plane + 1; 287 const int plane = intel_plane->plane + 1;
297 288
298 /*
299 * We only populate skl_results on watermark updates, and if the
300 * plane's visiblity isn't actually changing neither is its watermarks.
301 */
302 if (!dplane->state->visible)
303 skl_write_plane_wm(to_intel_crtc(crtc),
304 &cstate->wm.skl.optimal.planes[plane],
305 &dev_priv->wm.skl_results.ddb, plane);
306
307 I915_WRITE(PLANE_CTL(pipe, plane), 0); 289 I915_WRITE(PLANE_CTL(pipe, plane), 0);
308 290
309 I915_WRITE(PLANE_SURF(pipe, plane), 0); 291 I915_WRITE(PLANE_SURF(pipe, plane), 0);
@@ -362,7 +344,7 @@ vlv_update_plane(struct drm_plane *dplane,
362 int plane = intel_plane->plane; 344 int plane = intel_plane->plane;
363 u32 sprctl; 345 u32 sprctl;
364 u32 sprsurf_offset, linear_offset; 346 u32 sprsurf_offset, linear_offset;
365 unsigned int rotation = dplane->state->rotation; 347 unsigned int rotation = plane_state->base.rotation;
366 const struct drm_intel_sprite_colorkey *key = &plane_state->ckey; 348 const struct drm_intel_sprite_colorkey *key = &plane_state->ckey;
367 int crtc_x = plane_state->base.dst.x1; 349 int crtc_x = plane_state->base.dst.x1;
368 int crtc_y = plane_state->base.dst.y1; 350 int crtc_y = plane_state->base.dst.y1;
@@ -427,6 +409,12 @@ vlv_update_plane(struct drm_plane *dplane,
427 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) 409 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
428 sprctl |= SP_TILED; 410 sprctl |= SP_TILED;
429 411
412 if (rotation & DRM_ROTATE_180)
413 sprctl |= SP_ROTATE_180;
414
415 if (rotation & DRM_REFLECT_X)
416 sprctl |= SP_MIRROR;
417
430 /* Sizes are 0 based */ 418 /* Sizes are 0 based */
431 src_w--; 419 src_w--;
432 src_h--; 420 src_h--;
@@ -436,11 +424,11 @@ vlv_update_plane(struct drm_plane *dplane,
436 intel_add_fb_offsets(&x, &y, plane_state, 0); 424 intel_add_fb_offsets(&x, &y, plane_state, 0);
437 sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); 425 sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
438 426
439 if (rotation == DRM_ROTATE_180) { 427 if (rotation & DRM_ROTATE_180) {
440 sprctl |= SP_ROTATE_180;
441
442 x += src_w; 428 x += src_w;
443 y += src_h; 429 y += src_h;
430 } else if (rotation & DRM_REFLECT_X) {
431 x += src_w;
444 } 432 }
445 433
446 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); 434 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -546,6 +534,9 @@ ivb_update_plane(struct drm_plane *plane,
546 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) 534 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
547 sprctl |= SPRITE_TILED; 535 sprctl |= SPRITE_TILED;
548 536
537 if (rotation & DRM_ROTATE_180)
538 sprctl |= SPRITE_ROTATE_180;
539
549 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) 540 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
550 sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE; 541 sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE;
551 else 542 else
@@ -566,14 +557,11 @@ ivb_update_plane(struct drm_plane *plane,
566 intel_add_fb_offsets(&x, &y, plane_state, 0); 557 intel_add_fb_offsets(&x, &y, plane_state, 0);
567 sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); 558 sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
568 559
569 if (rotation == DRM_ROTATE_180) { 560 /* HSW+ does this automagically in hardware */
570 sprctl |= SPRITE_ROTATE_180; 561 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) &&
571 562 rotation & DRM_ROTATE_180) {
572 /* HSW and BDW does this automagically in hardware */ 563 x += src_w;
573 if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv)) { 564 y += src_h;
574 x += src_w;
575 y += src_h;
576 }
577 } 565 }
578 566
579 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0); 567 linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -684,6 +672,9 @@ ilk_update_plane(struct drm_plane *plane,
684 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) 672 if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
685 dvscntr |= DVS_TILED; 673 dvscntr |= DVS_TILED;
686 674
675 if (rotation & DRM_ROTATE_180)
676 dvscntr |= DVS_ROTATE_180;
677
687 if (IS_GEN6(dev_priv)) 678 if (IS_GEN6(dev_priv))
688 dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */ 679 dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */
689 680
@@ -700,9 +691,7 @@ ilk_update_plane(struct drm_plane *plane,
700 intel_add_fb_offsets(&x, &y, plane_state, 0); 691 intel_add_fb_offsets(&x, &y, plane_state, 0);
701 dvssurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0); 692 dvssurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
702 693
703 if (rotation == DRM_ROTATE_180) { 694 if (rotation & DRM_ROTATE_180) {
704 dvscntr |= DVS_ROTATE_180;
705
706 x += src_w; 695 x += src_w;
707 y += src_h; 696 y += src_h;
708 } 697 }
@@ -1112,6 +1101,10 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
1112 supported_rotations = 1101 supported_rotations =
1113 DRM_ROTATE_0 | DRM_ROTATE_90 | 1102 DRM_ROTATE_0 | DRM_ROTATE_90 |
1114 DRM_ROTATE_180 | DRM_ROTATE_270; 1103 DRM_ROTATE_180 | DRM_ROTATE_270;
1104 } else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
1105 supported_rotations =
1106 DRM_ROTATE_0 | DRM_ROTATE_180 |
1107 DRM_REFLECT_X;
1115 } else { 1108 } else {
1116 supported_rotations = 1109 supported_rotations =
1117 DRM_ROTATE_0 | DRM_ROTATE_180; 1110 DRM_ROTATE_0 | DRM_ROTATE_180;
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 9212f00d5752..78cdfc6833d6 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1029,8 +1029,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
1029 struct intel_crtc_state *pipe_config, 1029 struct intel_crtc_state *pipe_config,
1030 struct drm_connector_state *conn_state) 1030 struct drm_connector_state *conn_state)
1031{ 1031{
1032 struct drm_device *dev = encoder->base.dev; 1032 struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1033 struct drm_i915_private *dev_priv = to_i915(dev);
1034 struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); 1033 struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
1035 struct intel_tv *intel_tv = enc_to_tv(encoder); 1034 struct intel_tv *intel_tv = enc_to_tv(encoder);
1036 const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv); 1035 const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
@@ -1116,7 +1115,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
1116 1115
1117 set_color_conversion(dev_priv, color_conversion); 1116 set_color_conversion(dev_priv, color_conversion);
1118 1117
1119 if (INTEL_INFO(dev)->gen >= 4) 1118 if (INTEL_GEN(dev_priv) >= 4)
1120 I915_WRITE(TV_CLR_KNOBS, 0x00404000); 1119 I915_WRITE(TV_CLR_KNOBS, 0x00404000);
1121 else 1120 else
1122 I915_WRITE(TV_CLR_KNOBS, 0x00606000); 1121 I915_WRITE(TV_CLR_KNOBS, 0x00606000);
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index e2b188dcf908..d7be0d94ba4d 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -402,6 +402,8 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
402static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, 402static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
403 bool restore_forcewake) 403 bool restore_forcewake)
404{ 404{
405 struct intel_device_info *info = mkwrite_device_info(dev_priv);
406
405 /* clear out unclaimed reg detection bit */ 407 /* clear out unclaimed reg detection bit */
406 if (check_for_unclaimed_mmio(dev_priv)) 408 if (check_for_unclaimed_mmio(dev_priv))
407 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); 409 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
@@ -419,6 +421,10 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
419 GT_FIFO_CTL_RC6_POLICY_STALL); 421 GT_FIFO_CTL_RC6_POLICY_STALL);
420 } 422 }
421 423
424 /* Enable Decoupled MMIO only on BXT C stepping onwards */
425 if (!IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
426 info->has_decoupled_mmio = false;
427
422 intel_uncore_forcewake_reset(dev_priv, restore_forcewake); 428 intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
423} 429}
424 430
@@ -641,6 +647,8 @@ intel_fw_table_check(struct drm_i915_private *dev_priv)
641 num_ranges = dev_priv->uncore.fw_domains_table_entries; 647 num_ranges = dev_priv->uncore.fw_domains_table_entries;
642 648
643 for (i = 0, prev = -1; i < num_ranges; i++, ranges++) { 649 for (i = 0, prev = -1; i < num_ranges; i++, ranges++) {
650 WARN_ON_ONCE(IS_GEN9(dev_priv) &&
651 (prev + 1) != (s32)ranges->start);
644 WARN_ON_ONCE(prev >= (s32)ranges->start); 652 WARN_ON_ONCE(prev >= (s32)ranges->start);
645 prev = ranges->start; 653 prev = ranges->start;
646 WARN_ON_ONCE(prev >= (s32)ranges->end); 654 WARN_ON_ONCE(prev >= (s32)ranges->end);
@@ -783,7 +791,7 @@ static const struct intel_forcewake_range __gen9_fw_ranges[] = {
783 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA), 791 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
784 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), 792 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
785 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), 793 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
786 GEN_FW_RANGE(0xb480, 0xbfff, FORCEWAKE_BLITTER), 794 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
787 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA), 795 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
788 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER), 796 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
789 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER), 797 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
@@ -831,6 +839,66 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv,
831 __unclaimed_reg_debug(dev_priv, reg, read, before); 839 __unclaimed_reg_debug(dev_priv, reg, read, before);
832} 840}
833 841
842static const enum decoupled_power_domain fw2dpd_domain[] = {
843 GEN9_DECOUPLED_PD_RENDER,
844 GEN9_DECOUPLED_PD_BLITTER,
845 GEN9_DECOUPLED_PD_ALL,
846 GEN9_DECOUPLED_PD_MEDIA,
847 GEN9_DECOUPLED_PD_ALL,
848 GEN9_DECOUPLED_PD_ALL,
849 GEN9_DECOUPLED_PD_ALL
850};
851
852/*
853 * Decoupled MMIO access for only 1 DWORD
854 */
855static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
856 u32 reg,
857 enum forcewake_domains fw_domain,
858 enum decoupled_ops operation)
859{
860 enum decoupled_power_domain dp_domain;
861 u32 ctrl_reg_data = 0;
862
863 dp_domain = fw2dpd_domain[fw_domain - 1];
864
865 ctrl_reg_data |= reg;
866 ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
867 ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
868 ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
869 __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);
870
871 if (wait_for_atomic((__raw_i915_read32(dev_priv,
872 GEN9_DECOUPLED_REG0_DW1) &
873 GEN9_DECOUPLED_DW1_GO) == 0,
874 FORCEWAKE_ACK_TIMEOUT_MS))
875 DRM_ERROR("Decoupled MMIO wait timed out\n");
876}
877
878static inline u32
879__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
880 u32 reg,
881 enum forcewake_domains fw_domain)
882{
883 __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
884 GEN9_DECOUPLED_OP_READ);
885
886 return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
887}
888
889static inline void
890__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
891 u32 reg, u32 data,
892 enum forcewake_domains fw_domain)
893{
894
895 __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);
896
897 __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
898 GEN9_DECOUPLED_OP_WRITE);
899}
900
901
834#define GEN2_READ_HEADER(x) \ 902#define GEN2_READ_HEADER(x) \
835 u##x val = 0; \ 903 u##x val = 0; \
836 assert_rpm_wakelock_held(dev_priv); 904 assert_rpm_wakelock_held(dev_priv);
@@ -935,6 +1003,28 @@ fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) {
935 GEN6_READ_FOOTER; \ 1003 GEN6_READ_FOOTER; \
936} 1004}
937 1005
1006#define __gen9_decoupled_read(x) \
1007static u##x \
1008gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
1009 i915_reg_t reg, bool trace) { \
1010 enum forcewake_domains fw_engine; \
1011 GEN6_READ_HEADER(x); \
1012 fw_engine = __fwtable_reg_read_fw_domains(offset); \
1013 if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
1014 unsigned i; \
1015 u32 *ptr_data = (u32 *) &val; \
1016 for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
1017 *ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
1018 offset, \
1019 fw_engine); \
1020 } else { \
1021 val = __raw_i915_read##x(dev_priv, reg); \
1022 } \
1023 GEN6_READ_FOOTER; \
1024}
1025
1026__gen9_decoupled_read(32)
1027__gen9_decoupled_read(64)
938__fwtable_read(8) 1028__fwtable_read(8)
939__fwtable_read(16) 1029__fwtable_read(16)
940__fwtable_read(32) 1030__fwtable_read(32)
@@ -1064,6 +1154,25 @@ fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bo
1064 GEN6_WRITE_FOOTER; \ 1154 GEN6_WRITE_FOOTER; \
1065} 1155}
1066 1156
1157#define __gen9_decoupled_write(x) \
1158static void \
1159gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
1160 i915_reg_t reg, u##x val, \
1161 bool trace) { \
1162 enum forcewake_domains fw_engine; \
1163 GEN6_WRITE_HEADER; \
1164 fw_engine = __fwtable_reg_write_fw_domains(offset); \
1165 if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
1166 __gen9_decoupled_mmio_write(dev_priv, \
1167 offset, \
1168 val, \
1169 fw_engine); \
1170 else \
1171 __raw_i915_write##x(dev_priv, reg, val); \
1172 GEN6_WRITE_FOOTER; \
1173}
1174
1175__gen9_decoupled_write(32)
1067__fwtable_write(8) 1176__fwtable_write(8)
1068__fwtable_write(16) 1177__fwtable_write(16)
1069__fwtable_write(32) 1178__fwtable_write(32)
@@ -1287,6 +1396,14 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
1287 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges); 1396 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1288 ASSIGN_WRITE_MMIO_VFUNCS(fwtable); 1397 ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
1289 ASSIGN_READ_MMIO_VFUNCS(fwtable); 1398 ASSIGN_READ_MMIO_VFUNCS(fwtable);
1399 if (HAS_DECOUPLED_MMIO(dev_priv)) {
1400 dev_priv->uncore.funcs.mmio_readl =
1401 gen9_decoupled_read32;
1402 dev_priv->uncore.funcs.mmio_readq =
1403 gen9_decoupled_read64;
1404 dev_priv->uncore.funcs.mmio_writel =
1405 gen9_decoupled_write32;
1406 }
1290 break; 1407 break;
1291 case 8: 1408 case 8:
1292 if (IS_CHERRYVIEW(dev_priv)) { 1409 if (IS_CHERRYVIEW(dev_priv)) {
@@ -1368,7 +1485,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
1368 1485
1369 for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { 1486 for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
1370 if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) && 1487 if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
1371 (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask)) 1488 (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask))
1372 break; 1489 break;
1373 } 1490 }
1374 1491
diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h
index 68db9621f1f0..8886cab19f98 100644
--- a/drivers/gpu/drm/i915/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/intel_vbt_defs.h
@@ -280,7 +280,8 @@ struct common_child_dev_config {
280 u8 dp_support:1; 280 u8 dp_support:1;
281 u8 tmds_support:1; 281 u8 tmds_support:1;
282 u8 support_reserved:5; 282 u8 support_reserved:5;
283 u8 not_common3[12]; 283 u8 aux_channel;
284 u8 not_common3[11];
284 u8 iboost_level; 285 u8 iboost_level;
285} __packed; 286} __packed;
286 287
diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h
index c3a7d440bc11..38eabf65f19d 100644
--- a/include/drm/drm_edid.h
+++ b/include/drm/drm_edid.h
@@ -330,7 +330,6 @@ int drm_edid_to_sad(struct edid *edid, struct cea_sad **sads);
330int drm_edid_to_speaker_allocation(struct edid *edid, u8 **sadb); 330int drm_edid_to_speaker_allocation(struct edid *edid, u8 **sadb);
331int drm_av_sync_delay(struct drm_connector *connector, 331int drm_av_sync_delay(struct drm_connector *connector,
332 const struct drm_display_mode *mode); 332 const struct drm_display_mode *mode);
333struct drm_connector *drm_select_eld(struct drm_encoder *encoder);
334 333
335#ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE 334#ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE
336int drm_load_edid_firmware(struct drm_connector *connector); 335int drm_load_edid_firmware(struct drm_connector *connector);
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 03725fe89859..1c12a350eca3 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -389,6 +389,11 @@ typedef struct drm_i915_irq_wait {
389#define I915_PARAM_MIN_EU_IN_POOL 39 389#define I915_PARAM_MIN_EU_IN_POOL 39
390#define I915_PARAM_MMAP_GTT_VERSION 40 390#define I915_PARAM_MMAP_GTT_VERSION 40
391 391
392/* Query whether DRM_I915_GEM_EXECBUFFER2 supports user defined execution
393 * priorities and the driver will attempt to execute batches in priority order.
394 */
395#define I915_PARAM_HAS_SCHEDULER 41
396
392typedef struct drm_i915_getparam { 397typedef struct drm_i915_getparam {
393 __s32 param; 398 __s32 param;
394 /* 399 /*
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 12:43:45 -0400