aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-01-10 20:11:39 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2011-01-10 20:11:39 -0500
commit5b2eef966cb2ae307aa4ef1767f7307774bc96ca (patch)
tree095a251e145903598dd8d90d5b2eb880f0d6ff93
parent8adbf8d46718a8f110de55ec82c40d04d0c362cc (diff)
parent56bec7c009872ef33fe452ea75fecba481351b44 (diff)
Merge branch 'drm-core-next' of git://git.kernel.org/pub/scm/linux/kernel/git/airlied/drm-2.6
* 'drm-core-next' of git://git.kernel.org/pub/scm/linux/kernel/git/airlied/drm-2.6: (390 commits) drm/radeon/kms: disable underscan by default drm/radeon/kms: only enable hdmi features if the monitor supports audio drm: Restore the old_fb upon modeset failure drm/nouveau: fix hwmon device binding radeon: consolidate asic-specific function decls for pre-r600 vga_switcheroo: comparing too few characters in strncmp() drm/radeon/kms: add NI pci ids drm/radeon/kms: don't enable pcie gen2 on NI yet drm/radeon/kms: add radeon_asic struct for NI asics drm/radeon/kms/ni: load default sclk/mclk/vddc at pm init drm/radeon/kms: add ucode loader for NI drm/radeon/kms: add support for DCE5 display LUTs drm/radeon/kms: add ni_reg.h drm/radeon/kms: add bo blit support for NI drm/radeon/kms: always use writeback/events for fences on NI drm/radeon/kms: adjust default clock/vddc tracking for pm on DCE5 drm/radeon/kms: add backend map workaround for barts drm/radeon/kms: fill gpu init for NI asics drm/radeon/kms: add disabled vbios accessor for NI asics drm/radeon/kms: handle NI thermal controller ...
-rw-r--r--drivers/char/agp/agp.h1
-rw-r--r--drivers/char/agp/compat_ioctl.c1
-rw-r--r--drivers/char/agp/compat_ioctl.h1
-rw-r--r--drivers/char/agp/frontend.c8
-rw-r--r--drivers/char/agp/generic.c27
-rw-r--r--drivers/char/agp/intel-agp.c5
-rw-r--r--drivers/char/agp/intel-agp.h14
-rw-r--r--drivers/char/agp/intel-gtt.c778
-rw-r--r--drivers/gpu/drm/drm_agpsupport.c6
-rw-r--r--drivers/gpu/drm/drm_crtc_helper.c18
-rw-r--r--drivers/gpu/drm/drm_fb_helper.c61
-rw-r--r--drivers/gpu/drm/drm_fops.c2
-rw-r--r--drivers/gpu/drm/drm_irq.c566
-rw-r--r--drivers/gpu/drm/drm_mm.c40
-rw-r--r--drivers/gpu/drm/drm_stub.c10
-rw-r--r--drivers/gpu/drm/i915/Makefile2
-rw-r--r--drivers/gpu/drm/i915/i915_debugfs.c471
-rw-r--r--drivers/gpu/drm/i915/i915_dma.c794
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c80
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h605
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c3764
-rw-r--r--drivers/gpu/drm/i915/i915_gem_debug.c23
-rw-r--r--drivers/gpu/drm/i915/i915_gem_evict.c125
-rw-r--r--drivers/gpu/drm/i915/i915_gem_execbuffer.c1343
-rw-r--r--drivers/gpu/drm/i915/i915_gem_gtt.c99
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c139
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c724
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h193
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c104
-rw-r--r--drivers/gpu/drm/i915/i915_trace.h91
-rw-r--r--drivers/gpu/drm/i915/intel_display.c999
-rw-r--r--drivers/gpu/drm/i915/intel_dp.c3
-rw-r--r--drivers/gpu/drm/i915/intel_drv.h22
-rw-r--r--drivers/gpu/drm/i915/intel_fb.c32
-rw-r--r--drivers/gpu/drm/i915/intel_i2c.c21
-rw-r--r--drivers/gpu/drm/i915/intel_lvds.c43
-rw-r--r--drivers/gpu/drm/i915/intel_opregion.c8
-rw-r--r--drivers/gpu/drm/i915/intel_overlay.c116
-rw-r--r--drivers/gpu/drm/i915/intel_panel.c52
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c1007
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.h136
-rw-r--r--drivers/gpu/drm/i915/intel_sdvo.c7
-rw-r--r--drivers/gpu/drm/i915/intel_tv.c14
-rw-r--r--drivers/gpu/drm/nouveau/Kconfig2
-rw-r--r--drivers/gpu/drm/nouveau/Makefile17
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_acpi.c11
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bios.c102
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_bo.c319
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_channel.c383
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_connector.c54
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_display.c207
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dma.c32
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dma.h9
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_dp.c6
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_drv.c58
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_drv.h425
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fbcon.c189
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fbcon.h18
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_fence.c117
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_gem.c171
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_hw.c11
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_irq.c1210
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_mem.c426
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_mm.c271
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_mm.h67
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_notifier.c44
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_object.c754
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_pm.c33
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_ramht.c11
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_ramht.h2
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_reg.h75
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_sgdma.c212
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_state.c300
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_util.c69
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_util.h45
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vm.c439
-rw-r--r--drivers/gpu/drm/nouveau/nouveau_vm.h113
-rw-r--r--drivers/gpu/drm/nouveau/nv04_crtc.c8
-rw-r--r--drivers/gpu/drm/nouveau/nv04_dac.c12
-rw-r--r--drivers/gpu/drm/nouveau/nv04_display.c21
-rw-r--r--drivers/gpu/drm/nouveau/nv04_fbcon.c102
-rw-r--r--drivers/gpu/drm/nouveau/nv04_fifo.c240
-rw-r--r--drivers/gpu/drm/nouveau/nv04_graph.c645
-rw-r--r--drivers/gpu/drm/nouveau/nv04_instmem.c50
-rw-r--r--drivers/gpu/drm/nouveau/nv10_fb.c124
-rw-r--r--drivers/gpu/drm/nouveau/nv10_fifo.c19
-rw-r--r--drivers/gpu/drm/nouveau/nv10_graph.c203
-rw-r--r--drivers/gpu/drm/nouveau/nv20_graph.c244
-rw-r--r--drivers/gpu/drm/nouveau/nv30_fb.c23
-rw-r--r--drivers/gpu/drm/nouveau/nv40_fb.c22
-rw-r--r--drivers/gpu/drm/nouveau/nv40_fifo.c20
-rw-r--r--drivers/gpu/drm/nouveau/nv40_graph.c205
-rw-r--r--drivers/gpu/drm/nouveau/nv50_crtc.c27
-rw-r--r--drivers/gpu/drm/nouveau/nv50_display.c422
-rw-r--r--drivers/gpu/drm/nouveau/nv50_display.h2
-rw-r--r--drivers/gpu/drm/nouveau/nv50_evo.c345
-rw-r--r--drivers/gpu/drm/nouveau/nv50_evo.h10
-rw-r--r--drivers/gpu/drm/nouveau/nv50_fb.c71
-rw-r--r--drivers/gpu/drm/nouveau/nv50_fbcon.c114
-rw-r--r--drivers/gpu/drm/nouveau/nv50_fifo.c42
-rw-r--r--drivers/gpu/drm/nouveau/nv50_gpio.c198
-rw-r--r--drivers/gpu/drm/nouveau/nv50_graph.c677
-rw-r--r--drivers/gpu/drm/nouveau/nv50_instmem.c375
-rw-r--r--drivers/gpu/drm/nouveau/nv50_vm.c180
-rw-r--r--drivers/gpu/drm/nouveau/nv50_vram.c190
-rw-r--r--drivers/gpu/drm/nouveau/nv84_crypt.c140
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_fbcon.c269
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_fifo.c365
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_graph.c705
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_graph.h64
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_grctx.c2874
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_instmem.c317
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_vm.c123
-rw-r--r--drivers/gpu/drm/nouveau/nvc0_vram.c99
-rw-r--r--drivers/gpu/drm/nouveau/nvreg.h3
-rw-r--r--drivers/gpu/drm/radeon/Makefile5
-rw-r--r--drivers/gpu/drm/radeon/ObjectID.h48
-rw-r--r--drivers/gpu/drm/radeon/atom.c14
-rw-r--r--drivers/gpu/drm/radeon/atombios.h997
-rw-r--r--drivers/gpu/drm/radeon/atombios_crtc.c57
-rw-r--r--drivers/gpu/drm/radeon/evergreen.c806
-rw-r--r--drivers/gpu/drm/radeon/evergreen_blit_kms.c92
-rw-r--r--drivers/gpu/drm/radeon/evergreen_reg.h6
-rw-r--r--drivers/gpu/drm/radeon/evergreend.h53
-rw-r--r--drivers/gpu/drm/radeon/ni.c316
-rw-r--r--drivers/gpu/drm/radeon/ni_reg.h86
-rw-r--r--drivers/gpu/drm/radeon/nid.h41
-rw-r--r--drivers/gpu/drm/radeon/r100.c78
-rw-r--r--drivers/gpu/drm/radeon/r100d.h2
-rw-r--r--drivers/gpu/drm/radeon/r300.c21
-rw-r--r--drivers/gpu/drm/radeon/r300d.h1
-rw-r--r--drivers/gpu/drm/radeon/r500_reg.h4
-rw-r--r--drivers/gpu/drm/radeon/r600.c357
-rw-r--r--drivers/gpu/drm/radeon/r600d.h48
-rw-r--r--drivers/gpu/drm/radeon/radeon.h151
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.c153
-rw-r--r--drivers/gpu/drm/radeon/radeon_asic.h65
-rw-r--r--drivers/gpu/drm/radeon/radeon_atombios.c1246
-rw-r--r--drivers/gpu/drm/radeon/radeon_bios.c41
-rw-r--r--drivers/gpu/drm/radeon/radeon_combios.c3
-rw-r--r--drivers/gpu/drm/radeon/radeon_connectors.c9
-rw-r--r--drivers/gpu/drm/radeon/radeon_cs.c17
-rw-r--r--drivers/gpu/drm/radeon/radeon_device.c35
-rw-r--r--drivers/gpu/drm/radeon/radeon_display.c391
-rw-r--r--drivers/gpu/drm/radeon/radeon_drv.c11
-rw-r--r--drivers/gpu/drm/radeon/radeon_encoders.c205
-rw-r--r--drivers/gpu/drm/radeon/radeon_family.h4
-rw-r--r--drivers/gpu/drm/radeon/radeon_fb.c4
-rw-r--r--drivers/gpu/drm/radeon/radeon_fence.c4
-rw-r--r--drivers/gpu/drm/radeon/radeon_irq_kms.c46
-rw-r--r--drivers/gpu/drm/radeon/radeon_kms.c64
-rw-r--r--drivers/gpu/drm/radeon/radeon_mode.h16
-rw-r--r--drivers/gpu/drm/radeon/radeon_object.c57
-rw-r--r--drivers/gpu/drm/radeon/radeon_object.h7
-rw-r--r--drivers/gpu/drm/radeon/radeon_pm.c94
-rw-r--r--drivers/gpu/drm/radeon/radeon_reg.h13
-rw-r--r--drivers/gpu/drm/radeon/radeon_trace.h82
-rw-r--r--drivers/gpu/drm/radeon/radeon_trace_points.c9
-rw-r--r--drivers/gpu/drm/radeon/reg_srcs/rv51516
-rw-r--r--drivers/gpu/drm/radeon/rs600.c118
-rw-r--r--drivers/gpu/drm/radeon/rv770.c196
-rw-r--r--drivers/gpu/drm/radeon/rv770d.h47
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c156
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_util.c138
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo_vm.c29
-rw-r--r--drivers/gpu/drm/ttm/ttm_execbuf_util.c169
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_drv.h1
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c3
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_fb.c3
-rw-r--r--drivers/gpu/vga/vga_switcheroo.c78
-rw-r--r--include/drm/drmP.h102
-rw-r--r--include/drm/drm_crtc.h9
-rw-r--r--include/drm/drm_fb_helper.h3
-rw-r--r--include/drm/drm_mm.h7
-rw-r--r--include/drm/drm_pciids.h40
-rw-r--r--include/drm/i915_drm.h12
-rw-r--r--include/drm/intel-gtt.h35
-rw-r--r--include/drm/nouveau_drm.h5
-rw-r--r--include/drm/radeon_drm.h1
-rw-r--r--include/drm/ttm/ttm_bo_api.h50
-rw-r--r--include/drm/ttm/ttm_bo_driver.h152
-rw-r--r--include/drm/ttm/ttm_execbuf_util.h11
-rw-r--r--include/linux/agp_backend.h2
-rw-r--r--include/linux/intel-gtt.h20
-rw-r--r--include/linux/kref.h2
-rw-r--r--include/linux/vga_switcheroo.h2
-rw-r--r--lib/kref.c30
187 files changed, 24781 insertions, 10722 deletions
diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
index 5259065f3c79..3e67ddde9e16 100644
--- a/drivers/char/agp/agp.h
+++ b/drivers/char/agp/agp.h
@@ -120,7 +120,6 @@ struct agp_bridge_driver {
120 void (*agp_destroy_page)(struct page *, int flags); 120 void (*agp_destroy_page)(struct page *, int flags);
121 void (*agp_destroy_pages)(struct agp_memory *); 121 void (*agp_destroy_pages)(struct agp_memory *);
122 int (*agp_type_to_mask_type) (struct agp_bridge_data *, int); 122 int (*agp_type_to_mask_type) (struct agp_bridge_data *, int);
123 void (*chipset_flush)(struct agp_bridge_data *);
124}; 123};
125 124
126struct agp_bridge_data { 125struct agp_bridge_data {
diff --git a/drivers/char/agp/compat_ioctl.c b/drivers/char/agp/compat_ioctl.c
index 9d2c97a69cdd..a48e05b31593 100644
--- a/drivers/char/agp/compat_ioctl.c
+++ b/drivers/char/agp/compat_ioctl.c
@@ -276,7 +276,6 @@ long compat_agp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
276 break; 276 break;
277 277
278 case AGPIOC_CHIPSET_FLUSH32: 278 case AGPIOC_CHIPSET_FLUSH32:
279 ret_val = agpioc_chipset_flush_wrap(curr_priv);
280 break; 279 break;
281 } 280 }
282 281
diff --git a/drivers/char/agp/compat_ioctl.h b/drivers/char/agp/compat_ioctl.h
index 0c9678ac0371..f30e0fd97963 100644
--- a/drivers/char/agp/compat_ioctl.h
+++ b/drivers/char/agp/compat_ioctl.h
@@ -102,6 +102,5 @@ void agp_free_memory_wrap(struct agp_memory *memory);
102struct agp_memory *agp_allocate_memory_wrap(size_t pg_count, u32 type); 102struct agp_memory *agp_allocate_memory_wrap(size_t pg_count, u32 type);
103struct agp_memory *agp_find_mem_by_key(int key); 103struct agp_memory *agp_find_mem_by_key(int key);
104struct agp_client *agp_find_client_by_pid(pid_t id); 104struct agp_client *agp_find_client_by_pid(pid_t id);
105int agpioc_chipset_flush_wrap(struct agp_file_private *priv);
106 105
107#endif /* _AGP_COMPAT_H */ 106#endif /* _AGP_COMPAT_H */
diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c
index 3cb4539a98b2..2e044338753c 100644
--- a/drivers/char/agp/frontend.c
+++ b/drivers/char/agp/frontend.c
@@ -957,13 +957,6 @@ static int agpioc_unbind_wrap(struct agp_file_private *priv, void __user *arg)
957 return agp_unbind_memory(memory); 957 return agp_unbind_memory(memory);
958} 958}
959 959
960int agpioc_chipset_flush_wrap(struct agp_file_private *priv)
961{
962 DBG("");
963 agp_flush_chipset(agp_bridge);
964 return 0;
965}
966
967static long agp_ioctl(struct file *file, 960static long agp_ioctl(struct file *file,
968 unsigned int cmd, unsigned long arg) 961 unsigned int cmd, unsigned long arg)
969{ 962{
@@ -1039,7 +1032,6 @@ static long agp_ioctl(struct file *file,
1039 break; 1032 break;
1040 1033
1041 case AGPIOC_CHIPSET_FLUSH: 1034 case AGPIOC_CHIPSET_FLUSH:
1042 ret_val = agpioc_chipset_flush_wrap(curr_priv);
1043 break; 1035 break;
1044 } 1036 }
1045 1037
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 4956f1c8f9d5..012cba0d6d96 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -81,13 +81,6 @@ static int agp_get_key(void)
81 return -1; 81 return -1;
82} 82}
83 83
84void agp_flush_chipset(struct agp_bridge_data *bridge)
85{
86 if (bridge->driver->chipset_flush)
87 bridge->driver->chipset_flush(bridge);
88}
89EXPORT_SYMBOL(agp_flush_chipset);
90
91/* 84/*
92 * Use kmalloc if possible for the page list. Otherwise fall back to 85 * Use kmalloc if possible for the page list. Otherwise fall back to
93 * vmalloc. This speeds things up and also saves memory for small AGP 86 * vmalloc. This speeds things up and also saves memory for small AGP
@@ -487,26 +480,6 @@ int agp_unbind_memory(struct agp_memory *curr)
487} 480}
488EXPORT_SYMBOL(agp_unbind_memory); 481EXPORT_SYMBOL(agp_unbind_memory);
489 482
490/**
491 * agp_rebind_emmory - Rewrite the entire GATT, useful on resume
492 */
493int agp_rebind_memory(void)
494{
495 struct agp_memory *curr;
496 int ret_val = 0;
497
498 spin_lock(&agp_bridge->mapped_lock);
499 list_for_each_entry(curr, &agp_bridge->mapped_list, mapped_list) {
500 ret_val = curr->bridge->driver->insert_memory(curr,
501 curr->pg_start,
502 curr->type);
503 if (ret_val != 0)
504 break;
505 }
506 spin_unlock(&agp_bridge->mapped_lock);
507 return ret_val;
508}
509EXPORT_SYMBOL(agp_rebind_memory);
510 483
511/* End - Routines for handling swapping of agp_memory into the GATT */ 484/* End - Routines for handling swapping of agp_memory into the GATT */
512 485
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index e72f49d52202..07e9796fead7 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -828,14 +828,9 @@ static void __devexit agp_intel_remove(struct pci_dev *pdev)
828static int agp_intel_resume(struct pci_dev *pdev) 828static int agp_intel_resume(struct pci_dev *pdev)
829{ 829{
830 struct agp_bridge_data *bridge = pci_get_drvdata(pdev); 830 struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
831 int ret_val;
832 831
833 bridge->driver->configure(); 832 bridge->driver->configure();
834 833
835 ret_val = agp_rebind_memory();
836 if (ret_val != 0)
837 return ret_val;
838
839 return 0; 834 return 0;
840} 835}
841#endif 836#endif
diff --git a/drivers/char/agp/intel-agp.h b/drivers/char/agp/intel-agp.h
index 90539df02504..010e3defd6c3 100644
--- a/drivers/char/agp/intel-agp.h
+++ b/drivers/char/agp/intel-agp.h
@@ -75,6 +75,8 @@
75#define I810_GMS_DISABLE 0x00000000 75#define I810_GMS_DISABLE 0x00000000
76#define I810_PGETBL_CTL 0x2020 76#define I810_PGETBL_CTL 0x2020
77#define I810_PGETBL_ENABLED 0x00000001 77#define I810_PGETBL_ENABLED 0x00000001
78/* Note: PGETBL_CTL2 has a different offset on G33. */
79#define I965_PGETBL_CTL2 0x20c4
78#define I965_PGETBL_SIZE_MASK 0x0000000e 80#define I965_PGETBL_SIZE_MASK 0x0000000e
79#define I965_PGETBL_SIZE_512KB (0 << 1) 81#define I965_PGETBL_SIZE_512KB (0 << 1)
80#define I965_PGETBL_SIZE_256KB (1 << 1) 82#define I965_PGETBL_SIZE_256KB (1 << 1)
@@ -82,9 +84,15 @@
82#define I965_PGETBL_SIZE_1MB (3 << 1) 84#define I965_PGETBL_SIZE_1MB (3 << 1)
83#define I965_PGETBL_SIZE_2MB (4 << 1) 85#define I965_PGETBL_SIZE_2MB (4 << 1)
84#define I965_PGETBL_SIZE_1_5MB (5 << 1) 86#define I965_PGETBL_SIZE_1_5MB (5 << 1)
85#define G33_PGETBL_SIZE_MASK (3 << 8) 87#define G33_GMCH_SIZE_MASK (3 << 8)
86#define G33_PGETBL_SIZE_1M (1 << 8) 88#define G33_GMCH_SIZE_1M (1 << 8)
87#define G33_PGETBL_SIZE_2M (2 << 8) 89#define G33_GMCH_SIZE_2M (2 << 8)
90#define G4x_GMCH_SIZE_MASK (0xf << 8)
91#define G4x_GMCH_SIZE_1M (0x1 << 8)
92#define G4x_GMCH_SIZE_2M (0x3 << 8)
93#define G4x_GMCH_SIZE_VT_1M (0x9 << 8)
94#define G4x_GMCH_SIZE_VT_1_5M (0xa << 8)
95#define G4x_GMCH_SIZE_VT_2M (0xc << 8)
88 96
89#define I810_DRAM_CTL 0x3000 97#define I810_DRAM_CTL 0x3000
90#define I810_DRAM_ROW_0 0x00000001 98#define I810_DRAM_ROW_0 0x00000001
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 29ac6d499fa6..356f73e0d17e 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -24,7 +24,6 @@
24#include <asm/smp.h> 24#include <asm/smp.h>
25#include "agp.h" 25#include "agp.h"
26#include "intel-agp.h" 26#include "intel-agp.h"
27#include <linux/intel-gtt.h>
28#include <drm/intel-gtt.h> 27#include <drm/intel-gtt.h>
29 28
30/* 29/*
@@ -39,40 +38,12 @@
39#define USE_PCI_DMA_API 0 38#define USE_PCI_DMA_API 0
40#endif 39#endif
41 40
42/* Max amount of stolen space, anything above will be returned to Linux */
43int intel_max_stolen = 32 * 1024 * 1024;
44
45static const struct aper_size_info_fixed intel_i810_sizes[] =
46{
47 {64, 16384, 4},
48 /* The 32M mode still requires a 64k gatt */
49 {32, 8192, 4}
50};
51
52#define AGP_DCACHE_MEMORY 1
53#define AGP_PHYS_MEMORY 2
54#define INTEL_AGP_CACHED_MEMORY 3
55
56static struct gatt_mask intel_i810_masks[] =
57{
58 {.mask = I810_PTE_VALID, .type = 0},
59 {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
60 {.mask = I810_PTE_VALID, .type = 0},
61 {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
62 .type = INTEL_AGP_CACHED_MEMORY}
63};
64
65#define INTEL_AGP_UNCACHED_MEMORY 0
66#define INTEL_AGP_CACHED_MEMORY_LLC 1
67#define INTEL_AGP_CACHED_MEMORY_LLC_GFDT 2
68#define INTEL_AGP_CACHED_MEMORY_LLC_MLC 3
69#define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT 4
70
71struct intel_gtt_driver { 41struct intel_gtt_driver {
72 unsigned int gen : 8; 42 unsigned int gen : 8;
73 unsigned int is_g33 : 1; 43 unsigned int is_g33 : 1;
74 unsigned int is_pineview : 1; 44 unsigned int is_pineview : 1;
75 unsigned int is_ironlake : 1; 45 unsigned int is_ironlake : 1;
46 unsigned int has_pgtbl_enable : 1;
76 unsigned int dma_mask_size : 8; 47 unsigned int dma_mask_size : 8;
77 /* Chipset specific GTT setup */ 48 /* Chipset specific GTT setup */
78 int (*setup)(void); 49 int (*setup)(void);
@@ -95,13 +66,14 @@ static struct _intel_private {
95 u8 __iomem *registers; 66 u8 __iomem *registers;
96 phys_addr_t gtt_bus_addr; 67 phys_addr_t gtt_bus_addr;
97 phys_addr_t gma_bus_addr; 68 phys_addr_t gma_bus_addr;
98 phys_addr_t pte_bus_addr; 69 u32 PGETBL_save;
99 u32 __iomem *gtt; /* I915G */ 70 u32 __iomem *gtt; /* I915G */
100 int num_dcache_entries; 71 int num_dcache_entries;
101 union { 72 union {
102 void __iomem *i9xx_flush_page; 73 void __iomem *i9xx_flush_page;
103 void *i8xx_flush_page; 74 void *i8xx_flush_page;
104 }; 75 };
76 char *i81x_gtt_table;
105 struct page *i8xx_page; 77 struct page *i8xx_page;
106 struct resource ifp_resource; 78 struct resource ifp_resource;
107 int resource_valid; 79 int resource_valid;
@@ -113,42 +85,31 @@ static struct _intel_private {
113#define IS_G33 intel_private.driver->is_g33 85#define IS_G33 intel_private.driver->is_g33
114#define IS_PINEVIEW intel_private.driver->is_pineview 86#define IS_PINEVIEW intel_private.driver->is_pineview
115#define IS_IRONLAKE intel_private.driver->is_ironlake 87#define IS_IRONLAKE intel_private.driver->is_ironlake
88#define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable
116 89
117static void intel_agp_free_sglist(struct agp_memory *mem) 90int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
118{ 91 struct scatterlist **sg_list, int *num_sg)
119 struct sg_table st;
120
121 st.sgl = mem->sg_list;
122 st.orig_nents = st.nents = mem->page_count;
123
124 sg_free_table(&st);
125
126 mem->sg_list = NULL;
127 mem->num_sg = 0;
128}
129
130static int intel_agp_map_memory(struct agp_memory *mem)
131{ 92{
132 struct sg_table st; 93 struct sg_table st;
133 struct scatterlist *sg; 94 struct scatterlist *sg;
134 int i; 95 int i;
135 96
136 if (mem->sg_list) 97 if (*sg_list)
137 return 0; /* already mapped (for e.g. resume */ 98 return 0; /* already mapped (for e.g. resume */
138 99
139 DBG("try mapping %lu pages\n", (unsigned long)mem->page_count); 100 DBG("try mapping %lu pages\n", (unsigned long)num_entries);
140 101
141 if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL)) 102 if (sg_alloc_table(&st, num_entries, GFP_KERNEL))
142 goto err; 103 goto err;
143 104
144 mem->sg_list = sg = st.sgl; 105 *sg_list = sg = st.sgl;
145 106
146 for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg)) 107 for (i = 0 ; i < num_entries; i++, sg = sg_next(sg))
147 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0); 108 sg_set_page(sg, pages[i], PAGE_SIZE, 0);
148 109
149 mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list, 110 *num_sg = pci_map_sg(intel_private.pcidev, *sg_list,
150 mem->page_count, PCI_DMA_BIDIRECTIONAL); 111 num_entries, PCI_DMA_BIDIRECTIONAL);
151 if (unlikely(!mem->num_sg)) 112 if (unlikely(!*num_sg))
152 goto err; 113 goto err;
153 114
154 return 0; 115 return 0;
@@ -157,90 +118,22 @@ err:
157 sg_free_table(&st); 118 sg_free_table(&st);
158 return -ENOMEM; 119 return -ENOMEM;
159} 120}
121EXPORT_SYMBOL(intel_gtt_map_memory);
160 122
161static void intel_agp_unmap_memory(struct agp_memory *mem) 123void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
162{ 124{
125 struct sg_table st;
163 DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count); 126 DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
164 127
165 pci_unmap_sg(intel_private.pcidev, mem->sg_list, 128 pci_unmap_sg(intel_private.pcidev, sg_list,
166 mem->page_count, PCI_DMA_BIDIRECTIONAL); 129 num_sg, PCI_DMA_BIDIRECTIONAL);
167 intel_agp_free_sglist(mem);
168}
169
170static int intel_i810_fetch_size(void)
171{
172 u32 smram_miscc;
173 struct aper_size_info_fixed *values;
174
175 pci_read_config_dword(intel_private.bridge_dev,
176 I810_SMRAM_MISCC, &smram_miscc);
177 values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
178
179 if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
180 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
181 return 0;
182 }
183 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
184 agp_bridge->current_size = (void *) (values + 1);
185 agp_bridge->aperture_size_idx = 1;
186 return values[1].size;
187 } else {
188 agp_bridge->current_size = (void *) (values);
189 agp_bridge->aperture_size_idx = 0;
190 return values[0].size;
191 }
192
193 return 0;
194}
195
196static int intel_i810_configure(void)
197{
198 struct aper_size_info_fixed *current_size;
199 u32 temp;
200 int i;
201
202 current_size = A_SIZE_FIX(agp_bridge->current_size);
203 130
204 if (!intel_private.registers) { 131 st.sgl = sg_list;
205 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp); 132 st.orig_nents = st.nents = num_sg;
206 temp &= 0xfff80000;
207 133
208 intel_private.registers = ioremap(temp, 128 * 4096); 134 sg_free_table(&st);
209 if (!intel_private.registers) {
210 dev_err(&intel_private.pcidev->dev,
211 "can't remap memory\n");
212 return -ENOMEM;
213 }
214 }
215
216 if ((readl(intel_private.registers+I810_DRAM_CTL)
217 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
218 /* This will need to be dynamically assigned */
219 dev_info(&intel_private.pcidev->dev,
220 "detected 4MB dedicated video ram\n");
221 intel_private.num_dcache_entries = 1024;
222 }
223 pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
224 agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
225 writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
226 readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
227
228 if (agp_bridge->driver->needs_scratch_page) {
229 for (i = 0; i < current_size->num_entries; i++) {
230 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
231 }
232 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
233 }
234 global_cache_flush();
235 return 0;
236}
237
238static void intel_i810_cleanup(void)
239{
240 writel(0, intel_private.registers+I810_PGETBL_CTL);
241 readl(intel_private.registers); /* PCI Posting. */
242 iounmap(intel_private.registers);
243} 135}
136EXPORT_SYMBOL(intel_gtt_unmap_memory);
244 137
245static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode) 138static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
246{ 139{
@@ -277,80 +170,64 @@ static void i8xx_destroy_pages(struct page *page)
277 atomic_dec(&agp_bridge->current_memory_agp); 170 atomic_dec(&agp_bridge->current_memory_agp);
278} 171}
279 172
280static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start, 173#define I810_GTT_ORDER 4
281 int type) 174static int i810_setup(void)
282{ 175{
283 int i, j, num_entries; 176 u32 reg_addr;
284 void *temp; 177 char *gtt_table;
285 int ret = -EINVAL;
286 int mask_type;
287
288 if (mem->page_count == 0)
289 goto out;
290
291 temp = agp_bridge->current_size;
292 num_entries = A_SIZE_FIX(temp)->num_entries;
293 178
294 if ((pg_start + mem->page_count) > num_entries) 179 /* i81x does not preallocate the gtt. It's always 64kb in size. */
295 goto out_err; 180 gtt_table = alloc_gatt_pages(I810_GTT_ORDER);
181 if (gtt_table == NULL)
182 return -ENOMEM;
183 intel_private.i81x_gtt_table = gtt_table;
296 184
185 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
186 reg_addr &= 0xfff80000;
297 187
298 for (j = pg_start; j < (pg_start + mem->page_count); j++) { 188 intel_private.registers = ioremap(reg_addr, KB(64));
299 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) { 189 if (!intel_private.registers)
300 ret = -EBUSY; 190 return -ENOMEM;
301 goto out_err;
302 }
303 }
304 191
305 if (type != mem->type) 192 writel(virt_to_phys(gtt_table) | I810_PGETBL_ENABLED,
306 goto out_err; 193 intel_private.registers+I810_PGETBL_CTL);
307 194
308 mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type); 195 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
309 196
310 switch (mask_type) { 197 if ((readl(intel_private.registers+I810_DRAM_CTL)
311 case AGP_DCACHE_MEMORY: 198 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
312 if (!mem->is_flushed) 199 dev_info(&intel_private.pcidev->dev,
313 global_cache_flush(); 200 "detected 4MB dedicated video ram\n");
314 for (i = pg_start; i < (pg_start + mem->page_count); i++) { 201 intel_private.num_dcache_entries = 1024;
315 writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
316 intel_private.registers+I810_PTE_BASE+(i*4));
317 }
318 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
319 break;
320 case AGP_PHYS_MEMORY:
321 case AGP_NORMAL_MEMORY:
322 if (!mem->is_flushed)
323 global_cache_flush();
324 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
325 writel(agp_bridge->driver->mask_memory(agp_bridge,
326 page_to_phys(mem->pages[i]), mask_type),
327 intel_private.registers+I810_PTE_BASE+(j*4));
328 }
329 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
330 break;
331 default:
332 goto out_err;
333 } 202 }
334 203
335out: 204 return 0;
336 ret = 0;
337out_err:
338 mem->is_flushed = true;
339 return ret;
340} 205}
341 206
342static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start, 207static void i810_cleanup(void)
343 int type) 208{
209 writel(0, intel_private.registers+I810_PGETBL_CTL);
210 free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER);
211}
212
213static int i810_insert_dcache_entries(struct agp_memory *mem, off_t pg_start,
214 int type)
344{ 215{
345 int i; 216 int i;
346 217
347 if (mem->page_count == 0) 218 if ((pg_start + mem->page_count)
348 return 0; 219 > intel_private.num_dcache_entries)
220 return -EINVAL;
221
222 if (!mem->is_flushed)
223 global_cache_flush();
349 224
350 for (i = pg_start; i < (mem->page_count + pg_start); i++) { 225 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
351 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4)); 226 dma_addr_t addr = i << PAGE_SHIFT;
227 intel_private.driver->write_entry(addr,
228 i, type);
352 } 229 }
353 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); 230 readl(intel_private.gtt+i-1);
354 231
355 return 0; 232 return 0;
356} 233}
@@ -397,29 +274,6 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
397 return new; 274 return new;
398} 275}
399 276
400static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
401{
402 struct agp_memory *new;
403
404 if (type == AGP_DCACHE_MEMORY) {
405 if (pg_count != intel_private.num_dcache_entries)
406 return NULL;
407
408 new = agp_create_memory(1);
409 if (new == NULL)
410 return NULL;
411
412 new->type = AGP_DCACHE_MEMORY;
413 new->page_count = pg_count;
414 new->num_scratch_pages = 0;
415 agp_free_page_array(new);
416 return new;
417 }
418 if (type == AGP_PHYS_MEMORY)
419 return alloc_agpphysmem_i8xx(pg_count, type);
420 return NULL;
421}
422
423static void intel_i810_free_by_type(struct agp_memory *curr) 277static void intel_i810_free_by_type(struct agp_memory *curr)
424{ 278{
425 agp_free_key(curr->key); 279 agp_free_key(curr->key);
@@ -437,13 +291,6 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
437 kfree(curr); 291 kfree(curr);
438} 292}
439 293
440static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
441 dma_addr_t addr, int type)
442{
443 /* Type checking must be done elsewhere */
444 return addr | bridge->driver->masks[type].mask;
445}
446
447static int intel_gtt_setup_scratch_page(void) 294static int intel_gtt_setup_scratch_page(void)
448{ 295{
449 struct page *page; 296 struct page *page;
@@ -455,7 +302,7 @@ static int intel_gtt_setup_scratch_page(void)
455 get_page(page); 302 get_page(page);
456 set_pages_uc(page, 1); 303 set_pages_uc(page, 1);
457 304
458 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { 305 if (intel_private.base.needs_dmar) {
459 dma_addr = pci_map_page(intel_private.pcidev, page, 0, 306 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
460 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 307 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
461 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr)) 308 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
@@ -470,34 +317,45 @@ static int intel_gtt_setup_scratch_page(void)
470 return 0; 317 return 0;
471} 318}
472 319
473static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = { 320static void i810_write_entry(dma_addr_t addr, unsigned int entry,
321 unsigned int flags)
322{
323 u32 pte_flags = I810_PTE_VALID;
324
325 switch (flags) {
326 case AGP_DCACHE_MEMORY:
327 pte_flags |= I810_PTE_LOCAL;
328 break;
329 case AGP_USER_CACHED_MEMORY:
330 pte_flags |= I830_PTE_SYSTEM_CACHED;
331 break;
332 }
333
334 writel(addr | pte_flags, intel_private.gtt + entry);
335}
336
337static const struct aper_size_info_fixed intel_fake_agp_sizes[] = {
338 {32, 8192, 3},
339 {64, 16384, 4},
474 {128, 32768, 5}, 340 {128, 32768, 5},
475 /* The 64M mode still requires a 128k gatt */
476 {64, 16384, 5},
477 {256, 65536, 6}, 341 {256, 65536, 6},
478 {512, 131072, 7}, 342 {512, 131072, 7},
479}; 343};
480 344
481static unsigned int intel_gtt_stolen_entries(void) 345static unsigned int intel_gtt_stolen_size(void)
482{ 346{
483 u16 gmch_ctrl; 347 u16 gmch_ctrl;
484 u8 rdct; 348 u8 rdct;
485 int local = 0; 349 int local = 0;
486 static const int ddt[4] = { 0, 16, 32, 64 }; 350 static const int ddt[4] = { 0, 16, 32, 64 };
487 unsigned int overhead_entries, stolen_entries;
488 unsigned int stolen_size = 0; 351 unsigned int stolen_size = 0;
489 352
353 if (INTEL_GTT_GEN == 1)
354 return 0; /* no stolen mem on i81x */
355
490 pci_read_config_word(intel_private.bridge_dev, 356 pci_read_config_word(intel_private.bridge_dev,
491 I830_GMCH_CTRL, &gmch_ctrl); 357 I830_GMCH_CTRL, &gmch_ctrl);
492 358
493 if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
494 overhead_entries = 0;
495 else
496 overhead_entries = intel_private.base.gtt_mappable_entries
497 / 1024;
498
499 overhead_entries += 1; /* BIOS popup */
500
501 if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB || 359 if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
502 intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) { 360 intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
503 switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 361 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
@@ -623,12 +481,7 @@ static unsigned int intel_gtt_stolen_entries(void)
623 } 481 }
624 } 482 }
625 483
626 if (!local && stolen_size > intel_max_stolen) { 484 if (stolen_size > 0) {
627 dev_info(&intel_private.bridge_dev->dev,
628 "detected %dK stolen memory, trimming to %dK\n",
629 stolen_size / KB(1), intel_max_stolen / KB(1));
630 stolen_size = intel_max_stolen;
631 } else if (stolen_size > 0) {
632 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n", 485 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
633 stolen_size / KB(1), local ? "local" : "stolen"); 486 stolen_size / KB(1), local ? "local" : "stolen");
634 } else { 487 } else {
@@ -637,46 +490,88 @@ static unsigned int intel_gtt_stolen_entries(void)
637 stolen_size = 0; 490 stolen_size = 0;
638 } 491 }
639 492
640 stolen_entries = stolen_size/KB(4) - overhead_entries; 493 return stolen_size;
494}
641 495
642 return stolen_entries; 496static void i965_adjust_pgetbl_size(unsigned int size_flag)
497{
498 u32 pgetbl_ctl, pgetbl_ctl2;
499
500 /* ensure that ppgtt is disabled */
501 pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);
502 pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
503 writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);
504
505 /* write the new ggtt size */
506 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
507 pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
508 pgetbl_ctl |= size_flag;
509 writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);
643} 510}
644 511
645static unsigned int intel_gtt_total_entries(void) 512static unsigned int i965_gtt_total_entries(void)
646{ 513{
647 int size; 514 int size;
515 u32 pgetbl_ctl;
516 u16 gmch_ctl;
648 517
649 if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) { 518 pci_read_config_word(intel_private.bridge_dev,
650 u32 pgetbl_ctl; 519 I830_GMCH_CTRL, &gmch_ctl);
651 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
652 520
653 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) { 521 if (INTEL_GTT_GEN == 5) {
654 case I965_PGETBL_SIZE_128KB: 522 switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
655 size = KB(128); 523 case G4x_GMCH_SIZE_1M:
656 break; 524 case G4x_GMCH_SIZE_VT_1M:
657 case I965_PGETBL_SIZE_256KB: 525 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB);
658 size = KB(256);
659 break;
660 case I965_PGETBL_SIZE_512KB:
661 size = KB(512);
662 break;
663 case I965_PGETBL_SIZE_1MB:
664 size = KB(1024);
665 break; 526 break;
666 case I965_PGETBL_SIZE_2MB: 527 case G4x_GMCH_SIZE_VT_1_5M:
667 size = KB(2048); 528 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB);
668 break; 529 break;
669 case I965_PGETBL_SIZE_1_5MB: 530 case G4x_GMCH_SIZE_2M:
670 size = KB(1024 + 512); 531 case G4x_GMCH_SIZE_VT_2M:
532 i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB);
671 break; 533 break;
672 default:
673 dev_info(&intel_private.pcidev->dev,
674 "unknown page table size, assuming 512KB\n");
675 size = KB(512);
676 } 534 }
535 }
677 536
678 return size/4; 537 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
679 } else if (INTEL_GTT_GEN == 6) { 538
539 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
540 case I965_PGETBL_SIZE_128KB:
541 size = KB(128);
542 break;
543 case I965_PGETBL_SIZE_256KB:
544 size = KB(256);
545 break;
546 case I965_PGETBL_SIZE_512KB:
547 size = KB(512);
548 break;
549 /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
550 case I965_PGETBL_SIZE_1MB:
551 size = KB(1024);
552 break;
553 case I965_PGETBL_SIZE_2MB:
554 size = KB(2048);
555 break;
556 case I965_PGETBL_SIZE_1_5MB:
557 size = KB(1024 + 512);
558 break;
559 default:
560 dev_info(&intel_private.pcidev->dev,
561 "unknown page table size, assuming 512KB\n");
562 size = KB(512);
563 }
564
565 return size/4;
566}
567
568static unsigned int intel_gtt_total_entries(void)
569{
570 int size;
571
572 if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)
573 return i965_gtt_total_entries();
574 else if (INTEL_GTT_GEN == 6) {
680 u16 snb_gmch_ctl; 575 u16 snb_gmch_ctl;
681 576
682 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl); 577 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
@@ -706,7 +601,18 @@ static unsigned int intel_gtt_mappable_entries(void)
706{ 601{
707 unsigned int aperture_size; 602 unsigned int aperture_size;
708 603
709 if (INTEL_GTT_GEN == 2) { 604 if (INTEL_GTT_GEN == 1) {
605 u32 smram_miscc;
606
607 pci_read_config_dword(intel_private.bridge_dev,
608 I810_SMRAM_MISCC, &smram_miscc);
609
610 if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
611 == I810_GFX_MEM_WIN_32M)
612 aperture_size = MB(32);
613 else
614 aperture_size = MB(64);
615 } else if (INTEL_GTT_GEN == 2) {
710 u16 gmch_ctrl; 616 u16 gmch_ctrl;
711 617
712 pci_read_config_word(intel_private.bridge_dev, 618 pci_read_config_word(intel_private.bridge_dev,
@@ -739,7 +645,7 @@ static void intel_gtt_cleanup(void)
739 645
740 iounmap(intel_private.gtt); 646 iounmap(intel_private.gtt);
741 iounmap(intel_private.registers); 647 iounmap(intel_private.registers);
742 648
743 intel_gtt_teardown_scratch_page(); 649 intel_gtt_teardown_scratch_page();
744} 650}
745 651
@@ -755,6 +661,14 @@ static int intel_gtt_init(void)
755 intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries(); 661 intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
756 intel_private.base.gtt_total_entries = intel_gtt_total_entries(); 662 intel_private.base.gtt_total_entries = intel_gtt_total_entries();
757 663
664 /* save the PGETBL reg for resume */
665 intel_private.PGETBL_save =
666 readl(intel_private.registers+I810_PGETBL_CTL)
667 & ~I810_PGETBL_ENABLED;
668 /* we only ever restore the register when enabling the PGTBL... */
669 if (HAS_PGTBL_EN)
670 intel_private.PGETBL_save |= I810_PGETBL_ENABLED;
671
758 dev_info(&intel_private.bridge_dev->dev, 672 dev_info(&intel_private.bridge_dev->dev,
759 "detected gtt size: %dK total, %dK mappable\n", 673 "detected gtt size: %dK total, %dK mappable\n",
760 intel_private.base.gtt_total_entries * 4, 674 intel_private.base.gtt_total_entries * 4,
@@ -772,14 +686,7 @@ static int intel_gtt_init(void)
772 686
773 global_cache_flush(); /* FIXME: ? */ 687 global_cache_flush(); /* FIXME: ? */
774 688
775 /* we have to call this as early as possible after the MMIO base address is known */ 689 intel_private.base.stolen_size = intel_gtt_stolen_size();
776 intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
777 if (intel_private.base.gtt_stolen_entries == 0) {
778 intel_private.driver->cleanup();
779 iounmap(intel_private.registers);
780 iounmap(intel_private.gtt);
781 return -ENOMEM;
782 }
783 690
784 ret = intel_gtt_setup_scratch_page(); 691 ret = intel_gtt_setup_scratch_page();
785 if (ret != 0) { 692 if (ret != 0) {
@@ -787,6 +694,8 @@ static int intel_gtt_init(void)
787 return ret; 694 return ret;
788 } 695 }
789 696
697 intel_private.base.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2;
698
790 return 0; 699 return 0;
791} 700}
792 701
@@ -862,25 +771,19 @@ static void i830_write_entry(dma_addr_t addr, unsigned int entry,
862 unsigned int flags) 771 unsigned int flags)
863{ 772{
864 u32 pte_flags = I810_PTE_VALID; 773 u32 pte_flags = I810_PTE_VALID;
865 774
866 switch (flags) { 775 if (flags == AGP_USER_CACHED_MEMORY)
867 case AGP_DCACHE_MEMORY:
868 pte_flags |= I810_PTE_LOCAL;
869 break;
870 case AGP_USER_CACHED_MEMORY:
871 pte_flags |= I830_PTE_SYSTEM_CACHED; 776 pte_flags |= I830_PTE_SYSTEM_CACHED;
872 break;
873 }
874 777
875 writel(addr | pte_flags, intel_private.gtt + entry); 778 writel(addr | pte_flags, intel_private.gtt + entry);
876} 779}
877 780
878static void intel_enable_gtt(void) 781static bool intel_enable_gtt(void)
879{ 782{
880 u32 gma_addr; 783 u32 gma_addr;
881 u16 gmch_ctrl; 784 u8 __iomem *reg;
882 785
883 if (INTEL_GTT_GEN == 2) 786 if (INTEL_GTT_GEN <= 2)
884 pci_read_config_dword(intel_private.pcidev, I810_GMADDR, 787 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
885 &gma_addr); 788 &gma_addr);
886 else 789 else
@@ -889,13 +792,38 @@ static void intel_enable_gtt(void)
889 792
890 intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK); 793 intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
891 794
892 pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl); 795 if (INTEL_GTT_GEN >= 6)
893 gmch_ctrl |= I830_GMCH_ENABLED; 796 return true;
894 pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
895 797
896 writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED, 798 if (INTEL_GTT_GEN == 2) {
897 intel_private.registers+I810_PGETBL_CTL); 799 u16 gmch_ctrl;
898 readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */ 800
801 pci_read_config_word(intel_private.bridge_dev,
802 I830_GMCH_CTRL, &gmch_ctrl);
803 gmch_ctrl |= I830_GMCH_ENABLED;
804 pci_write_config_word(intel_private.bridge_dev,
805 I830_GMCH_CTRL, gmch_ctrl);
806
807 pci_read_config_word(intel_private.bridge_dev,
808 I830_GMCH_CTRL, &gmch_ctrl);
809 if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) {
810 dev_err(&intel_private.pcidev->dev,
811 "failed to enable the GTT: GMCH_CTRL=%x\n",
812 gmch_ctrl);
813 return false;
814 }
815 }
816
817 reg = intel_private.registers+I810_PGETBL_CTL;
818 writel(intel_private.PGETBL_save, reg);
819 if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) {
820 dev_err(&intel_private.pcidev->dev,
821 "failed to enable the GTT: PGETBL=%x [expected %x]\n",
822 readl(reg), intel_private.PGETBL_save);
823 return false;
824 }
825
826 return true;
899} 827}
900 828
901static int i830_setup(void) 829static int i830_setup(void)
@@ -910,8 +838,6 @@ static int i830_setup(void)
910 return -ENOMEM; 838 return -ENOMEM;
911 839
912 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE; 840 intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
913 intel_private.pte_bus_addr =
914 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
915 841
916 intel_i830_setup_flush(); 842 intel_i830_setup_flush();
917 843
@@ -936,12 +862,12 @@ static int intel_fake_agp_configure(void)
936{ 862{
937 int i; 863 int i;
938 864
939 intel_enable_gtt(); 865 if (!intel_enable_gtt())
866 return -EIO;
940 867
941 agp_bridge->gart_bus_addr = intel_private.gma_bus_addr; 868 agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
942 869
943 for (i = intel_private.base.gtt_stolen_entries; 870 for (i = 0; i < intel_private.base.gtt_total_entries; i++) {
944 i < intel_private.base.gtt_total_entries; i++) {
945 intel_private.driver->write_entry(intel_private.scratch_page_dma, 871 intel_private.driver->write_entry(intel_private.scratch_page_dma,
946 i, 0); 872 i, 0);
947 } 873 }
@@ -965,10 +891,10 @@ static bool i830_check_flags(unsigned int flags)
965 return false; 891 return false;
966} 892}
967 893
968static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list, 894void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
969 unsigned int sg_len, 895 unsigned int sg_len,
970 unsigned int pg_start, 896 unsigned int pg_start,
971 unsigned int flags) 897 unsigned int flags)
972{ 898{
973 struct scatterlist *sg; 899 struct scatterlist *sg;
974 unsigned int len, m; 900 unsigned int len, m;
@@ -989,27 +915,34 @@ static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
989 } 915 }
990 readl(intel_private.gtt+j-1); 916 readl(intel_private.gtt+j-1);
991} 917}
918EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
919
920void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
921 struct page **pages, unsigned int flags)
922{
923 int i, j;
924
925 for (i = 0, j = first_entry; i < num_entries; i++, j++) {
926 dma_addr_t addr = page_to_phys(pages[i]);
927 intel_private.driver->write_entry(addr,
928 j, flags);
929 }
930 readl(intel_private.gtt+j-1);
931}
932EXPORT_SYMBOL(intel_gtt_insert_pages);
992 933
993static int intel_fake_agp_insert_entries(struct agp_memory *mem, 934static int intel_fake_agp_insert_entries(struct agp_memory *mem,
994 off_t pg_start, int type) 935 off_t pg_start, int type)
995{ 936{
996 int i, j;
997 int ret = -EINVAL; 937 int ret = -EINVAL;
998 938
939 if (INTEL_GTT_GEN == 1 && type == AGP_DCACHE_MEMORY)
940 return i810_insert_dcache_entries(mem, pg_start, type);
941
999 if (mem->page_count == 0) 942 if (mem->page_count == 0)
1000 goto out; 943 goto out;
1001 944
1002 if (pg_start < intel_private.base.gtt_stolen_entries) { 945 if (pg_start + mem->page_count > intel_private.base.gtt_total_entries)
1003 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1004 "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
1005 pg_start, intel_private.base.gtt_stolen_entries);
1006
1007 dev_info(&intel_private.pcidev->dev,
1008 "trying to insert into local/stolen memory\n");
1009 goto out_err;
1010 }
1011
1012 if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
1013 goto out_err; 946 goto out_err;
1014 947
1015 if (type != mem->type) 948 if (type != mem->type)
@@ -1021,21 +954,17 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem,
1021 if (!mem->is_flushed) 954 if (!mem->is_flushed)
1022 global_cache_flush(); 955 global_cache_flush();
1023 956
1024 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) { 957 if (intel_private.base.needs_dmar) {
1025 ret = intel_agp_map_memory(mem); 958 ret = intel_gtt_map_memory(mem->pages, mem->page_count,
959 &mem->sg_list, &mem->num_sg);
1026 if (ret != 0) 960 if (ret != 0)
1027 return ret; 961 return ret;
1028 962
1029 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg, 963 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
1030 pg_start, type); 964 pg_start, type);
1031 } else { 965 } else
1032 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { 966 intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
1033 dma_addr_t addr = page_to_phys(mem->pages[i]); 967 type);
1034 intel_private.driver->write_entry(addr,
1035 j, type);
1036 }
1037 readl(intel_private.gtt+j-1);
1038 }
1039 968
1040out: 969out:
1041 ret = 0; 970 ret = 0;
@@ -1044,40 +973,54 @@ out_err:
1044 return ret; 973 return ret;
1045} 974}
1046 975
976void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries)
977{
978 unsigned int i;
979
980 for (i = first_entry; i < (first_entry + num_entries); i++) {
981 intel_private.driver->write_entry(intel_private.scratch_page_dma,
982 i, 0);
983 }
984 readl(intel_private.gtt+i-1);
985}
986EXPORT_SYMBOL(intel_gtt_clear_range);
987
1047static int intel_fake_agp_remove_entries(struct agp_memory *mem, 988static int intel_fake_agp_remove_entries(struct agp_memory *mem,
1048 off_t pg_start, int type) 989 off_t pg_start, int type)
1049{ 990{
1050 int i;
1051
1052 if (mem->page_count == 0) 991 if (mem->page_count == 0)
1053 return 0; 992 return 0;
1054 993
1055 if (pg_start < intel_private.base.gtt_stolen_entries) { 994 if (intel_private.base.needs_dmar) {
1056 dev_info(&intel_private.pcidev->dev, 995 intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);
1057 "trying to disable local/stolen memory\n"); 996 mem->sg_list = NULL;
1058 return -EINVAL; 997 mem->num_sg = 0;
1059 } 998 }
1060 999
1061 if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) 1000 intel_gtt_clear_range(pg_start, mem->page_count);
1062 intel_agp_unmap_memory(mem);
1063
1064 for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1065 intel_private.driver->write_entry(intel_private.scratch_page_dma,
1066 i, 0);
1067 }
1068 readl(intel_private.gtt+i-1);
1069 1001
1070 return 0; 1002 return 0;
1071} 1003}
1072 1004
1073static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
1074{
1075 intel_private.driver->chipset_flush();
1076}
1077
1078static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count, 1005static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1079 int type) 1006 int type)
1080{ 1007{
1008 struct agp_memory *new;
1009
1010 if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) {
1011 if (pg_count != intel_private.num_dcache_entries)
1012 return NULL;
1013
1014 new = agp_create_memory(1);
1015 if (new == NULL)
1016 return NULL;
1017
1018 new->type = AGP_DCACHE_MEMORY;
1019 new->page_count = pg_count;
1020 new->num_scratch_pages = 0;
1021 agp_free_page_array(new);
1022 return new;
1023 }
1081 if (type == AGP_PHYS_MEMORY) 1024 if (type == AGP_PHYS_MEMORY)
1082 return alloc_agpphysmem_i8xx(pg_count, type); 1025 return alloc_agpphysmem_i8xx(pg_count, type);
1083 /* always return NULL for other allocation types for now */ 1026 /* always return NULL for other allocation types for now */
@@ -1274,40 +1217,11 @@ static int i9xx_setup(void)
1274 intel_private.gtt_bus_addr = reg_addr + gtt_offset; 1217 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1275 } 1218 }
1276 1219
1277 intel_private.pte_bus_addr =
1278 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1279
1280 intel_i9xx_setup_flush(); 1220 intel_i9xx_setup_flush();
1281 1221
1282 return 0; 1222 return 0;
1283} 1223}
1284 1224
1285static const struct agp_bridge_driver intel_810_driver = {
1286 .owner = THIS_MODULE,
1287 .aperture_sizes = intel_i810_sizes,
1288 .size_type = FIXED_APER_SIZE,
1289 .num_aperture_sizes = 2,
1290 .needs_scratch_page = true,
1291 .configure = intel_i810_configure,
1292 .fetch_size = intel_i810_fetch_size,
1293 .cleanup = intel_i810_cleanup,
1294 .mask_memory = intel_i810_mask_memory,
1295 .masks = intel_i810_masks,
1296 .agp_enable = intel_fake_agp_enable,
1297 .cache_flush = global_cache_flush,
1298 .create_gatt_table = agp_generic_create_gatt_table,
1299 .free_gatt_table = agp_generic_free_gatt_table,
1300 .insert_memory = intel_i810_insert_entries,
1301 .remove_memory = intel_i810_remove_entries,
1302 .alloc_by_type = intel_i810_alloc_by_type,
1303 .free_by_type = intel_i810_free_by_type,
1304 .agp_alloc_page = agp_generic_alloc_page,
1305 .agp_alloc_pages = agp_generic_alloc_pages,
1306 .agp_destroy_page = agp_generic_destroy_page,
1307 .agp_destroy_pages = agp_generic_destroy_pages,
1308 .agp_type_to_mask_type = agp_generic_type_to_mask_type,
1309};
1310
1311static const struct agp_bridge_driver intel_fake_agp_driver = { 1225static const struct agp_bridge_driver intel_fake_agp_driver = {
1312 .owner = THIS_MODULE, 1226 .owner = THIS_MODULE,
1313 .size_type = FIXED_APER_SIZE, 1227 .size_type = FIXED_APER_SIZE,
@@ -1328,15 +1242,20 @@ static const struct agp_bridge_driver intel_fake_agp_driver = {
1328 .agp_alloc_pages = agp_generic_alloc_pages, 1242 .agp_alloc_pages = agp_generic_alloc_pages,
1329 .agp_destroy_page = agp_generic_destroy_page, 1243 .agp_destroy_page = agp_generic_destroy_page,
1330 .agp_destroy_pages = agp_generic_destroy_pages, 1244 .agp_destroy_pages = agp_generic_destroy_pages,
1331 .chipset_flush = intel_fake_agp_chipset_flush,
1332}; 1245};
1333 1246
1334static const struct intel_gtt_driver i81x_gtt_driver = { 1247static const struct intel_gtt_driver i81x_gtt_driver = {
1335 .gen = 1, 1248 .gen = 1,
1249 .has_pgtbl_enable = 1,
1336 .dma_mask_size = 32, 1250 .dma_mask_size = 32,
1251 .setup = i810_setup,
1252 .cleanup = i810_cleanup,
1253 .check_flags = i830_check_flags,
1254 .write_entry = i810_write_entry,
1337}; 1255};
1338static const struct intel_gtt_driver i8xx_gtt_driver = { 1256static const struct intel_gtt_driver i8xx_gtt_driver = {
1339 .gen = 2, 1257 .gen = 2,
1258 .has_pgtbl_enable = 1,
1340 .setup = i830_setup, 1259 .setup = i830_setup,
1341 .cleanup = i830_cleanup, 1260 .cleanup = i830_cleanup,
1342 .write_entry = i830_write_entry, 1261 .write_entry = i830_write_entry,
@@ -1346,10 +1265,11 @@ static const struct intel_gtt_driver i8xx_gtt_driver = {
1346}; 1265};
1347static const struct intel_gtt_driver i915_gtt_driver = { 1266static const struct intel_gtt_driver i915_gtt_driver = {
1348 .gen = 3, 1267 .gen = 3,
1268 .has_pgtbl_enable = 1,
1349 .setup = i9xx_setup, 1269 .setup = i9xx_setup,
1350 .cleanup = i9xx_cleanup, 1270 .cleanup = i9xx_cleanup,
1351 /* i945 is the last gpu to need phys mem (for overlay and cursors). */ 1271 /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1352 .write_entry = i830_write_entry, 1272 .write_entry = i830_write_entry,
1353 .dma_mask_size = 32, 1273 .dma_mask_size = 32,
1354 .check_flags = i830_check_flags, 1274 .check_flags = i830_check_flags,
1355 .chipset_flush = i9xx_chipset_flush, 1275 .chipset_flush = i9xx_chipset_flush,
@@ -1376,6 +1296,7 @@ static const struct intel_gtt_driver pineview_gtt_driver = {
1376}; 1296};
1377static const struct intel_gtt_driver i965_gtt_driver = { 1297static const struct intel_gtt_driver i965_gtt_driver = {
1378 .gen = 4, 1298 .gen = 4,
1299 .has_pgtbl_enable = 1,
1379 .setup = i9xx_setup, 1300 .setup = i9xx_setup,
1380 .cleanup = i9xx_cleanup, 1301 .cleanup = i9xx_cleanup,
1381 .write_entry = i965_write_entry, 1302 .write_entry = i965_write_entry,
@@ -1419,93 +1340,92 @@ static const struct intel_gtt_driver sandybridge_gtt_driver = {
1419static const struct intel_gtt_driver_description { 1340static const struct intel_gtt_driver_description {
1420 unsigned int gmch_chip_id; 1341 unsigned int gmch_chip_id;
1421 char *name; 1342 char *name;
1422 const struct agp_bridge_driver *gmch_driver;
1423 const struct intel_gtt_driver *gtt_driver; 1343 const struct intel_gtt_driver *gtt_driver;
1424} intel_gtt_chipsets[] = { 1344} intel_gtt_chipsets[] = {
1425 { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver, 1345 { PCI_DEVICE_ID_INTEL_82810_IG1, "i810",
1426 &i81x_gtt_driver}, 1346 &i81x_gtt_driver},
1427 { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver, 1347 { PCI_DEVICE_ID_INTEL_82810_IG3, "i810",
1428 &i81x_gtt_driver}, 1348 &i81x_gtt_driver},
1429 { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver, 1349 { PCI_DEVICE_ID_INTEL_82810E_IG, "i810",
1430 &i81x_gtt_driver}, 1350 &i81x_gtt_driver},
1431 { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver, 1351 { PCI_DEVICE_ID_INTEL_82815_CGC, "i815",
1432 &i81x_gtt_driver}, 1352 &i81x_gtt_driver},
1433 { PCI_DEVICE_ID_INTEL_82830_CGC, "830M", 1353 { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1434 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1354 &i8xx_gtt_driver},
1435 { PCI_DEVICE_ID_INTEL_82845G_IG, "830M", 1355 { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1436 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1356 &i8xx_gtt_driver},
1437 { PCI_DEVICE_ID_INTEL_82854_IG, "854", 1357 { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1438 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1358 &i8xx_gtt_driver},
1439 { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM", 1359 { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1440 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1360 &i8xx_gtt_driver},
1441 { PCI_DEVICE_ID_INTEL_82865_IG, "865", 1361 { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1442 &intel_fake_agp_driver, &i8xx_gtt_driver}, 1362 &i8xx_gtt_driver},
1443 { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)", 1363 { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1444 &intel_fake_agp_driver, &i915_gtt_driver }, 1364 &i915_gtt_driver },
1445 { PCI_DEVICE_ID_INTEL_82915G_IG, "915G", 1365 { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1446 &intel_fake_agp_driver, &i915_gtt_driver }, 1366 &i915_gtt_driver },
1447 { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM", 1367 { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1448 &intel_fake_agp_driver, &i915_gtt_driver }, 1368 &i915_gtt_driver },
1449 { PCI_DEVICE_ID_INTEL_82945G_IG, "945G", 1369 { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1450 &intel_fake_agp_driver, &i915_gtt_driver }, 1370 &i915_gtt_driver },
1451 { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM", 1371 { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1452 &intel_fake_agp_driver, &i915_gtt_driver }, 1372 &i915_gtt_driver },
1453 { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME", 1373 { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1454 &intel_fake_agp_driver, &i915_gtt_driver }, 1374 &i915_gtt_driver },
1455 { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ", 1375 { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1456 &intel_fake_agp_driver, &i965_gtt_driver }, 1376 &i965_gtt_driver },
1457 { PCI_DEVICE_ID_INTEL_82G35_IG, "G35", 1377 { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1458 &intel_fake_agp_driver, &i965_gtt_driver }, 1378 &i965_gtt_driver },
1459 { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q", 1379 { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1460 &intel_fake_agp_driver, &i965_gtt_driver }, 1380 &i965_gtt_driver },
1461 { PCI_DEVICE_ID_INTEL_82965G_IG, "965G", 1381 { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1462 &intel_fake_agp_driver, &i965_gtt_driver }, 1382 &i965_gtt_driver },
1463 { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM", 1383 { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1464 &intel_fake_agp_driver, &i965_gtt_driver }, 1384 &i965_gtt_driver },
1465 { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE", 1385 { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1466 &intel_fake_agp_driver, &i965_gtt_driver }, 1386 &i965_gtt_driver },
1467 { PCI_DEVICE_ID_INTEL_G33_IG, "G33", 1387 { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1468 &intel_fake_agp_driver, &g33_gtt_driver }, 1388 &g33_gtt_driver },
1469 { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35", 1389 { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1470 &intel_fake_agp_driver, &g33_gtt_driver }, 1390 &g33_gtt_driver },
1471 { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33", 1391 { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1472 &intel_fake_agp_driver, &g33_gtt_driver }, 1392 &g33_gtt_driver },
1473 { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150", 1393 { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1474 &intel_fake_agp_driver, &pineview_gtt_driver }, 1394 &pineview_gtt_driver },
1475 { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150", 1395 { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1476 &intel_fake_agp_driver, &pineview_gtt_driver }, 1396 &pineview_gtt_driver },
1477 { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45", 1397 { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1478 &intel_fake_agp_driver, &g4x_gtt_driver }, 1398 &g4x_gtt_driver },
1479 { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake", 1399 { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1480 &intel_fake_agp_driver, &g4x_gtt_driver }, 1400 &g4x_gtt_driver },
1481 { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43", 1401 { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1482 &intel_fake_agp_driver, &g4x_gtt_driver }, 1402 &g4x_gtt_driver },
1483 { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43", 1403 { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1484 &intel_fake_agp_driver, &g4x_gtt_driver }, 1404 &g4x_gtt_driver },
1485 { PCI_DEVICE_ID_INTEL_B43_IG, "B43", 1405 { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1486 &intel_fake_agp_driver, &g4x_gtt_driver }, 1406 &g4x_gtt_driver },
1487 { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43", 1407 { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1488 &intel_fake_agp_driver, &g4x_gtt_driver }, 1408 &g4x_gtt_driver },
1489 { PCI_DEVICE_ID_INTEL_G41_IG, "G41", 1409 { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1490 &intel_fake_agp_driver, &g4x_gtt_driver }, 1410 &g4x_gtt_driver },
1491 { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG, 1411 { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1492 "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, 1412 "HD Graphics", &ironlake_gtt_driver },
1493 { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG, 1413 { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1494 "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver }, 1414 "HD Graphics", &ironlake_gtt_driver },
1495 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG, 1415 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1496 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1416 "Sandybridge", &sandybridge_gtt_driver },
1497 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG, 1417 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1498 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1418 "Sandybridge", &sandybridge_gtt_driver },
1499 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG, 1419 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1500 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1420 "Sandybridge", &sandybridge_gtt_driver },
1501 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG, 1421 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1502 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1422 "Sandybridge", &sandybridge_gtt_driver },
1503 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG, 1423 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1504 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1424 "Sandybridge", &sandybridge_gtt_driver },
1505 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG, 1425 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1506 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1426 "Sandybridge", &sandybridge_gtt_driver },
1507 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG, 1427 { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1508 "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver }, 1428 "Sandybridge", &sandybridge_gtt_driver },
1509 { 0, NULL, NULL } 1429 { 0, NULL, NULL }
1510}; 1430};
1511 1431
@@ -1530,21 +1450,20 @@ int intel_gmch_probe(struct pci_dev *pdev,
1530 struct agp_bridge_data *bridge) 1450 struct agp_bridge_data *bridge)
1531{ 1451{
1532 int i, mask; 1452 int i, mask;
1533 bridge->driver = NULL; 1453 intel_private.driver = NULL;
1534 1454
1535 for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) { 1455 for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1536 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) { 1456 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1537 bridge->driver = 1457 intel_private.driver =
1538 intel_gtt_chipsets[i].gmch_driver;
1539 intel_private.driver =
1540 intel_gtt_chipsets[i].gtt_driver; 1458 intel_gtt_chipsets[i].gtt_driver;
1541 break; 1459 break;
1542 } 1460 }
1543 } 1461 }
1544 1462
1545 if (!bridge->driver) 1463 if (!intel_private.driver)
1546 return 0; 1464 return 0;
1547 1465
1466 bridge->driver = &intel_fake_agp_driver;
1548 bridge->dev_private_data = &intel_private; 1467 bridge->dev_private_data = &intel_private;
1549 bridge->dev = pdev; 1468 bridge->dev = pdev;
1550 1469
@@ -1560,8 +1479,8 @@ int intel_gmch_probe(struct pci_dev *pdev,
1560 pci_set_consistent_dma_mask(intel_private.pcidev, 1479 pci_set_consistent_dma_mask(intel_private.pcidev,
1561 DMA_BIT_MASK(mask)); 1480 DMA_BIT_MASK(mask));
1562 1481
1563 if (bridge->driver == &intel_810_driver) 1482 /*if (bridge->driver == &intel_810_driver)
1564 return 1; 1483 return 1;*/
1565 1484
1566 if (intel_gtt_init() != 0) 1485 if (intel_gtt_init() != 0)
1567 return 0; 1486 return 0;
@@ -1570,12 +1489,19 @@ int intel_gmch_probe(struct pci_dev *pdev,
1570} 1489}
1571EXPORT_SYMBOL(intel_gmch_probe); 1490EXPORT_SYMBOL(intel_gmch_probe);
1572 1491
1573struct intel_gtt *intel_gtt_get(void) 1492const struct intel_gtt *intel_gtt_get(void)
1574{ 1493{
1575 return &intel_private.base; 1494 return &intel_private.base;
1576} 1495}
1577EXPORT_SYMBOL(intel_gtt_get); 1496EXPORT_SYMBOL(intel_gtt_get);
1578 1497
1498void intel_gtt_chipset_flush(void)
1499{
1500 if (intel_private.driver->chipset_flush)
1501 intel_private.driver->chipset_flush();
1502}
1503EXPORT_SYMBOL(intel_gtt_chipset_flush);
1504
1579void intel_gmch_remove(struct pci_dev *pdev) 1505void intel_gmch_remove(struct pci_dev *pdev)
1580{ 1506{
1581 if (intel_private.pcidev) 1507 if (intel_private.pcidev)
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index 252fdb98b73a..0cb2ba50af53 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -466,10 +466,4 @@ drm_agp_bind_pages(struct drm_device *dev,
466} 466}
467EXPORT_SYMBOL(drm_agp_bind_pages); 467EXPORT_SYMBOL(drm_agp_bind_pages);
468 468
469void drm_agp_chipset_flush(struct drm_device *dev)
470{
471 agp_flush_chipset(dev->agp->bridge);
472}
473EXPORT_SYMBOL(drm_agp_chipset_flush);
474
475#endif /* __OS_HAS_AGP */ 469#endif /* __OS_HAS_AGP */
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 2d4e17a004db..952b3d4fb2a6 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -336,7 +336,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
336 struct drm_framebuffer *old_fb) 336 struct drm_framebuffer *old_fb)
337{ 337{
338 struct drm_device *dev = crtc->dev; 338 struct drm_device *dev = crtc->dev;
339 struct drm_display_mode *adjusted_mode, saved_mode; 339 struct drm_display_mode *adjusted_mode, saved_mode, saved_hwmode;
340 struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; 340 struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
341 struct drm_encoder_helper_funcs *encoder_funcs; 341 struct drm_encoder_helper_funcs *encoder_funcs;
342 int saved_x, saved_y; 342 int saved_x, saved_y;
@@ -350,6 +350,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
350 if (!crtc->enabled) 350 if (!crtc->enabled)
351 return true; 351 return true;
352 352
353 saved_hwmode = crtc->hwmode;
353 saved_mode = crtc->mode; 354 saved_mode = crtc->mode;
354 saved_x = crtc->x; 355 saved_x = crtc->x;
355 saved_y = crtc->y; 356 saved_y = crtc->y;
@@ -427,11 +428,21 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
427 428
428 } 429 }
429 430
431 /* Store real post-adjustment hardware mode. */
432 crtc->hwmode = *adjusted_mode;
433
434 /* Calculate and store various constants which
435 * are later needed by vblank and swap-completion
436 * timestamping. They are derived from true hwmode.
437 */
438 drm_calc_timestamping_constants(crtc);
439
430 /* XXX free adjustedmode */ 440 /* XXX free adjustedmode */
431 drm_mode_destroy(dev, adjusted_mode); 441 drm_mode_destroy(dev, adjusted_mode);
432 /* FIXME: add subpixel order */ 442 /* FIXME: add subpixel order */
433done: 443done:
434 if (!ret) { 444 if (!ret) {
445 crtc->hwmode = saved_hwmode;
435 crtc->mode = saved_mode; 446 crtc->mode = saved_mode;
436 crtc->x = saved_x; 447 crtc->x = saved_x;
437 crtc->y = saved_y; 448 crtc->y = saved_y;
@@ -650,6 +661,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
650 old_fb)) { 661 old_fb)) {
651 DRM_ERROR("failed to set mode on [CRTC:%d]\n", 662 DRM_ERROR("failed to set mode on [CRTC:%d]\n",
652 set->crtc->base.id); 663 set->crtc->base.id);
664 set->crtc->fb = old_fb;
653 ret = -EINVAL; 665 ret = -EINVAL;
654 goto fail; 666 goto fail;
655 } 667 }
@@ -664,8 +676,10 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
664 set->crtc->fb = set->fb; 676 set->crtc->fb = set->fb;
665 ret = crtc_funcs->mode_set_base(set->crtc, 677 ret = crtc_funcs->mode_set_base(set->crtc,
666 set->x, set->y, old_fb); 678 set->x, set->y, old_fb);
667 if (ret != 0) 679 if (ret != 0) {
680 set->crtc->fb = old_fb;
668 goto fail; 681 goto fail;
682 }
669 } 683 }
670 DRM_DEBUG_KMS("Setting connector DPMS state to on\n"); 684 DRM_DEBUG_KMS("Setting connector DPMS state to on\n");
671 for (i = 0; i < set->num_connectors; i++) { 685 for (i = 0; i < set->num_connectors; i++) {
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index d2849e4ea4d0..0307d601f5e5 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -607,6 +607,25 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
607} 607}
608EXPORT_SYMBOL(drm_fb_helper_fini); 608EXPORT_SYMBOL(drm_fb_helper_fini);
609 609
610void drm_fb_helper_fill_fix(struct fb_info *info, struct drm_framebuffer *fb)
611{
612 info->fix.type = FB_TYPE_PACKED_PIXELS;
613 info->fix.visual = fb->depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
614 FB_VISUAL_TRUECOLOR;
615 info->fix.mmio_start = 0;
616 info->fix.mmio_len = 0;
617 info->fix.type_aux = 0;
618 info->fix.xpanstep = 1; /* doing it in hw */
619 info->fix.ypanstep = 1; /* doing it in hw */
620 info->fix.ywrapstep = 0;
621 info->fix.accel = FB_ACCEL_NONE;
622 info->fix.type_aux = 0;
623
624 info->fix.line_length = fb->pitch;
625 return;
626}
627EXPORT_SYMBOL(drm_fb_helper_fill_fix);
628
610static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green, 629static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
611 u16 blue, u16 regno, struct fb_info *info) 630 u16 blue, u16 regno, struct fb_info *info)
612{ 631{
@@ -816,6 +835,7 @@ int drm_fb_helper_set_par(struct fb_info *info)
816 mutex_unlock(&dev->mode_config.mutex); 835 mutex_unlock(&dev->mode_config.mutex);
817 return ret; 836 return ret;
818 } 837 }
838 drm_fb_helper_fill_fix(info, fb_helper->fb);
819 } 839 }
820 mutex_unlock(&dev->mode_config.mutex); 840 mutex_unlock(&dev->mode_config.mutex);
821 841
@@ -953,6 +973,7 @@ int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
953 973
954 if (new_fb) { 974 if (new_fb) {
955 info->var.pixclock = 0; 975 info->var.pixclock = 0;
976 drm_fb_helper_fill_fix(info, fb_helper->fb);
956 if (register_framebuffer(info) < 0) { 977 if (register_framebuffer(info) < 0) {
957 return -EINVAL; 978 return -EINVAL;
958 } 979 }
@@ -979,24 +1000,6 @@ int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
979} 1000}
980EXPORT_SYMBOL(drm_fb_helper_single_fb_probe); 1001EXPORT_SYMBOL(drm_fb_helper_single_fb_probe);
981 1002
982void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
983 uint32_t depth)
984{
985 info->fix.type = FB_TYPE_PACKED_PIXELS;
986 info->fix.visual = depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
987 FB_VISUAL_TRUECOLOR;
988 info->fix.type_aux = 0;
989 info->fix.xpanstep = 1; /* doing it in hw */
990 info->fix.ypanstep = 1; /* doing it in hw */
991 info->fix.ywrapstep = 0;
992 info->fix.accel = FB_ACCEL_NONE;
993 info->fix.type_aux = 0;
994
995 info->fix.line_length = pitch;
996 return;
997}
998EXPORT_SYMBOL(drm_fb_helper_fill_fix);
999
1000void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper, 1003void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper,
1001 uint32_t fb_width, uint32_t fb_height) 1004 uint32_t fb_width, uint32_t fb_height)
1002{ 1005{
@@ -1005,6 +1008,7 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
1005 info->var.xres_virtual = fb->width; 1008 info->var.xres_virtual = fb->width;
1006 info->var.yres_virtual = fb->height; 1009 info->var.yres_virtual = fb->height;
1007 info->var.bits_per_pixel = fb->bits_per_pixel; 1010 info->var.bits_per_pixel = fb->bits_per_pixel;
1011 info->var.accel_flags = FB_ACCELF_TEXT;
1008 info->var.xoffset = 0; 1012 info->var.xoffset = 0;
1009 info->var.yoffset = 0; 1013 info->var.yoffset = 0;
1010 info->var.activate = FB_ACTIVATE_NOW; 1014 info->var.activate = FB_ACTIVATE_NOW;
@@ -1530,3 +1534,24 @@ bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
1530} 1534}
1531EXPORT_SYMBOL(drm_fb_helper_hotplug_event); 1535EXPORT_SYMBOL(drm_fb_helper_hotplug_event);
1532 1536
1537/* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EMBEDDED)
1538 * but the module doesn't depend on any fb console symbols. At least
1539 * attempt to load fbcon to avoid leaving the system without a usable console.
1540 */
1541#if defined(CONFIG_FRAMEBUFFER_CONSOLE_MODULE) && !defined(CONFIG_EMBEDDED)
1542static int __init drm_fb_helper_modinit(void)
1543{
1544 const char *name = "fbcon";
1545 struct module *fbcon;
1546
1547 mutex_lock(&module_mutex);
1548 fbcon = find_module(name);
1549 mutex_unlock(&module_mutex);
1550
1551 if (!fbcon)
1552 request_module_nowait(name);
1553 return 0;
1554}
1555
1556module_init(drm_fb_helper_modinit);
1557#endif
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index a39794bac04b..2ec7d48fc4a8 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -236,6 +236,8 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
236 return -EBUSY; /* No exclusive opens */ 236 return -EBUSY; /* No exclusive opens */
237 if (!drm_cpu_valid()) 237 if (!drm_cpu_valid())
238 return -EINVAL; 238 return -EINVAL;
239 if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
240 return -EINVAL;
239 241
240 DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor_id); 242 DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor_id);
241 243
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 16d5155edad1..0054e957203f 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -40,6 +40,22 @@
40#include <linux/slab.h> 40#include <linux/slab.h>
41 41
42#include <linux/vgaarb.h> 42#include <linux/vgaarb.h>
43
44/* Access macro for slots in vblank timestamp ringbuffer. */
45#define vblanktimestamp(dev, crtc, count) ( \
46 (dev)->_vblank_time[(crtc) * DRM_VBLANKTIME_RBSIZE + \
47 ((count) % DRM_VBLANKTIME_RBSIZE)])
48
49/* Retry timestamp calculation up to 3 times to satisfy
50 * drm_timestamp_precision before giving up.
51 */
52#define DRM_TIMESTAMP_MAXRETRIES 3
53
54/* Threshold in nanoseconds for detection of redundant
55 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
56 */
57#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
58
43/** 59/**
44 * Get interrupt from bus id. 60 * Get interrupt from bus id.
45 * 61 *
@@ -77,6 +93,87 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
77 return 0; 93 return 0;
78} 94}
79 95
96/*
97 * Clear vblank timestamp buffer for a crtc.
98 */
99static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
100{
101 memset(&dev->_vblank_time[crtc * DRM_VBLANKTIME_RBSIZE], 0,
102 DRM_VBLANKTIME_RBSIZE * sizeof(struct timeval));
103}
104
105/*
106 * Disable vblank irq's on crtc, make sure that last vblank count
107 * of hardware and corresponding consistent software vblank counter
108 * are preserved, even if there are any spurious vblank irq's after
109 * disable.
110 */
111static void vblank_disable_and_save(struct drm_device *dev, int crtc)
112{
113 unsigned long irqflags;
114 u32 vblcount;
115 s64 diff_ns;
116 int vblrc;
117 struct timeval tvblank;
118
119 /* Prevent vblank irq processing while disabling vblank irqs,
120 * so no updates of timestamps or count can happen after we've
121 * disabled. Needed to prevent races in case of delayed irq's.
122 * Disable preemption, so vblank_time_lock is held as short as
123 * possible, even under a kernel with PREEMPT_RT patches.
124 */
125 preempt_disable();
126 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
127
128 dev->driver->disable_vblank(dev, crtc);
129 dev->vblank_enabled[crtc] = 0;
130
131 /* No further vblank irq's will be processed after
132 * this point. Get current hardware vblank count and
133 * vblank timestamp, repeat until they are consistent.
134 *
135 * FIXME: There is still a race condition here and in
136 * drm_update_vblank_count() which can cause off-by-one
137 * reinitialization of software vblank counter. If gpu
138 * vblank counter doesn't increment exactly at the leading
139 * edge of a vblank interval, then we can lose 1 count if
140 * we happen to execute between start of vblank and the
141 * delayed gpu counter increment.
142 */
143 do {
144 dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);
145 vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
146 } while (dev->last_vblank[crtc] != dev->driver->get_vblank_counter(dev, crtc));
147
148 /* Compute time difference to stored timestamp of last vblank
149 * as updated by last invocation of drm_handle_vblank() in vblank irq.
150 */
151 vblcount = atomic_read(&dev->_vblank_count[crtc]);
152 diff_ns = timeval_to_ns(&tvblank) -
153 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
154
155 /* If there is at least 1 msec difference between the last stored
156 * timestamp and tvblank, then we are currently executing our
157 * disable inside a new vblank interval, the tvblank timestamp
158 * corresponds to this new vblank interval and the irq handler
159 * for this vblank didn't run yet and won't run due to our disable.
160 * Therefore we need to do the job of drm_handle_vblank() and
161 * increment the vblank counter by one to account for this vblank.
162 *
163 * Skip this step if there isn't any high precision timestamp
164 * available. In that case we can't account for this and just
165 * hope for the best.
166 */
167 if ((vblrc > 0) && (abs(diff_ns) > 1000000))
168 atomic_inc(&dev->_vblank_count[crtc]);
169
170 /* Invalidate all timestamps while vblank irq's are off. */
171 clear_vblank_timestamps(dev, crtc);
172
173 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
174 preempt_enable();
175}
176
80static void vblank_disable_fn(unsigned long arg) 177static void vblank_disable_fn(unsigned long arg)
81{ 178{
82 struct drm_device *dev = (struct drm_device *)arg; 179 struct drm_device *dev = (struct drm_device *)arg;
@@ -91,10 +188,7 @@ static void vblank_disable_fn(unsigned long arg)
91 if (atomic_read(&dev->vblank_refcount[i]) == 0 && 188 if (atomic_read(&dev->vblank_refcount[i]) == 0 &&
92 dev->vblank_enabled[i]) { 189 dev->vblank_enabled[i]) {
93 DRM_DEBUG("disabling vblank on crtc %d\n", i); 190 DRM_DEBUG("disabling vblank on crtc %d\n", i);
94 dev->last_vblank[i] = 191 vblank_disable_and_save(dev, i);
95 dev->driver->get_vblank_counter(dev, i);
96 dev->driver->disable_vblank(dev, i);
97 dev->vblank_enabled[i] = 0;
98 } 192 }
99 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 193 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
100 } 194 }
@@ -117,6 +211,7 @@ void drm_vblank_cleanup(struct drm_device *dev)
117 kfree(dev->last_vblank); 211 kfree(dev->last_vblank);
118 kfree(dev->last_vblank_wait); 212 kfree(dev->last_vblank_wait);
119 kfree(dev->vblank_inmodeset); 213 kfree(dev->vblank_inmodeset);
214 kfree(dev->_vblank_time);
120 215
121 dev->num_crtcs = 0; 216 dev->num_crtcs = 0;
122} 217}
@@ -129,6 +224,8 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
129 setup_timer(&dev->vblank_disable_timer, vblank_disable_fn, 224 setup_timer(&dev->vblank_disable_timer, vblank_disable_fn,
130 (unsigned long)dev); 225 (unsigned long)dev);
131 spin_lock_init(&dev->vbl_lock); 226 spin_lock_init(&dev->vbl_lock);
227 spin_lock_init(&dev->vblank_time_lock);
228
132 dev->num_crtcs = num_crtcs; 229 dev->num_crtcs = num_crtcs;
133 230
134 dev->vbl_queue = kmalloc(sizeof(wait_queue_head_t) * num_crtcs, 231 dev->vbl_queue = kmalloc(sizeof(wait_queue_head_t) * num_crtcs,
@@ -161,6 +258,19 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
161 if (!dev->vblank_inmodeset) 258 if (!dev->vblank_inmodeset)
162 goto err; 259 goto err;
163 260
261 dev->_vblank_time = kcalloc(num_crtcs * DRM_VBLANKTIME_RBSIZE,
262 sizeof(struct timeval), GFP_KERNEL);
263 if (!dev->_vblank_time)
264 goto err;
265
266 DRM_INFO("Supports vblank timestamp caching Rev 1 (10.10.2010).\n");
267
268 /* Driver specific high-precision vblank timestamping supported? */
269 if (dev->driver->get_vblank_timestamp)
270 DRM_INFO("Driver supports precise vblank timestamp query.\n");
271 else
272 DRM_INFO("No driver support for vblank timestamp query.\n");
273
164 /* Zero per-crtc vblank stuff */ 274 /* Zero per-crtc vblank stuff */
165 for (i = 0; i < num_crtcs; i++) { 275 for (i = 0; i < num_crtcs; i++) {
166 init_waitqueue_head(&dev->vbl_queue[i]); 276 init_waitqueue_head(&dev->vbl_queue[i]);
@@ -279,7 +389,7 @@ EXPORT_SYMBOL(drm_irq_install);
279 * 389 *
280 * Calls the driver's \c drm_driver_irq_uninstall() function, and stops the irq. 390 * Calls the driver's \c drm_driver_irq_uninstall() function, and stops the irq.
281 */ 391 */
282int drm_irq_uninstall(struct drm_device * dev) 392int drm_irq_uninstall(struct drm_device *dev)
283{ 393{
284 unsigned long irqflags; 394 unsigned long irqflags;
285 int irq_enabled, i; 395 int irq_enabled, i;
@@ -335,7 +445,9 @@ int drm_control(struct drm_device *dev, void *data,
335{ 445{
336 struct drm_control *ctl = data; 446 struct drm_control *ctl = data;
337 447
338 /* if we haven't irq we fallback for compatibility reasons - this used to be a separate function in drm_dma.h */ 448 /* if we haven't irq we fallback for compatibility reasons -
449 * this used to be a separate function in drm_dma.h
450 */
339 451
340 452
341 switch (ctl->func) { 453 switch (ctl->func) {
@@ -360,6 +472,287 @@ int drm_control(struct drm_device *dev, void *data,
360} 472}
361 473
362/** 474/**
475 * drm_calc_timestamping_constants - Calculate and
476 * store various constants which are later needed by
477 * vblank and swap-completion timestamping, e.g, by
478 * drm_calc_vbltimestamp_from_scanoutpos().
479 * They are derived from crtc's true scanout timing,
480 * so they take things like panel scaling or other
481 * adjustments into account.
482 *
483 * @crtc drm_crtc whose timestamp constants should be updated.
484 *
485 */
486void drm_calc_timestamping_constants(struct drm_crtc *crtc)
487{
488 s64 linedur_ns = 0, pixeldur_ns = 0, framedur_ns = 0;
489 u64 dotclock;
490
491 /* Dot clock in Hz: */
492 dotclock = (u64) crtc->hwmode.clock * 1000;
493
494 /* Valid dotclock? */
495 if (dotclock > 0) {
496 /* Convert scanline length in pixels and video dot clock to
497 * line duration, frame duration and pixel duration in
498 * nanoseconds:
499 */
500 pixeldur_ns = (s64) div64_u64(1000000000, dotclock);
501 linedur_ns = (s64) div64_u64(((u64) crtc->hwmode.crtc_htotal *
502 1000000000), dotclock);
503 framedur_ns = (s64) crtc->hwmode.crtc_vtotal * linedur_ns;
504 } else
505 DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n",
506 crtc->base.id);
507
508 crtc->pixeldur_ns = pixeldur_ns;
509 crtc->linedur_ns = linedur_ns;
510 crtc->framedur_ns = framedur_ns;
511
512 DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
513 crtc->base.id, crtc->hwmode.crtc_htotal,
514 crtc->hwmode.crtc_vtotal, crtc->hwmode.crtc_vdisplay);
515 DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n",
516 crtc->base.id, (int) dotclock/1000, (int) framedur_ns,
517 (int) linedur_ns, (int) pixeldur_ns);
518}
519EXPORT_SYMBOL(drm_calc_timestamping_constants);
520
521/**
522 * drm_calc_vbltimestamp_from_scanoutpos - helper routine for kms
523 * drivers. Implements calculation of exact vblank timestamps from
524 * given drm_display_mode timings and current video scanout position
525 * of a crtc. This can be called from within get_vblank_timestamp()
526 * implementation of a kms driver to implement the actual timestamping.
527 *
528 * Should return timestamps conforming to the OML_sync_control OpenML
529 * extension specification. The timestamp corresponds to the end of
530 * the vblank interval, aka start of scanout of topmost-leftmost display
531 * pixel in the following video frame.
532 *
533 * Requires support for optional dev->driver->get_scanout_position()
534 * in kms driver, plus a bit of setup code to provide a drm_display_mode
535 * that corresponds to the true scanout timing.
536 *
537 * The current implementation only handles standard video modes. It
538 * returns as no operation if a doublescan or interlaced video mode is
539 * active. Higher level code is expected to handle this.
540 *
541 * @dev: DRM device.
542 * @crtc: Which crtc's vblank timestamp to retrieve.
543 * @max_error: Desired maximum allowable error in timestamps (nanosecs).
544 * On return contains true maximum error of timestamp.
545 * @vblank_time: Pointer to struct timeval which should receive the timestamp.
546 * @flags: Flags to pass to driver:
547 * 0 = Default.
548 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
549 * @refcrtc: drm_crtc* of crtc which defines scanout timing.
550 *
551 * Returns negative value on error, failure or if not supported in current
552 * video mode:
553 *
554 * -EINVAL - Invalid crtc.
555 * -EAGAIN - Temporary unavailable, e.g., called before initial modeset.
556 * -ENOTSUPP - Function not supported in current display mode.
557 * -EIO - Failed, e.g., due to failed scanout position query.
558 *
559 * Returns or'ed positive status flags on success:
560 *
561 * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
562 * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
563 *
564 */
565int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
566 int *max_error,
567 struct timeval *vblank_time,
568 unsigned flags,
569 struct drm_crtc *refcrtc)
570{
571 struct timeval stime, raw_time;
572 struct drm_display_mode *mode;
573 int vbl_status, vtotal, vdisplay;
574 int vpos, hpos, i;
575 s64 framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns;
576 bool invbl;
577
578 if (crtc < 0 || crtc >= dev->num_crtcs) {
579 DRM_ERROR("Invalid crtc %d\n", crtc);
580 return -EINVAL;
581 }
582
583 /* Scanout position query not supported? Should not happen. */
584 if (!dev->driver->get_scanout_position) {
585 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
586 return -EIO;
587 }
588
589 mode = &refcrtc->hwmode;
590 vtotal = mode->crtc_vtotal;
591 vdisplay = mode->crtc_vdisplay;
592
593 /* Durations of frames, lines, pixels in nanoseconds. */
594 framedur_ns = refcrtc->framedur_ns;
595 linedur_ns = refcrtc->linedur_ns;
596 pixeldur_ns = refcrtc->pixeldur_ns;
597
598 /* If mode timing undefined, just return as no-op:
599 * Happens during initial modesetting of a crtc.
600 */
601 if (vtotal <= 0 || vdisplay <= 0 || framedur_ns == 0) {
602 DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc);
603 return -EAGAIN;
604 }
605
606 /* Don't know yet how to handle interlaced or
607 * double scan modes. Just no-op for now.
608 */
609 if (mode->flags & (DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLSCAN)) {
610 DRM_DEBUG("crtc %d: Noop due to unsupported mode.\n", crtc);
611 return -ENOTSUPP;
612 }
613
614 /* Get current scanout position with system timestamp.
615 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
616 * if single query takes longer than max_error nanoseconds.
617 *
618 * This guarantees a tight bound on maximum error if
619 * code gets preempted or delayed for some reason.
620 */
621 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
622 /* Disable preemption to make it very likely to
623 * succeed in the first iteration even on PREEMPT_RT kernel.
624 */
625 preempt_disable();
626
627 /* Get system timestamp before query. */
628 do_gettimeofday(&stime);
629
630 /* Get vertical and horizontal scanout pos. vpos, hpos. */
631 vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos);
632
633 /* Get system timestamp after query. */
634 do_gettimeofday(&raw_time);
635
636 preempt_enable();
637
638 /* Return as no-op if scanout query unsupported or failed. */
639 if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
640 DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n",
641 crtc, vbl_status);
642 return -EIO;
643 }
644
645 duration_ns = timeval_to_ns(&raw_time) - timeval_to_ns(&stime);
646
647 /* Accept result with < max_error nsecs timing uncertainty. */
648 if (duration_ns <= (s64) *max_error)
649 break;
650 }
651
652 /* Noisy system timing? */
653 if (i == DRM_TIMESTAMP_MAXRETRIES) {
654 DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n",
655 crtc, (int) duration_ns/1000, *max_error/1000, i);
656 }
657
658 /* Return upper bound of timestamp precision error. */
659 *max_error = (int) duration_ns;
660
661 /* Check if in vblank area:
662 * vpos is >=0 in video scanout area, but negative
663 * within vblank area, counting down the number of lines until
664 * start of scanout.
665 */
666 invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
667
668 /* Convert scanout position into elapsed time at raw_time query
669 * since start of scanout at first display scanline. delta_ns
670 * can be negative if start of scanout hasn't happened yet.
671 */
672 delta_ns = (s64) vpos * linedur_ns + (s64) hpos * pixeldur_ns;
673
674 /* Is vpos outside nominal vblank area, but less than
675 * 1/100 of a frame height away from start of vblank?
676 * If so, assume this isn't a massively delayed vblank
677 * interrupt, but a vblank interrupt that fired a few
678 * microseconds before true start of vblank. Compensate
679 * by adding a full frame duration to the final timestamp.
680 * Happens, e.g., on ATI R500, R600.
681 *
682 * We only do this if DRM_CALLED_FROM_VBLIRQ.
683 */
684 if ((flags & DRM_CALLED_FROM_VBLIRQ) && !invbl &&
685 ((vdisplay - vpos) < vtotal / 100)) {
686 delta_ns = delta_ns - framedur_ns;
687
688 /* Signal this correction as "applied". */
689 vbl_status |= 0x8;
690 }
691
692 /* Subtract time delta from raw timestamp to get final
693 * vblank_time timestamp for end of vblank.
694 */
695 *vblank_time = ns_to_timeval(timeval_to_ns(&raw_time) - delta_ns);
696
697 DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %d.%d -> %d.%d [e %d us, %d rep]\n",
698 crtc, (int) vbl_status, hpos, vpos, raw_time.tv_sec,
699 raw_time.tv_usec, vblank_time->tv_sec, vblank_time->tv_usec,
700 (int) duration_ns/1000, i);
701
702 vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
703 if (invbl)
704 vbl_status |= DRM_VBLANKTIME_INVBL;
705
706 return vbl_status;
707}
708EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
709
710/**
711 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
712 * vblank interval.
713 *
714 * @dev: DRM device
715 * @crtc: which crtc's vblank timestamp to retrieve
716 * @tvblank: Pointer to target struct timeval which should receive the timestamp
717 * @flags: Flags to pass to driver:
718 * 0 = Default.
719 * DRM_CALLED_FROM_VBLIRQ = If function is called from vbl irq handler.
720 *
721 * Fetches the system timestamp corresponding to the time of the most recent
722 * vblank interval on specified crtc. May call into kms-driver to
723 * compute the timestamp with a high-precision GPU specific method.
724 *
725 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
726 * call, i.e., it isn't very precisely locked to the true vblank.
727 *
728 * Returns non-zero if timestamp is considered to be very precise.
729 */
730u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
731 struct timeval *tvblank, unsigned flags)
732{
733 int ret = 0;
734
735 /* Define requested maximum error on timestamps (nanoseconds). */
736 int max_error = (int) drm_timestamp_precision * 1000;
737
738 /* Query driver if possible and precision timestamping enabled. */
739 if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
740 ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
741 tvblank, flags);
742 if (ret > 0)
743 return (u32) ret;
744 }
745
746 /* GPU high precision timestamp query unsupported or failed.
747 * Return gettimeofday timestamp as best estimate.
748 */
749 do_gettimeofday(tvblank);
750
751 return 0;
752}
753EXPORT_SYMBOL(drm_get_last_vbltimestamp);
754
755/**
363 * drm_vblank_count - retrieve "cooked" vblank counter value 756 * drm_vblank_count - retrieve "cooked" vblank counter value
364 * @dev: DRM device 757 * @dev: DRM device
365 * @crtc: which counter to retrieve 758 * @crtc: which counter to retrieve
@@ -375,6 +768,40 @@ u32 drm_vblank_count(struct drm_device *dev, int crtc)
375EXPORT_SYMBOL(drm_vblank_count); 768EXPORT_SYMBOL(drm_vblank_count);
376 769
377/** 770/**
771 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value
772 * and the system timestamp corresponding to that vblank counter value.
773 *
774 * @dev: DRM device
775 * @crtc: which counter to retrieve
776 * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
777 *
778 * Fetches the "cooked" vblank count value that represents the number of
779 * vblank events since the system was booted, including lost events due to
780 * modesetting activity. Returns corresponding system timestamp of the time
781 * of the vblank interval that corresponds to the current value vblank counter
782 * value.
783 */
784u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
785 struct timeval *vblanktime)
786{
787 u32 cur_vblank;
788
789 /* Read timestamp from slot of _vblank_time ringbuffer
790 * that corresponds to current vblank count. Retry if
791 * count has incremented during readout. This works like
792 * a seqlock.
793 */
794 do {
795 cur_vblank = atomic_read(&dev->_vblank_count[crtc]);
796 *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
797 smp_rmb();
798 } while (cur_vblank != atomic_read(&dev->_vblank_count[crtc]));
799
800 return cur_vblank;
801}
802EXPORT_SYMBOL(drm_vblank_count_and_time);
803
804/**
378 * drm_update_vblank_count - update the master vblank counter 805 * drm_update_vblank_count - update the master vblank counter
379 * @dev: DRM device 806 * @dev: DRM device
380 * @crtc: counter to update 807 * @crtc: counter to update
@@ -392,7 +819,8 @@ EXPORT_SYMBOL(drm_vblank_count);
392 */ 819 */
393static void drm_update_vblank_count(struct drm_device *dev, int crtc) 820static void drm_update_vblank_count(struct drm_device *dev, int crtc)
394{ 821{
395 u32 cur_vblank, diff; 822 u32 cur_vblank, diff, tslot, rc;
823 struct timeval t_vblank;
396 824
397 /* 825 /*
398 * Interrupts were disabled prior to this call, so deal with counter 826 * Interrupts were disabled prior to this call, so deal with counter
@@ -400,8 +828,18 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
400 * NOTE! It's possible we lost a full dev->max_vblank_count events 828 * NOTE! It's possible we lost a full dev->max_vblank_count events
401 * here if the register is small or we had vblank interrupts off for 829 * here if the register is small or we had vblank interrupts off for
402 * a long time. 830 * a long time.
831 *
832 * We repeat the hardware vblank counter & timestamp query until
833 * we get consistent results. This to prevent races between gpu
834 * updating its hardware counter while we are retrieving the
835 * corresponding vblank timestamp.
403 */ 836 */
404 cur_vblank = dev->driver->get_vblank_counter(dev, crtc); 837 do {
838 cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
839 rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
840 } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
841
842 /* Deal with counter wrap */
405 diff = cur_vblank - dev->last_vblank[crtc]; 843 diff = cur_vblank - dev->last_vblank[crtc];
406 if (cur_vblank < dev->last_vblank[crtc]) { 844 if (cur_vblank < dev->last_vblank[crtc]) {
407 diff += dev->max_vblank_count; 845 diff += dev->max_vblank_count;
@@ -413,6 +851,16 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
413 DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n", 851 DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
414 crtc, diff); 852 crtc, diff);
415 853
854 /* Reinitialize corresponding vblank timestamp if high-precision query
855 * available. Skip this step if query unsupported or failed. Will
856 * reinitialize delayed at next vblank interrupt in that case.
857 */
858 if (rc) {
859 tslot = atomic_read(&dev->_vblank_count[crtc]) + diff;
860 vblanktimestamp(dev, crtc, tslot) = t_vblank;
861 smp_wmb();
862 }
863
416 atomic_add(diff, &dev->_vblank_count[crtc]); 864 atomic_add(diff, &dev->_vblank_count[crtc]);
417} 865}
418 866
@@ -429,15 +877,27 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
429 */ 877 */
430int drm_vblank_get(struct drm_device *dev, int crtc) 878int drm_vblank_get(struct drm_device *dev, int crtc)
431{ 879{
432 unsigned long irqflags; 880 unsigned long irqflags, irqflags2;
433 int ret = 0; 881 int ret = 0;
434 882
435 spin_lock_irqsave(&dev->vbl_lock, irqflags); 883 spin_lock_irqsave(&dev->vbl_lock, irqflags);
436 /* Going from 0->1 means we have to enable interrupts again */ 884 /* Going from 0->1 means we have to enable interrupts again */
437 if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) { 885 if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) {
886 /* Disable preemption while holding vblank_time_lock. Do
887 * it explicitely to guard against PREEMPT_RT kernel.
888 */
889 preempt_disable();
890 spin_lock_irqsave(&dev->vblank_time_lock, irqflags2);
438 if (!dev->vblank_enabled[crtc]) { 891 if (!dev->vblank_enabled[crtc]) {
892 /* Enable vblank irqs under vblank_time_lock protection.
893 * All vblank count & timestamp updates are held off
894 * until we are done reinitializing master counter and
895 * timestamps. Filtercode in drm_handle_vblank() will
896 * prevent double-accounting of same vblank interval.
897 */
439 ret = dev->driver->enable_vblank(dev, crtc); 898 ret = dev->driver->enable_vblank(dev, crtc);
440 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret); 899 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n",
900 crtc, ret);
441 if (ret) 901 if (ret)
442 atomic_dec(&dev->vblank_refcount[crtc]); 902 atomic_dec(&dev->vblank_refcount[crtc]);
443 else { 903 else {
@@ -445,6 +905,8 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
445 drm_update_vblank_count(dev, crtc); 905 drm_update_vblank_count(dev, crtc);
446 } 906 }
447 } 907 }
908 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags2);
909 preempt_enable();
448 } else { 910 } else {
449 if (!dev->vblank_enabled[crtc]) { 911 if (!dev->vblank_enabled[crtc]) {
450 atomic_dec(&dev->vblank_refcount[crtc]); 912 atomic_dec(&dev->vblank_refcount[crtc]);
@@ -463,15 +925,17 @@ EXPORT_SYMBOL(drm_vblank_get);
463 * @crtc: which counter to give up 925 * @crtc: which counter to give up
464 * 926 *
465 * Release ownership of a given vblank counter, turning off interrupts 927 * Release ownership of a given vblank counter, turning off interrupts
466 * if possible. 928 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
467 */ 929 */
468void drm_vblank_put(struct drm_device *dev, int crtc) 930void drm_vblank_put(struct drm_device *dev, int crtc)
469{ 931{
470 BUG_ON (atomic_read (&dev->vblank_refcount[crtc]) == 0); 932 BUG_ON(atomic_read(&dev->vblank_refcount[crtc]) == 0);
471 933
472 /* Last user schedules interrupt disable */ 934 /* Last user schedules interrupt disable */
473 if (atomic_dec_and_test(&dev->vblank_refcount[crtc])) 935 if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) &&
474 mod_timer(&dev->vblank_disable_timer, jiffies + 5*DRM_HZ); 936 (drm_vblank_offdelay > 0))
937 mod_timer(&dev->vblank_disable_timer,
938 jiffies + ((drm_vblank_offdelay * DRM_HZ)/1000));
475} 939}
476EXPORT_SYMBOL(drm_vblank_put); 940EXPORT_SYMBOL(drm_vblank_put);
477 941
@@ -480,10 +944,8 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
480 unsigned long irqflags; 944 unsigned long irqflags;
481 945
482 spin_lock_irqsave(&dev->vbl_lock, irqflags); 946 spin_lock_irqsave(&dev->vbl_lock, irqflags);
483 dev->driver->disable_vblank(dev, crtc); 947 vblank_disable_and_save(dev, crtc);
484 DRM_WAKEUP(&dev->vbl_queue[crtc]); 948 DRM_WAKEUP(&dev->vbl_queue[crtc]);
485 dev->vblank_enabled[crtc] = 0;
486 dev->last_vblank[crtc] = dev->driver->get_vblank_counter(dev, crtc);
487 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 949 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
488} 950}
489EXPORT_SYMBOL(drm_vblank_off); 951EXPORT_SYMBOL(drm_vblank_off);
@@ -602,7 +1064,6 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
602 e->base.file_priv = file_priv; 1064 e->base.file_priv = file_priv;
603 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree; 1065 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
604 1066
605 do_gettimeofday(&now);
606 spin_lock_irqsave(&dev->event_lock, flags); 1067 spin_lock_irqsave(&dev->event_lock, flags);
607 1068
608 if (file_priv->event_space < sizeof e->event) { 1069 if (file_priv->event_space < sizeof e->event) {
@@ -611,7 +1072,8 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
611 } 1072 }
612 1073
613 file_priv->event_space -= sizeof e->event; 1074 file_priv->event_space -= sizeof e->event;
614 seq = drm_vblank_count(dev, pipe); 1075 seq = drm_vblank_count_and_time(dev, pipe, &now);
1076
615 if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) && 1077 if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
616 (seq - vblwait->request.sequence) <= (1 << 23)) { 1078 (seq - vblwait->request.sequence) <= (1 << 23)) {
617 vblwait->request.sequence = seq + 1; 1079 vblwait->request.sequence = seq + 1;
@@ -626,15 +1088,18 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
626 1088
627 e->event.sequence = vblwait->request.sequence; 1089 e->event.sequence = vblwait->request.sequence;
628 if ((seq - vblwait->request.sequence) <= (1 << 23)) { 1090 if ((seq - vblwait->request.sequence) <= (1 << 23)) {
1091 e->event.sequence = seq;
629 e->event.tv_sec = now.tv_sec; 1092 e->event.tv_sec = now.tv_sec;
630 e->event.tv_usec = now.tv_usec; 1093 e->event.tv_usec = now.tv_usec;
631 drm_vblank_put(dev, pipe); 1094 drm_vblank_put(dev, pipe);
632 list_add_tail(&e->base.link, &e->base.file_priv->event_list); 1095 list_add_tail(&e->base.link, &e->base.file_priv->event_list);
633 wake_up_interruptible(&e->base.file_priv->event_wait); 1096 wake_up_interruptible(&e->base.file_priv->event_wait);
1097 vblwait->reply.sequence = seq;
634 trace_drm_vblank_event_delivered(current->pid, pipe, 1098 trace_drm_vblank_event_delivered(current->pid, pipe,
635 vblwait->request.sequence); 1099 vblwait->request.sequence);
636 } else { 1100 } else {
637 list_add_tail(&e->base.link, &dev->vblank_event_list); 1101 list_add_tail(&e->base.link, &dev->vblank_event_list);
1102 vblwait->reply.sequence = vblwait->request.sequence;
638 } 1103 }
639 1104
640 spin_unlock_irqrestore(&dev->event_lock, flags); 1105 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -727,11 +1192,10 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
727 if (ret != -EINTR) { 1192 if (ret != -EINTR) {
728 struct timeval now; 1193 struct timeval now;
729 1194
730 do_gettimeofday(&now); 1195 vblwait->reply.sequence = drm_vblank_count_and_time(dev, crtc, &now);
731
732 vblwait->reply.tval_sec = now.tv_sec; 1196 vblwait->reply.tval_sec = now.tv_sec;
733 vblwait->reply.tval_usec = now.tv_usec; 1197 vblwait->reply.tval_usec = now.tv_usec;
734 vblwait->reply.sequence = drm_vblank_count(dev, crtc); 1198
735 DRM_DEBUG("returning %d to client\n", 1199 DRM_DEBUG("returning %d to client\n",
736 vblwait->reply.sequence); 1200 vblwait->reply.sequence);
737 } else { 1201 } else {
@@ -750,8 +1214,7 @@ void drm_handle_vblank_events(struct drm_device *dev, int crtc)
750 unsigned long flags; 1214 unsigned long flags;
751 unsigned int seq; 1215 unsigned int seq;
752 1216
753 do_gettimeofday(&now); 1217 seq = drm_vblank_count_and_time(dev, crtc, &now);
754 seq = drm_vblank_count(dev, crtc);
755 1218
756 spin_lock_irqsave(&dev->event_lock, flags); 1219 spin_lock_irqsave(&dev->event_lock, flags);
757 1220
@@ -789,11 +1252,64 @@ void drm_handle_vblank_events(struct drm_device *dev, int crtc)
789 */ 1252 */
790void drm_handle_vblank(struct drm_device *dev, int crtc) 1253void drm_handle_vblank(struct drm_device *dev, int crtc)
791{ 1254{
1255 u32 vblcount;
1256 s64 diff_ns;
1257 struct timeval tvblank;
1258 unsigned long irqflags;
1259
792 if (!dev->num_crtcs) 1260 if (!dev->num_crtcs)
793 return; 1261 return;
794 1262
795 atomic_inc(&dev->_vblank_count[crtc]); 1263 /* Need timestamp lock to prevent concurrent execution with
1264 * vblank enable/disable, as this would cause inconsistent
1265 * or corrupted timestamps and vblank counts.
1266 */
1267 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
1268
1269 /* Vblank irq handling disabled. Nothing to do. */
1270 if (!dev->vblank_enabled[crtc]) {
1271 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
1272 return;
1273 }
1274
1275 /* Fetch corresponding timestamp for this vblank interval from
1276 * driver and store it in proper slot of timestamp ringbuffer.
1277 */
1278
1279 /* Get current timestamp and count. */
1280 vblcount = atomic_read(&dev->_vblank_count[crtc]);
1281 drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
1282
1283 /* Compute time difference to timestamp of last vblank */
1284 diff_ns = timeval_to_ns(&tvblank) -
1285 timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
1286
1287 /* Update vblank timestamp and count if at least
1288 * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds
1289 * difference between last stored timestamp and current
1290 * timestamp. A smaller difference means basically
1291 * identical timestamps. Happens if this vblank has
1292 * been already processed and this is a redundant call,
1293 * e.g., due to spurious vblank interrupts. We need to
1294 * ignore those for accounting.
1295 */
1296 if (abs(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) {
1297 /* Store new timestamp in ringbuffer. */
1298 vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
1299 smp_wmb();
1300
1301 /* Increment cooked vblank count. This also atomically commits
1302 * the timestamp computed above.
1303 */
1304 atomic_inc(&dev->_vblank_count[crtc]);
1305 } else {
1306 DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
1307 crtc, (int) diff_ns);
1308 }
1309
796 DRM_WAKEUP(&dev->vbl_queue[crtc]); 1310 DRM_WAKEUP(&dev->vbl_queue[crtc]);
797 drm_handle_vblank_events(dev, crtc); 1311 drm_handle_vblank_events(dev, crtc);
1312
1313 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
798} 1314}
799EXPORT_SYMBOL(drm_handle_vblank); 1315EXPORT_SYMBOL(drm_handle_vblank);
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index a6bfc302ed90..c59515ba7e69 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -392,10 +392,36 @@ void drm_mm_init_scan(struct drm_mm *mm, unsigned long size,
392 mm->scanned_blocks = 0; 392 mm->scanned_blocks = 0;
393 mm->scan_hit_start = 0; 393 mm->scan_hit_start = 0;
394 mm->scan_hit_size = 0; 394 mm->scan_hit_size = 0;
395 mm->scan_check_range = 0;
395} 396}
396EXPORT_SYMBOL(drm_mm_init_scan); 397EXPORT_SYMBOL(drm_mm_init_scan);
397 398
398/** 399/**
400 * Initializa lru scanning.
401 *
402 * This simply sets up the scanning routines with the parameters for the desired
403 * hole. This version is for range-restricted scans.
404 *
405 * Warning: As long as the scan list is non-empty, no other operations than
406 * adding/removing nodes to/from the scan list are allowed.
407 */
408void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size,
409 unsigned alignment,
410 unsigned long start,
411 unsigned long end)
412{
413 mm->scan_alignment = alignment;
414 mm->scan_size = size;
415 mm->scanned_blocks = 0;
416 mm->scan_hit_start = 0;
417 mm->scan_hit_size = 0;
418 mm->scan_start = start;
419 mm->scan_end = end;
420 mm->scan_check_range = 1;
421}
422EXPORT_SYMBOL(drm_mm_init_scan_with_range);
423
424/**
399 * Add a node to the scan list that might be freed to make space for the desired 425 * Add a node to the scan list that might be freed to make space for the desired
400 * hole. 426 * hole.
401 * 427 *
@@ -406,6 +432,8 @@ int drm_mm_scan_add_block(struct drm_mm_node *node)
406 struct drm_mm *mm = node->mm; 432 struct drm_mm *mm = node->mm;
407 struct list_head *prev_free, *next_free; 433 struct list_head *prev_free, *next_free;
408 struct drm_mm_node *prev_node, *next_node; 434 struct drm_mm_node *prev_node, *next_node;
435 unsigned long adj_start;
436 unsigned long adj_end;
409 437
410 mm->scanned_blocks++; 438 mm->scanned_blocks++;
411 439
@@ -452,7 +480,17 @@ int drm_mm_scan_add_block(struct drm_mm_node *node)
452 node->free_stack.prev = prev_free; 480 node->free_stack.prev = prev_free;
453 node->free_stack.next = next_free; 481 node->free_stack.next = next_free;
454 482
455 if (check_free_hole(node->start, node->start + node->size, 483 if (mm->scan_check_range) {
484 adj_start = node->start < mm->scan_start ?
485 mm->scan_start : node->start;
486 adj_end = node->start + node->size > mm->scan_end ?
487 mm->scan_end : node->start + node->size;
488 } else {
489 adj_start = node->start;
490 adj_end = node->start + node->size;
491 }
492
493 if (check_free_hole(adj_start , adj_end,
456 mm->scan_size, mm->scan_alignment)) { 494 mm->scan_size, mm->scan_alignment)) {
457 mm->scan_hit_start = node->start; 495 mm->scan_hit_start = node->start;
458 mm->scan_hit_size = node->size; 496 mm->scan_hit_size = node->size;
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index cdc89ee042cc..d59edc18301f 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -40,12 +40,22 @@
40unsigned int drm_debug = 0; /* 1 to enable debug output */ 40unsigned int drm_debug = 0; /* 1 to enable debug output */
41EXPORT_SYMBOL(drm_debug); 41EXPORT_SYMBOL(drm_debug);
42 42
43unsigned int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
44EXPORT_SYMBOL(drm_vblank_offdelay);
45
46unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
47EXPORT_SYMBOL(drm_timestamp_precision);
48
43MODULE_AUTHOR(CORE_AUTHOR); 49MODULE_AUTHOR(CORE_AUTHOR);
44MODULE_DESCRIPTION(CORE_DESC); 50MODULE_DESCRIPTION(CORE_DESC);
45MODULE_LICENSE("GPL and additional rights"); 51MODULE_LICENSE("GPL and additional rights");
46MODULE_PARM_DESC(debug, "Enable debug output"); 52MODULE_PARM_DESC(debug, "Enable debug output");
53MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs]");
54MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
47 55
48module_param_named(debug, drm_debug, int, 0600); 56module_param_named(debug, drm_debug, int, 0600);
57module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
58module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
49 59
50struct idr drm_minors_idr; 60struct idr drm_minors_idr;
51 61
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index fdc833d5cc7b..0ae6a7c5020f 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -9,6 +9,8 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
9 i915_gem.o \ 9 i915_gem.o \
10 i915_gem_debug.o \ 10 i915_gem_debug.o \
11 i915_gem_evict.o \ 11 i915_gem_evict.o \
12 i915_gem_execbuffer.o \
13 i915_gem_gtt.o \
12 i915_gem_tiling.o \ 14 i915_gem_tiling.o \
13 i915_trace_points.o \ 15 i915_trace_points.o \
14 intel_display.o \ 16 intel_display.o \
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 1f4f3ceb63c7..92f75782c332 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -32,6 +32,7 @@
32#include "drmP.h" 32#include "drmP.h"
33#include "drm.h" 33#include "drm.h"
34#include "intel_drv.h" 34#include "intel_drv.h"
35#include "intel_ringbuffer.h"
35#include "i915_drm.h" 36#include "i915_drm.h"
36#include "i915_drv.h" 37#include "i915_drv.h"
37 38
@@ -72,7 +73,6 @@ static int i915_capabilities(struct seq_file *m, void *data)
72 B(is_broadwater); 73 B(is_broadwater);
73 B(is_crestline); 74 B(is_crestline);
74 B(has_fbc); 75 B(has_fbc);
75 B(has_rc6);
76 B(has_pipe_cxsr); 76 B(has_pipe_cxsr);
77 B(has_hotplug); 77 B(has_hotplug);
78 B(cursor_needs_physical); 78 B(cursor_needs_physical);
@@ -86,19 +86,19 @@ static int i915_capabilities(struct seq_file *m, void *data)
86 return 0; 86 return 0;
87} 87}
88 88
89static const char *get_pin_flag(struct drm_i915_gem_object *obj_priv) 89static const char *get_pin_flag(struct drm_i915_gem_object *obj)
90{ 90{
91 if (obj_priv->user_pin_count > 0) 91 if (obj->user_pin_count > 0)
92 return "P"; 92 return "P";
93 else if (obj_priv->pin_count > 0) 93 else if (obj->pin_count > 0)
94 return "p"; 94 return "p";
95 else 95 else
96 return " "; 96 return " ";
97} 97}
98 98
99static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv) 99static const char *get_tiling_flag(struct drm_i915_gem_object *obj)
100{ 100{
101 switch (obj_priv->tiling_mode) { 101 switch (obj->tiling_mode) {
102 default: 102 default:
103 case I915_TILING_NONE: return " "; 103 case I915_TILING_NONE: return " ";
104 case I915_TILING_X: return "X"; 104 case I915_TILING_X: return "X";
@@ -109,7 +109,7 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv)
109static void 109static void
110describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) 110describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
111{ 111{
112 seq_printf(m, "%p: %s%s %8zd %08x %08x %d%s%s", 112 seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s",
113 &obj->base, 113 &obj->base,
114 get_pin_flag(obj), 114 get_pin_flag(obj),
115 get_tiling_flag(obj), 115 get_tiling_flag(obj),
@@ -117,6 +117,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
117 obj->base.read_domains, 117 obj->base.read_domains,
118 obj->base.write_domain, 118 obj->base.write_domain,
119 obj->last_rendering_seqno, 119 obj->last_rendering_seqno,
120 obj->last_fenced_seqno,
120 obj->dirty ? " dirty" : "", 121 obj->dirty ? " dirty" : "",
121 obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); 122 obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
122 if (obj->base.name) 123 if (obj->base.name)
@@ -124,7 +125,17 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
124 if (obj->fence_reg != I915_FENCE_REG_NONE) 125 if (obj->fence_reg != I915_FENCE_REG_NONE)
125 seq_printf(m, " (fence: %d)", obj->fence_reg); 126 seq_printf(m, " (fence: %d)", obj->fence_reg);
126 if (obj->gtt_space != NULL) 127 if (obj->gtt_space != NULL)
127 seq_printf(m, " (gtt_offset: %08x)", obj->gtt_offset); 128 seq_printf(m, " (gtt offset: %08x, size: %08x)",
129 obj->gtt_offset, (unsigned int)obj->gtt_space->size);
130 if (obj->pin_mappable || obj->fault_mappable) {
131 char s[3], *t = s;
132 if (obj->pin_mappable)
133 *t++ = 'p';
134 if (obj->fault_mappable)
135 *t++ = 'f';
136 *t = '\0';
137 seq_printf(m, " (%s mappable)", s);
138 }
128 if (obj->ring != NULL) 139 if (obj->ring != NULL)
129 seq_printf(m, " (%s)", obj->ring->name); 140 seq_printf(m, " (%s)", obj->ring->name);
130} 141}
@@ -136,7 +147,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
136 struct list_head *head; 147 struct list_head *head;
137 struct drm_device *dev = node->minor->dev; 148 struct drm_device *dev = node->minor->dev;
138 drm_i915_private_t *dev_priv = dev->dev_private; 149 drm_i915_private_t *dev_priv = dev->dev_private;
139 struct drm_i915_gem_object *obj_priv; 150 struct drm_i915_gem_object *obj;
140 size_t total_obj_size, total_gtt_size; 151 size_t total_obj_size, total_gtt_size;
141 int count, ret; 152 int count, ret;
142 153
@@ -171,12 +182,12 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
171 } 182 }
172 183
173 total_obj_size = total_gtt_size = count = 0; 184 total_obj_size = total_gtt_size = count = 0;
174 list_for_each_entry(obj_priv, head, mm_list) { 185 list_for_each_entry(obj, head, mm_list) {
175 seq_printf(m, " "); 186 seq_printf(m, " ");
176 describe_obj(m, obj_priv); 187 describe_obj(m, obj);
177 seq_printf(m, "\n"); 188 seq_printf(m, "\n");
178 total_obj_size += obj_priv->base.size; 189 total_obj_size += obj->base.size;
179 total_gtt_size += obj_priv->gtt_space->size; 190 total_gtt_size += obj->gtt_space->size;
180 count++; 191 count++;
181 } 192 }
182 mutex_unlock(&dev->struct_mutex); 193 mutex_unlock(&dev->struct_mutex);
@@ -186,24 +197,79 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
186 return 0; 197 return 0;
187} 198}
188 199
200#define count_objects(list, member) do { \
201 list_for_each_entry(obj, list, member) { \
202 size += obj->gtt_space->size; \
203 ++count; \
204 if (obj->map_and_fenceable) { \
205 mappable_size += obj->gtt_space->size; \
206 ++mappable_count; \
207 } \
208 } \
209} while(0)
210
189static int i915_gem_object_info(struct seq_file *m, void* data) 211static int i915_gem_object_info(struct seq_file *m, void* data)
190{ 212{
191 struct drm_info_node *node = (struct drm_info_node *) m->private; 213 struct drm_info_node *node = (struct drm_info_node *) m->private;
192 struct drm_device *dev = node->minor->dev; 214 struct drm_device *dev = node->minor->dev;
193 struct drm_i915_private *dev_priv = dev->dev_private; 215 struct drm_i915_private *dev_priv = dev->dev_private;
216 u32 count, mappable_count;
217 size_t size, mappable_size;
218 struct drm_i915_gem_object *obj;
194 int ret; 219 int ret;
195 220
196 ret = mutex_lock_interruptible(&dev->struct_mutex); 221 ret = mutex_lock_interruptible(&dev->struct_mutex);
197 if (ret) 222 if (ret)
198 return ret; 223 return ret;
199 224
200 seq_printf(m, "%u objects\n", dev_priv->mm.object_count); 225 seq_printf(m, "%u objects, %zu bytes\n",
201 seq_printf(m, "%zu object bytes\n", dev_priv->mm.object_memory); 226 dev_priv->mm.object_count,
202 seq_printf(m, "%u pinned\n", dev_priv->mm.pin_count); 227 dev_priv->mm.object_memory);
203 seq_printf(m, "%zu pin bytes\n", dev_priv->mm.pin_memory); 228
204 seq_printf(m, "%u objects in gtt\n", dev_priv->mm.gtt_count); 229 size = count = mappable_size = mappable_count = 0;
205 seq_printf(m, "%zu gtt bytes\n", dev_priv->mm.gtt_memory); 230 count_objects(&dev_priv->mm.gtt_list, gtt_list);
206 seq_printf(m, "%zu gtt total\n", dev_priv->mm.gtt_total); 231 seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
232 count, mappable_count, size, mappable_size);
233
234 size = count = mappable_size = mappable_count = 0;
235 count_objects(&dev_priv->mm.active_list, mm_list);
236 count_objects(&dev_priv->mm.flushing_list, mm_list);
237 seq_printf(m, " %u [%u] active objects, %zu [%zu] bytes\n",
238 count, mappable_count, size, mappable_size);
239
240 size = count = mappable_size = mappable_count = 0;
241 count_objects(&dev_priv->mm.pinned_list, mm_list);
242 seq_printf(m, " %u [%u] pinned objects, %zu [%zu] bytes\n",
243 count, mappable_count, size, mappable_size);
244
245 size = count = mappable_size = mappable_count = 0;
246 count_objects(&dev_priv->mm.inactive_list, mm_list);
247 seq_printf(m, " %u [%u] inactive objects, %zu [%zu] bytes\n",
248 count, mappable_count, size, mappable_size);
249
250 size = count = mappable_size = mappable_count = 0;
251 count_objects(&dev_priv->mm.deferred_free_list, mm_list);
252 seq_printf(m, " %u [%u] freed objects, %zu [%zu] bytes\n",
253 count, mappable_count, size, mappable_size);
254
255 size = count = mappable_size = mappable_count = 0;
256 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
257 if (obj->fault_mappable) {
258 size += obj->gtt_space->size;
259 ++count;
260 }
261 if (obj->pin_mappable) {
262 mappable_size += obj->gtt_space->size;
263 ++mappable_count;
264 }
265 }
266 seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
267 mappable_count, mappable_size);
268 seq_printf(m, "%u fault mappable objects, %zu bytes\n",
269 count, size);
270
271 seq_printf(m, "%zu [%zu] gtt total\n",
272 dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total);
207 273
208 mutex_unlock(&dev->struct_mutex); 274 mutex_unlock(&dev->struct_mutex);
209 275
@@ -243,14 +309,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
243 seq_printf(m, "%d prepares\n", work->pending); 309 seq_printf(m, "%d prepares\n", work->pending);
244 310
245 if (work->old_fb_obj) { 311 if (work->old_fb_obj) {
246 struct drm_i915_gem_object *obj_priv = to_intel_bo(work->old_fb_obj); 312 struct drm_i915_gem_object *obj = work->old_fb_obj;
247 if(obj_priv) 313 if (obj)
248 seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); 314 seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
249 } 315 }
250 if (work->pending_flip_obj) { 316 if (work->pending_flip_obj) {
251 struct drm_i915_gem_object *obj_priv = to_intel_bo(work->pending_flip_obj); 317 struct drm_i915_gem_object *obj = work->pending_flip_obj;
252 if(obj_priv) 318 if (obj)
253 seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); 319 seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset);
254 } 320 }
255 } 321 }
256 spin_unlock_irqrestore(&dev->event_lock, flags); 322 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -265,44 +331,80 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
265 struct drm_device *dev = node->minor->dev; 331 struct drm_device *dev = node->minor->dev;
266 drm_i915_private_t *dev_priv = dev->dev_private; 332 drm_i915_private_t *dev_priv = dev->dev_private;
267 struct drm_i915_gem_request *gem_request; 333 struct drm_i915_gem_request *gem_request;
268 int ret; 334 int ret, count;
269 335
270 ret = mutex_lock_interruptible(&dev->struct_mutex); 336 ret = mutex_lock_interruptible(&dev->struct_mutex);
271 if (ret) 337 if (ret)
272 return ret; 338 return ret;
273 339
274 seq_printf(m, "Request:\n"); 340 count = 0;
275 list_for_each_entry(gem_request, &dev_priv->render_ring.request_list, 341 if (!list_empty(&dev_priv->ring[RCS].request_list)) {
276 list) { 342 seq_printf(m, "Render requests:\n");
277 seq_printf(m, " %d @ %d\n", 343 list_for_each_entry(gem_request,
278 gem_request->seqno, 344 &dev_priv->ring[RCS].request_list,
279 (int) (jiffies - gem_request->emitted_jiffies)); 345 list) {
346 seq_printf(m, " %d @ %d\n",
347 gem_request->seqno,
348 (int) (jiffies - gem_request->emitted_jiffies));
349 }
350 count++;
351 }
352 if (!list_empty(&dev_priv->ring[VCS].request_list)) {
353 seq_printf(m, "BSD requests:\n");
354 list_for_each_entry(gem_request,
355 &dev_priv->ring[VCS].request_list,
356 list) {
357 seq_printf(m, " %d @ %d\n",
358 gem_request->seqno,
359 (int) (jiffies - gem_request->emitted_jiffies));
360 }
361 count++;
362 }
363 if (!list_empty(&dev_priv->ring[BCS].request_list)) {
364 seq_printf(m, "BLT requests:\n");
365 list_for_each_entry(gem_request,
366 &dev_priv->ring[BCS].request_list,
367 list) {
368 seq_printf(m, " %d @ %d\n",
369 gem_request->seqno,
370 (int) (jiffies - gem_request->emitted_jiffies));
371 }
372 count++;
280 } 373 }
281 mutex_unlock(&dev->struct_mutex); 374 mutex_unlock(&dev->struct_mutex);
282 375
376 if (count == 0)
377 seq_printf(m, "No requests\n");
378
283 return 0; 379 return 0;
284} 380}
285 381
382static void i915_ring_seqno_info(struct seq_file *m,
383 struct intel_ring_buffer *ring)
384{
385 if (ring->get_seqno) {
386 seq_printf(m, "Current sequence (%s): %d\n",
387 ring->name, ring->get_seqno(ring));
388 seq_printf(m, "Waiter sequence (%s): %d\n",
389 ring->name, ring->waiting_seqno);
390 seq_printf(m, "IRQ sequence (%s): %d\n",
391 ring->name, ring->irq_seqno);
392 }
393}
394
286static int i915_gem_seqno_info(struct seq_file *m, void *data) 395static int i915_gem_seqno_info(struct seq_file *m, void *data)
287{ 396{
288 struct drm_info_node *node = (struct drm_info_node *) m->private; 397 struct drm_info_node *node = (struct drm_info_node *) m->private;
289 struct drm_device *dev = node->minor->dev; 398 struct drm_device *dev = node->minor->dev;
290 drm_i915_private_t *dev_priv = dev->dev_private; 399 drm_i915_private_t *dev_priv = dev->dev_private;
291 int ret; 400 int ret, i;
292 401
293 ret = mutex_lock_interruptible(&dev->struct_mutex); 402 ret = mutex_lock_interruptible(&dev->struct_mutex);
294 if (ret) 403 if (ret)
295 return ret; 404 return ret;
296 405
297 if (dev_priv->render_ring.status_page.page_addr != NULL) { 406 for (i = 0; i < I915_NUM_RINGS; i++)
298 seq_printf(m, "Current sequence: %d\n", 407 i915_ring_seqno_info(m, &dev_priv->ring[i]);
299 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
300 } else {
301 seq_printf(m, "Current sequence: hws uninitialized\n");
302 }
303 seq_printf(m, "Waiter sequence: %d\n",
304 dev_priv->mm.waiting_gem_seqno);
305 seq_printf(m, "IRQ sequence: %d\n", dev_priv->mm.irq_gem_seqno);
306 408
307 mutex_unlock(&dev->struct_mutex); 409 mutex_unlock(&dev->struct_mutex);
308 410
@@ -315,7 +417,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
315 struct drm_info_node *node = (struct drm_info_node *) m->private; 417 struct drm_info_node *node = (struct drm_info_node *) m->private;
316 struct drm_device *dev = node->minor->dev; 418 struct drm_device *dev = node->minor->dev;
317 drm_i915_private_t *dev_priv = dev->dev_private; 419 drm_i915_private_t *dev_priv = dev->dev_private;
318 int ret; 420 int ret, i;
319 421
320 ret = mutex_lock_interruptible(&dev->struct_mutex); 422 ret = mutex_lock_interruptible(&dev->struct_mutex);
321 if (ret) 423 if (ret)
@@ -354,16 +456,8 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
354 } 456 }
355 seq_printf(m, "Interrupts received: %d\n", 457 seq_printf(m, "Interrupts received: %d\n",
356 atomic_read(&dev_priv->irq_received)); 458 atomic_read(&dev_priv->irq_received));
357 if (dev_priv->render_ring.status_page.page_addr != NULL) { 459 for (i = 0; i < I915_NUM_RINGS; i++)
358 seq_printf(m, "Current sequence: %d\n", 460 i915_ring_seqno_info(m, &dev_priv->ring[i]);
359 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring));
360 } else {
361 seq_printf(m, "Current sequence: hws uninitialized\n");
362 }
363 seq_printf(m, "Waiter sequence: %d\n",
364 dev_priv->mm.waiting_gem_seqno);
365 seq_printf(m, "IRQ sequence: %d\n",
366 dev_priv->mm.irq_gem_seqno);
367 mutex_unlock(&dev->struct_mutex); 461 mutex_unlock(&dev->struct_mutex);
368 462
369 return 0; 463 return 0;
@@ -383,29 +477,17 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
383 seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start); 477 seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start);
384 seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs); 478 seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs);
385 for (i = 0; i < dev_priv->num_fence_regs; i++) { 479 for (i = 0; i < dev_priv->num_fence_regs; i++) {
386 struct drm_gem_object *obj = dev_priv->fence_regs[i].obj; 480 struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
387 481
388 if (obj == NULL) { 482 seq_printf(m, "Fenced object[%2d] = ", i);
389 seq_printf(m, "Fenced object[%2d] = unused\n", i); 483 if (obj == NULL)
390 } else { 484 seq_printf(m, "unused");
391 struct drm_i915_gem_object *obj_priv; 485 else
392 486 describe_obj(m, obj);
393 obj_priv = to_intel_bo(obj); 487 seq_printf(m, "\n");
394 seq_printf(m, "Fenced object[%2d] = %p: %s "
395 "%08x %08zx %08x %s %08x %08x %d",
396 i, obj, get_pin_flag(obj_priv),
397 obj_priv->gtt_offset,
398 obj->size, obj_priv->stride,
399 get_tiling_flag(obj_priv),
400 obj->read_domains, obj->write_domain,
401 obj_priv->last_rendering_seqno);
402 if (obj->name)
403 seq_printf(m, " (name: %d)", obj->name);
404 seq_printf(m, "\n");
405 }
406 } 488 }
407 mutex_unlock(&dev->struct_mutex);
408 489
490 mutex_unlock(&dev->struct_mutex);
409 return 0; 491 return 0;
410} 492}
411 493
@@ -414,10 +496,12 @@ static int i915_hws_info(struct seq_file *m, void *data)
414 struct drm_info_node *node = (struct drm_info_node *) m->private; 496 struct drm_info_node *node = (struct drm_info_node *) m->private;
415 struct drm_device *dev = node->minor->dev; 497 struct drm_device *dev = node->minor->dev;
416 drm_i915_private_t *dev_priv = dev->dev_private; 498 drm_i915_private_t *dev_priv = dev->dev_private;
417 int i; 499 struct intel_ring_buffer *ring;
418 volatile u32 *hws; 500 volatile u32 *hws;
501 int i;
419 502
420 hws = (volatile u32 *)dev_priv->render_ring.status_page.page_addr; 503 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
504 hws = (volatile u32 *)ring->status_page.page_addr;
421 if (hws == NULL) 505 if (hws == NULL)
422 return 0; 506 return 0;
423 507
@@ -431,14 +515,14 @@ static int i915_hws_info(struct seq_file *m, void *data)
431 515
432static void i915_dump_object(struct seq_file *m, 516static void i915_dump_object(struct seq_file *m,
433 struct io_mapping *mapping, 517 struct io_mapping *mapping,
434 struct drm_i915_gem_object *obj_priv) 518 struct drm_i915_gem_object *obj)
435{ 519{
436 int page, page_count, i; 520 int page, page_count, i;
437 521
438 page_count = obj_priv->base.size / PAGE_SIZE; 522 page_count = obj->base.size / PAGE_SIZE;
439 for (page = 0; page < page_count; page++) { 523 for (page = 0; page < page_count; page++) {
440 u32 *mem = io_mapping_map_wc(mapping, 524 u32 *mem = io_mapping_map_wc(mapping,
441 obj_priv->gtt_offset + page * PAGE_SIZE); 525 obj->gtt_offset + page * PAGE_SIZE);
442 for (i = 0; i < PAGE_SIZE; i += 4) 526 for (i = 0; i < PAGE_SIZE; i += 4)
443 seq_printf(m, "%08x : %08x\n", i, mem[i / 4]); 527 seq_printf(m, "%08x : %08x\n", i, mem[i / 4]);
444 io_mapping_unmap(mem); 528 io_mapping_unmap(mem);
@@ -450,25 +534,21 @@ static int i915_batchbuffer_info(struct seq_file *m, void *data)
450 struct drm_info_node *node = (struct drm_info_node *) m->private; 534 struct drm_info_node *node = (struct drm_info_node *) m->private;
451 struct drm_device *dev = node->minor->dev; 535 struct drm_device *dev = node->minor->dev;
452 drm_i915_private_t *dev_priv = dev->dev_private; 536 drm_i915_private_t *dev_priv = dev->dev_private;
453 struct drm_gem_object *obj; 537 struct drm_i915_gem_object *obj;
454 struct drm_i915_gem_object *obj_priv;
455 int ret; 538 int ret;
456 539
457 ret = mutex_lock_interruptible(&dev->struct_mutex); 540 ret = mutex_lock_interruptible(&dev->struct_mutex);
458 if (ret) 541 if (ret)
459 return ret; 542 return ret;
460 543
461 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 544 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
462 obj = &obj_priv->base; 545 if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) {
463 if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) { 546 seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset);
464 seq_printf(m, "--- gtt_offset = 0x%08x\n", 547 i915_dump_object(m, dev_priv->mm.gtt_mapping, obj);
465 obj_priv->gtt_offset);
466 i915_dump_object(m, dev_priv->mm.gtt_mapping, obj_priv);
467 } 548 }
468 } 549 }
469 550
470 mutex_unlock(&dev->struct_mutex); 551 mutex_unlock(&dev->struct_mutex);
471
472 return 0; 552 return 0;
473} 553}
474 554
@@ -477,19 +557,21 @@ static int i915_ringbuffer_data(struct seq_file *m, void *data)
477 struct drm_info_node *node = (struct drm_info_node *) m->private; 557 struct drm_info_node *node = (struct drm_info_node *) m->private;
478 struct drm_device *dev = node->minor->dev; 558 struct drm_device *dev = node->minor->dev;
479 drm_i915_private_t *dev_priv = dev->dev_private; 559 drm_i915_private_t *dev_priv = dev->dev_private;
560 struct intel_ring_buffer *ring;
480 int ret; 561 int ret;
481 562
482 ret = mutex_lock_interruptible(&dev->struct_mutex); 563 ret = mutex_lock_interruptible(&dev->struct_mutex);
483 if (ret) 564 if (ret)
484 return ret; 565 return ret;
485 566
486 if (!dev_priv->render_ring.gem_object) { 567 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
568 if (!ring->obj) {
487 seq_printf(m, "No ringbuffer setup\n"); 569 seq_printf(m, "No ringbuffer setup\n");
488 } else { 570 } else {
489 u8 *virt = dev_priv->render_ring.virtual_start; 571 u8 *virt = ring->virtual_start;
490 uint32_t off; 572 uint32_t off;
491 573
492 for (off = 0; off < dev_priv->render_ring.size; off += 4) { 574 for (off = 0; off < ring->size; off += 4) {
493 uint32_t *ptr = (uint32_t *)(virt + off); 575 uint32_t *ptr = (uint32_t *)(virt + off);
494 seq_printf(m, "%08x : %08x\n", off, *ptr); 576 seq_printf(m, "%08x : %08x\n", off, *ptr);
495 } 577 }
@@ -504,19 +586,38 @@ static int i915_ringbuffer_info(struct seq_file *m, void *data)
504 struct drm_info_node *node = (struct drm_info_node *) m->private; 586 struct drm_info_node *node = (struct drm_info_node *) m->private;
505 struct drm_device *dev = node->minor->dev; 587 struct drm_device *dev = node->minor->dev;
506 drm_i915_private_t *dev_priv = dev->dev_private; 588 drm_i915_private_t *dev_priv = dev->dev_private;
507 unsigned int head, tail; 589 struct intel_ring_buffer *ring;
508 590
509 head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 591 ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
510 tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; 592 if (ring->size == 0)
593 return 0;
511 594
512 seq_printf(m, "RingHead : %08x\n", head); 595 seq_printf(m, "Ring %s:\n", ring->name);
513 seq_printf(m, "RingTail : %08x\n", tail); 596 seq_printf(m, " Head : %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
514 seq_printf(m, "RingSize : %08lx\n", dev_priv->render_ring.size); 597 seq_printf(m, " Tail : %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
515 seq_printf(m, "Acthd : %08x\n", I915_READ(INTEL_INFO(dev)->gen >= 4 ? ACTHD_I965 : ACTHD)); 598 seq_printf(m, " Size : %08x\n", ring->size);
599 seq_printf(m, " Active : %08x\n", intel_ring_get_active_head(ring));
600 seq_printf(m, " NOPID : %08x\n", I915_READ_NOPID(ring));
601 if (IS_GEN6(dev)) {
602 seq_printf(m, " Sync 0 : %08x\n", I915_READ_SYNC_0(ring));
603 seq_printf(m, " Sync 1 : %08x\n", I915_READ_SYNC_1(ring));
604 }
605 seq_printf(m, " Control : %08x\n", I915_READ_CTL(ring));
606 seq_printf(m, " Start : %08x\n", I915_READ_START(ring));
516 607
517 return 0; 608 return 0;
518} 609}
519 610
611static const char *ring_str(int ring)
612{
613 switch (ring) {
614 case RING_RENDER: return " render";
615 case RING_BSD: return " bsd";
616 case RING_BLT: return " blt";
617 default: return "";
618 }
619}
620
520static const char *pin_flag(int pinned) 621static const char *pin_flag(int pinned)
521{ 622{
522 if (pinned > 0) 623 if (pinned > 0)
@@ -547,6 +648,36 @@ static const char *purgeable_flag(int purgeable)
547 return purgeable ? " purgeable" : ""; 648 return purgeable ? " purgeable" : "";
548} 649}
549 650
651static void print_error_buffers(struct seq_file *m,
652 const char *name,
653 struct drm_i915_error_buffer *err,
654 int count)
655{
656 seq_printf(m, "%s [%d]:\n", name, count);
657
658 while (count--) {
659 seq_printf(m, " %08x %8zd %04x %04x %08x%s%s%s%s%s",
660 err->gtt_offset,
661 err->size,
662 err->read_domains,
663 err->write_domain,
664 err->seqno,
665 pin_flag(err->pinned),
666 tiling_flag(err->tiling),
667 dirty_flag(err->dirty),
668 purgeable_flag(err->purgeable),
669 ring_str(err->ring));
670
671 if (err->name)
672 seq_printf(m, " (name: %d)", err->name);
673 if (err->fence_reg != I915_FENCE_REG_NONE)
674 seq_printf(m, " (fence: %d)", err->fence_reg);
675
676 seq_printf(m, "\n");
677 err++;
678 }
679}
680
550static int i915_error_state(struct seq_file *m, void *unused) 681static int i915_error_state(struct seq_file *m, void *unused)
551{ 682{
552 struct drm_info_node *node = (struct drm_info_node *) m->private; 683 struct drm_info_node *node = (struct drm_info_node *) m->private;
@@ -568,41 +699,46 @@ static int i915_error_state(struct seq_file *m, void *unused)
568 error->time.tv_usec); 699 error->time.tv_usec);
569 seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device); 700 seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
570 seq_printf(m, "EIR: 0x%08x\n", error->eir); 701 seq_printf(m, "EIR: 0x%08x\n", error->eir);
571 seq_printf(m, " PGTBL_ER: 0x%08x\n", error->pgtbl_er); 702 seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
572 seq_printf(m, " INSTPM: 0x%08x\n", error->instpm); 703 if (INTEL_INFO(dev)->gen >= 6) {
704 seq_printf(m, "ERROR: 0x%08x\n", error->error);
705 seq_printf(m, "Blitter command stream:\n");
706 seq_printf(m, " ACTHD: 0x%08x\n", error->bcs_acthd);
707 seq_printf(m, " IPEIR: 0x%08x\n", error->bcs_ipeir);
708 seq_printf(m, " IPEHR: 0x%08x\n", error->bcs_ipehr);
709 seq_printf(m, " INSTDONE: 0x%08x\n", error->bcs_instdone);
710 seq_printf(m, " seqno: 0x%08x\n", error->bcs_seqno);
711 seq_printf(m, "Video (BSD) command stream:\n");
712 seq_printf(m, " ACTHD: 0x%08x\n", error->vcs_acthd);
713 seq_printf(m, " IPEIR: 0x%08x\n", error->vcs_ipeir);
714 seq_printf(m, " IPEHR: 0x%08x\n", error->vcs_ipehr);
715 seq_printf(m, " INSTDONE: 0x%08x\n", error->vcs_instdone);
716 seq_printf(m, " seqno: 0x%08x\n", error->vcs_seqno);
717 }
718 seq_printf(m, "Render command stream:\n");
719 seq_printf(m, " ACTHD: 0x%08x\n", error->acthd);
573 seq_printf(m, " IPEIR: 0x%08x\n", error->ipeir); 720 seq_printf(m, " IPEIR: 0x%08x\n", error->ipeir);
574 seq_printf(m, " IPEHR: 0x%08x\n", error->ipehr); 721 seq_printf(m, " IPEHR: 0x%08x\n", error->ipehr);
575 seq_printf(m, " INSTDONE: 0x%08x\n", error->instdone); 722 seq_printf(m, " INSTDONE: 0x%08x\n", error->instdone);
576 seq_printf(m, " ACTHD: 0x%08x\n", error->acthd);
577 if (INTEL_INFO(dev)->gen >= 4) { 723 if (INTEL_INFO(dev)->gen >= 4) {
578 seq_printf(m, " INSTPS: 0x%08x\n", error->instps);
579 seq_printf(m, " INSTDONE1: 0x%08x\n", error->instdone1); 724 seq_printf(m, " INSTDONE1: 0x%08x\n", error->instdone1);
725 seq_printf(m, " INSTPS: 0x%08x\n", error->instps);
580 } 726 }
581 seq_printf(m, "seqno: 0x%08x\n", error->seqno); 727 seq_printf(m, " INSTPM: 0x%08x\n", error->instpm);
582 728 seq_printf(m, " seqno: 0x%08x\n", error->seqno);
583 if (error->active_bo_count) { 729
584 seq_printf(m, "Buffers [%d]:\n", error->active_bo_count); 730 for (i = 0; i < 16; i++)
585 731 seq_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]);
586 for (i = 0; i < error->active_bo_count; i++) { 732
587 seq_printf(m, " %08x %8zd %08x %08x %08x%s%s%s%s", 733 if (error->active_bo)
588 error->active_bo[i].gtt_offset, 734 print_error_buffers(m, "Active",
589 error->active_bo[i].size, 735 error->active_bo,
590 error->active_bo[i].read_domains, 736 error->active_bo_count);
591 error->active_bo[i].write_domain, 737
592 error->active_bo[i].seqno, 738 if (error->pinned_bo)
593 pin_flag(error->active_bo[i].pinned), 739 print_error_buffers(m, "Pinned",
594 tiling_flag(error->active_bo[i].tiling), 740 error->pinned_bo,
595 dirty_flag(error->active_bo[i].dirty), 741 error->pinned_bo_count);
596 purgeable_flag(error->active_bo[i].purgeable));
597
598 if (error->active_bo[i].name)
599 seq_printf(m, " (name: %d)", error->active_bo[i].name);
600 if (error->active_bo[i].fence_reg != I915_FENCE_REG_NONE)
601 seq_printf(m, " (fence: %d)", error->active_bo[i].fence_reg);
602
603 seq_printf(m, "\n");
604 }
605 }
606 742
607 for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) { 743 for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) {
608 if (error->batchbuffer[i]) { 744 if (error->batchbuffer[i]) {
@@ -635,6 +771,9 @@ static int i915_error_state(struct seq_file *m, void *unused)
635 if (error->overlay) 771 if (error->overlay)
636 intel_overlay_print_error_state(m, error->overlay); 772 intel_overlay_print_error_state(m, error->overlay);
637 773
774 if (error->display)
775 intel_display_print_error_state(m, dev, error->display);
776
638out: 777out:
639 spin_unlock_irqrestore(&dev_priv->error_lock, flags); 778 spin_unlock_irqrestore(&dev_priv->error_lock, flags);
640 779
@@ -658,15 +797,51 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
658 struct drm_info_node *node = (struct drm_info_node *) m->private; 797 struct drm_info_node *node = (struct drm_info_node *) m->private;
659 struct drm_device *dev = node->minor->dev; 798 struct drm_device *dev = node->minor->dev;
660 drm_i915_private_t *dev_priv = dev->dev_private; 799 drm_i915_private_t *dev_priv = dev->dev_private;
661 u16 rgvswctl = I915_READ16(MEMSWCTL);
662 u16 rgvstat = I915_READ16(MEMSTAT_ILK);
663 800
664 seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); 801 if (IS_GEN5(dev)) {
665 seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); 802 u16 rgvswctl = I915_READ16(MEMSWCTL);
666 seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> 803 u16 rgvstat = I915_READ16(MEMSTAT_ILK);
667 MEMSTAT_VID_SHIFT); 804
668 seq_printf(m, "Current P-state: %d\n", 805 seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf);
669 (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); 806 seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f);
807 seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >>
808 MEMSTAT_VID_SHIFT);
809 seq_printf(m, "Current P-state: %d\n",
810 (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT);
811 } else if (IS_GEN6(dev)) {
812 u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
813 u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS);
814 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
815 int max_freq;
816
817 /* RPSTAT1 is in the GT power well */
818 __gen6_force_wake_get(dev_priv);
819
820 seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
821 seq_printf(m, "RPSTAT1: 0x%08x\n", I915_READ(GEN6_RPSTAT1));
822 seq_printf(m, "Render p-state ratio: %d\n",
823 (gt_perf_status & 0xff00) >> 8);
824 seq_printf(m, "Render p-state VID: %d\n",
825 gt_perf_status & 0xff);
826 seq_printf(m, "Render p-state limit: %d\n",
827 rp_state_limits & 0xff);
828
829 max_freq = (rp_state_cap & 0xff0000) >> 16;
830 seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
831 max_freq * 100);
832
833 max_freq = (rp_state_cap & 0xff00) >> 8;
834 seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
835 max_freq * 100);
836
837 max_freq = rp_state_cap & 0xff;
838 seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
839 max_freq * 100);
840
841 __gen6_force_wake_put(dev_priv);
842 } else {
843 seq_printf(m, "no P-state info available\n");
844 }
670 845
671 return 0; 846 return 0;
672} 847}
@@ -794,7 +969,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
794 drm_i915_private_t *dev_priv = dev->dev_private; 969 drm_i915_private_t *dev_priv = dev->dev_private;
795 bool sr_enabled = false; 970 bool sr_enabled = false;
796 971
797 if (IS_GEN5(dev)) 972 if (HAS_PCH_SPLIT(dev))
798 sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; 973 sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
799 else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev)) 974 else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev))
800 sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; 975 sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
@@ -886,7 +1061,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
886 fb->base.height, 1061 fb->base.height,
887 fb->base.depth, 1062 fb->base.depth,
888 fb->base.bits_per_pixel); 1063 fb->base.bits_per_pixel);
889 describe_obj(m, to_intel_bo(fb->obj)); 1064 describe_obj(m, fb->obj);
890 seq_printf(m, "\n"); 1065 seq_printf(m, "\n");
891 1066
892 list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) { 1067 list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) {
@@ -898,7 +1073,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
898 fb->base.height, 1073 fb->base.height,
899 fb->base.depth, 1074 fb->base.depth,
900 fb->base.bits_per_pixel); 1075 fb->base.bits_per_pixel);
901 describe_obj(m, to_intel_bo(fb->obj)); 1076 describe_obj(m, fb->obj);
902 seq_printf(m, "\n"); 1077 seq_printf(m, "\n");
903 } 1078 }
904 1079
@@ -943,7 +1118,6 @@ i915_wedged_write(struct file *filp,
943 loff_t *ppos) 1118 loff_t *ppos)
944{ 1119{
945 struct drm_device *dev = filp->private_data; 1120 struct drm_device *dev = filp->private_data;
946 drm_i915_private_t *dev_priv = dev->dev_private;
947 char buf[20]; 1121 char buf[20];
948 int val = 1; 1122 int val = 1;
949 1123
@@ -959,12 +1133,7 @@ i915_wedged_write(struct file *filp,
959 } 1133 }
960 1134
961 DRM_INFO("Manually setting wedged to %d\n", val); 1135 DRM_INFO("Manually setting wedged to %d\n", val);
962 1136 i915_handle_error(dev, val);
963 atomic_set(&dev_priv->mm.wedged, val);
964 if (val) {
965 wake_up_all(&dev_priv->irq_queue);
966 queue_work(dev_priv->wq, &dev_priv->error_work);
967 }
968 1137
969 return cnt; 1138 return cnt;
970} 1139}
@@ -1028,9 +1197,15 @@ static struct drm_info_list i915_debugfs_list[] = {
1028 {"i915_gem_seqno", i915_gem_seqno_info, 0}, 1197 {"i915_gem_seqno", i915_gem_seqno_info, 0},
1029 {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, 1198 {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
1030 {"i915_gem_interrupt", i915_interrupt_info, 0}, 1199 {"i915_gem_interrupt", i915_interrupt_info, 0},
1031 {"i915_gem_hws", i915_hws_info, 0}, 1200 {"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
1032 {"i915_ringbuffer_data", i915_ringbuffer_data, 0}, 1201 {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
1033 {"i915_ringbuffer_info", i915_ringbuffer_info, 0}, 1202 {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
1203 {"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
1204 {"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
1205 {"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
1206 {"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
1207 {"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
1208 {"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
1034 {"i915_batchbuffers", i915_batchbuffer_info, 0}, 1209 {"i915_batchbuffers", i915_batchbuffer_info, 0},
1035 {"i915_error_state", i915_error_state, 0}, 1210 {"i915_error_state", i915_error_state, 0},
1036 {"i915_rstdby_delays", i915_rstdby_delays, 0}, 1211 {"i915_rstdby_delays", i915_rstdby_delays, 0},
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index cb900dc83d95..0568dbdc10ef 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -50,6 +50,8 @@
50static int i915_init_phys_hws(struct drm_device *dev) 50static int i915_init_phys_hws(struct drm_device *dev)
51{ 51{
52 drm_i915_private_t *dev_priv = dev->dev_private; 52 drm_i915_private_t *dev_priv = dev->dev_private;
53 struct intel_ring_buffer *ring = LP_RING(dev_priv);
54
53 /* Program Hardware Status Page */ 55 /* Program Hardware Status Page */
54 dev_priv->status_page_dmah = 56 dev_priv->status_page_dmah =
55 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE); 57 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE);
@@ -58,11 +60,10 @@ static int i915_init_phys_hws(struct drm_device *dev)
58 DRM_ERROR("Can not allocate hardware status page\n"); 60 DRM_ERROR("Can not allocate hardware status page\n");
59 return -ENOMEM; 61 return -ENOMEM;
60 } 62 }
61 dev_priv->render_ring.status_page.page_addr 63 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
62 = dev_priv->status_page_dmah->vaddr;
63 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr; 64 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
64 65
65 memset(dev_priv->render_ring.status_page.page_addr, 0, PAGE_SIZE); 66 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
66 67
67 if (INTEL_INFO(dev)->gen >= 4) 68 if (INTEL_INFO(dev)->gen >= 4)
68 dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) & 69 dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) &
@@ -80,13 +81,15 @@ static int i915_init_phys_hws(struct drm_device *dev)
80static void i915_free_hws(struct drm_device *dev) 81static void i915_free_hws(struct drm_device *dev)
81{ 82{
82 drm_i915_private_t *dev_priv = dev->dev_private; 83 drm_i915_private_t *dev_priv = dev->dev_private;
84 struct intel_ring_buffer *ring = LP_RING(dev_priv);
85
83 if (dev_priv->status_page_dmah) { 86 if (dev_priv->status_page_dmah) {
84 drm_pci_free(dev, dev_priv->status_page_dmah); 87 drm_pci_free(dev, dev_priv->status_page_dmah);
85 dev_priv->status_page_dmah = NULL; 88 dev_priv->status_page_dmah = NULL;
86 } 89 }
87 90
88 if (dev_priv->render_ring.status_page.gfx_addr) { 91 if (ring->status_page.gfx_addr) {
89 dev_priv->render_ring.status_page.gfx_addr = 0; 92 ring->status_page.gfx_addr = 0;
90 drm_core_ioremapfree(&dev_priv->hws_map, dev); 93 drm_core_ioremapfree(&dev_priv->hws_map, dev);
91 } 94 }
92 95
@@ -98,7 +101,7 @@ void i915_kernel_lost_context(struct drm_device * dev)
98{ 101{
99 drm_i915_private_t *dev_priv = dev->dev_private; 102 drm_i915_private_t *dev_priv = dev->dev_private;
100 struct drm_i915_master_private *master_priv; 103 struct drm_i915_master_private *master_priv;
101 struct intel_ring_buffer *ring = &dev_priv->render_ring; 104 struct intel_ring_buffer *ring = LP_RING(dev_priv);
102 105
103 /* 106 /*
104 * We should never lose context on the ring with modesetting 107 * We should never lose context on the ring with modesetting
@@ -107,8 +110,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
107 if (drm_core_check_feature(dev, DRIVER_MODESET)) 110 if (drm_core_check_feature(dev, DRIVER_MODESET))
108 return; 111 return;
109 112
110 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR; 113 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
111 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; 114 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
112 ring->space = ring->head - (ring->tail + 8); 115 ring->space = ring->head - (ring->tail + 8);
113 if (ring->space < 0) 116 if (ring->space < 0)
114 ring->space += ring->size; 117 ring->space += ring->size;
@@ -124,6 +127,8 @@ void i915_kernel_lost_context(struct drm_device * dev)
124static int i915_dma_cleanup(struct drm_device * dev) 127static int i915_dma_cleanup(struct drm_device * dev)
125{ 128{
126 drm_i915_private_t *dev_priv = dev->dev_private; 129 drm_i915_private_t *dev_priv = dev->dev_private;
130 int i;
131
127 /* Make sure interrupts are disabled here because the uninstall ioctl 132 /* Make sure interrupts are disabled here because the uninstall ioctl
128 * may not have been called from userspace and after dev_private 133 * may not have been called from userspace and after dev_private
129 * is freed, it's too late. 134 * is freed, it's too late.
@@ -132,9 +137,8 @@ static int i915_dma_cleanup(struct drm_device * dev)
132 drm_irq_uninstall(dev); 137 drm_irq_uninstall(dev);
133 138
134 mutex_lock(&dev->struct_mutex); 139 mutex_lock(&dev->struct_mutex);
135 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 140 for (i = 0; i < I915_NUM_RINGS; i++)
136 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 141 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
137 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
138 mutex_unlock(&dev->struct_mutex); 142 mutex_unlock(&dev->struct_mutex);
139 143
140 /* Clear the HWS virtual address at teardown */ 144 /* Clear the HWS virtual address at teardown */
@@ -148,6 +152,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
148{ 152{
149 drm_i915_private_t *dev_priv = dev->dev_private; 153 drm_i915_private_t *dev_priv = dev->dev_private;
150 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; 154 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
155 struct intel_ring_buffer *ring = LP_RING(dev_priv);
151 156
152 master_priv->sarea = drm_getsarea(dev); 157 master_priv->sarea = drm_getsarea(dev);
153 if (master_priv->sarea) { 158 if (master_priv->sarea) {
@@ -158,24 +163,24 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
158 } 163 }
159 164
160 if (init->ring_size != 0) { 165 if (init->ring_size != 0) {
161 if (dev_priv->render_ring.gem_object != NULL) { 166 if (ring->obj != NULL) {
162 i915_dma_cleanup(dev); 167 i915_dma_cleanup(dev);
163 DRM_ERROR("Client tried to initialize ringbuffer in " 168 DRM_ERROR("Client tried to initialize ringbuffer in "
164 "GEM mode\n"); 169 "GEM mode\n");
165 return -EINVAL; 170 return -EINVAL;
166 } 171 }
167 172
168 dev_priv->render_ring.size = init->ring_size; 173 ring->size = init->ring_size;
169 174
170 dev_priv->render_ring.map.offset = init->ring_start; 175 ring->map.offset = init->ring_start;
171 dev_priv->render_ring.map.size = init->ring_size; 176 ring->map.size = init->ring_size;
172 dev_priv->render_ring.map.type = 0; 177 ring->map.type = 0;
173 dev_priv->render_ring.map.flags = 0; 178 ring->map.flags = 0;
174 dev_priv->render_ring.map.mtrr = 0; 179 ring->map.mtrr = 0;
175 180
176 drm_core_ioremap_wc(&dev_priv->render_ring.map, dev); 181 drm_core_ioremap_wc(&ring->map, dev);
177 182
178 if (dev_priv->render_ring.map.handle == NULL) { 183 if (ring->map.handle == NULL) {
179 i915_dma_cleanup(dev); 184 i915_dma_cleanup(dev);
180 DRM_ERROR("can not ioremap virtual address for" 185 DRM_ERROR("can not ioremap virtual address for"
181 " ring buffer\n"); 186 " ring buffer\n");
@@ -183,7 +188,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
183 } 188 }
184 } 189 }
185 190
186 dev_priv->render_ring.virtual_start = dev_priv->render_ring.map.handle; 191 ring->virtual_start = ring->map.handle;
187 192
188 dev_priv->cpp = init->cpp; 193 dev_priv->cpp = init->cpp;
189 dev_priv->back_offset = init->back_offset; 194 dev_priv->back_offset = init->back_offset;
@@ -202,12 +207,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
202static int i915_dma_resume(struct drm_device * dev) 207static int i915_dma_resume(struct drm_device * dev)
203{ 208{
204 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 209 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
210 struct intel_ring_buffer *ring = LP_RING(dev_priv);
205 211
206 struct intel_ring_buffer *ring;
207 DRM_DEBUG_DRIVER("%s\n", __func__); 212 DRM_DEBUG_DRIVER("%s\n", __func__);
208 213
209 ring = &dev_priv->render_ring;
210
211 if (ring->map.handle == NULL) { 214 if (ring->map.handle == NULL) {
212 DRM_ERROR("can not ioremap virtual address for" 215 DRM_ERROR("can not ioremap virtual address for"
213 " ring buffer\n"); 216 " ring buffer\n");
@@ -222,7 +225,7 @@ static int i915_dma_resume(struct drm_device * dev)
222 DRM_DEBUG_DRIVER("hw status page @ %p\n", 225 DRM_DEBUG_DRIVER("hw status page @ %p\n",
223 ring->status_page.page_addr); 226 ring->status_page.page_addr);
224 if (ring->status_page.gfx_addr != 0) 227 if (ring->status_page.gfx_addr != 0)
225 intel_ring_setup_status_page(dev, ring); 228 intel_ring_setup_status_page(ring);
226 else 229 else
227 I915_WRITE(HWS_PGA, dev_priv->dma_status_page); 230 I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
228 231
@@ -264,7 +267,7 @@ static int i915_dma_init(struct drm_device *dev, void *data,
264 * instruction detected will be given a size of zero, which is a 267 * instruction detected will be given a size of zero, which is a
265 * signal to abort the rest of the buffer. 268 * signal to abort the rest of the buffer.
266 */ 269 */
267static int do_validate_cmd(int cmd) 270static int validate_cmd(int cmd)
268{ 271{
269 switch (((cmd >> 29) & 0x7)) { 272 switch (((cmd >> 29) & 0x7)) {
270 case 0x0: 273 case 0x0:
@@ -322,40 +325,27 @@ static int do_validate_cmd(int cmd)
322 return 0; 325 return 0;
323} 326}
324 327
325static int validate_cmd(int cmd)
326{
327 int ret = do_validate_cmd(cmd);
328
329/* printk("validate_cmd( %x ): %d\n", cmd, ret); */
330
331 return ret;
332}
333
334static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) 328static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
335{ 329{
336 drm_i915_private_t *dev_priv = dev->dev_private; 330 drm_i915_private_t *dev_priv = dev->dev_private;
337 int i; 331 int i, ret;
338 332
339 if ((dwords+1) * sizeof(int) >= dev_priv->render_ring.size - 8) 333 if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
340 return -EINVAL; 334 return -EINVAL;
341 335
342 BEGIN_LP_RING((dwords+1)&~1);
343
344 for (i = 0; i < dwords;) { 336 for (i = 0; i < dwords;) {
345 int cmd, sz; 337 int sz = validate_cmd(buffer[i]);
346 338 if (sz == 0 || i + sz > dwords)
347 cmd = buffer[i];
348
349 if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
350 return -EINVAL; 339 return -EINVAL;
351 340 i += sz;
352 OUT_RING(cmd);
353
354 while (++i, --sz) {
355 OUT_RING(buffer[i]);
356 }
357 } 341 }
358 342
343 ret = BEGIN_LP_RING((dwords+1)&~1);
344 if (ret)
345 return ret;
346
347 for (i = 0; i < dwords; i++)
348 OUT_RING(buffer[i]);
359 if (dwords & 1) 349 if (dwords & 1)
360 OUT_RING(0); 350 OUT_RING(0);
361 351
@@ -366,34 +356,41 @@ static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
366 356
367int 357int
368i915_emit_box(struct drm_device *dev, 358i915_emit_box(struct drm_device *dev,
369 struct drm_clip_rect *boxes, 359 struct drm_clip_rect *box,
370 int i, int DR1, int DR4) 360 int DR1, int DR4)
371{ 361{
372 struct drm_clip_rect box = boxes[i]; 362 struct drm_i915_private *dev_priv = dev->dev_private;
363 int ret;
373 364
374 if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) { 365 if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
366 box->y2 <= 0 || box->x2 <= 0) {
375 DRM_ERROR("Bad box %d,%d..%d,%d\n", 367 DRM_ERROR("Bad box %d,%d..%d,%d\n",
376 box.x1, box.y1, box.x2, box.y2); 368 box->x1, box->y1, box->x2, box->y2);
377 return -EINVAL; 369 return -EINVAL;
378 } 370 }
379 371
380 if (INTEL_INFO(dev)->gen >= 4) { 372 if (INTEL_INFO(dev)->gen >= 4) {
381 BEGIN_LP_RING(4); 373 ret = BEGIN_LP_RING(4);
374 if (ret)
375 return ret;
376
382 OUT_RING(GFX_OP_DRAWRECT_INFO_I965); 377 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
383 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); 378 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
384 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); 379 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
385 OUT_RING(DR4); 380 OUT_RING(DR4);
386 ADVANCE_LP_RING();
387 } else { 381 } else {
388 BEGIN_LP_RING(6); 382 ret = BEGIN_LP_RING(6);
383 if (ret)
384 return ret;
385
389 OUT_RING(GFX_OP_DRAWRECT_INFO); 386 OUT_RING(GFX_OP_DRAWRECT_INFO);
390 OUT_RING(DR1); 387 OUT_RING(DR1);
391 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); 388 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
392 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); 389 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
393 OUT_RING(DR4); 390 OUT_RING(DR4);
394 OUT_RING(0); 391 OUT_RING(0);
395 ADVANCE_LP_RING();
396 } 392 }
393 ADVANCE_LP_RING();
397 394
398 return 0; 395 return 0;
399} 396}
@@ -413,12 +410,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
413 if (master_priv->sarea_priv) 410 if (master_priv->sarea_priv)
414 master_priv->sarea_priv->last_enqueue = dev_priv->counter; 411 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
415 412
416 BEGIN_LP_RING(4); 413 if (BEGIN_LP_RING(4) == 0) {
417 OUT_RING(MI_STORE_DWORD_INDEX); 414 OUT_RING(MI_STORE_DWORD_INDEX);
418 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 415 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
419 OUT_RING(dev_priv->counter); 416 OUT_RING(dev_priv->counter);
420 OUT_RING(0); 417 OUT_RING(0);
421 ADVANCE_LP_RING(); 418 ADVANCE_LP_RING();
419 }
422} 420}
423 421
424static int i915_dispatch_cmdbuffer(struct drm_device * dev, 422static int i915_dispatch_cmdbuffer(struct drm_device * dev,
@@ -440,7 +438,7 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev,
440 438
441 for (i = 0; i < count; i++) { 439 for (i = 0; i < count; i++) {
442 if (i < nbox) { 440 if (i < nbox) {
443 ret = i915_emit_box(dev, cliprects, i, 441 ret = i915_emit_box(dev, &cliprects[i],
444 cmd->DR1, cmd->DR4); 442 cmd->DR1, cmd->DR4);
445 if (ret) 443 if (ret)
446 return ret; 444 return ret;
@@ -459,8 +457,9 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
459 drm_i915_batchbuffer_t * batch, 457 drm_i915_batchbuffer_t * batch,
460 struct drm_clip_rect *cliprects) 458 struct drm_clip_rect *cliprects)
461{ 459{
460 struct drm_i915_private *dev_priv = dev->dev_private;
462 int nbox = batch->num_cliprects; 461 int nbox = batch->num_cliprects;
463 int i = 0, count; 462 int i, count, ret;
464 463
465 if ((batch->start | batch->used) & 0x7) { 464 if ((batch->start | batch->used) & 0x7) {
466 DRM_ERROR("alignment"); 465 DRM_ERROR("alignment");
@@ -470,17 +469,19 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
470 i915_kernel_lost_context(dev); 469 i915_kernel_lost_context(dev);
471 470
472 count = nbox ? nbox : 1; 471 count = nbox ? nbox : 1;
473
474 for (i = 0; i < count; i++) { 472 for (i = 0; i < count; i++) {
475 if (i < nbox) { 473 if (i < nbox) {
476 int ret = i915_emit_box(dev, cliprects, i, 474 ret = i915_emit_box(dev, &cliprects[i],
477 batch->DR1, batch->DR4); 475 batch->DR1, batch->DR4);
478 if (ret) 476 if (ret)
479 return ret; 477 return ret;
480 } 478 }
481 479
482 if (!IS_I830(dev) && !IS_845G(dev)) { 480 if (!IS_I830(dev) && !IS_845G(dev)) {
483 BEGIN_LP_RING(2); 481 ret = BEGIN_LP_RING(2);
482 if (ret)
483 return ret;
484
484 if (INTEL_INFO(dev)->gen >= 4) { 485 if (INTEL_INFO(dev)->gen >= 4) {
485 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); 486 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
486 OUT_RING(batch->start); 487 OUT_RING(batch->start);
@@ -488,26 +489,29 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
488 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); 489 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
489 OUT_RING(batch->start | MI_BATCH_NON_SECURE); 490 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
490 } 491 }
491 ADVANCE_LP_RING();
492 } else { 492 } else {
493 BEGIN_LP_RING(4); 493 ret = BEGIN_LP_RING(4);
494 if (ret)
495 return ret;
496
494 OUT_RING(MI_BATCH_BUFFER); 497 OUT_RING(MI_BATCH_BUFFER);
495 OUT_RING(batch->start | MI_BATCH_NON_SECURE); 498 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
496 OUT_RING(batch->start + batch->used - 4); 499 OUT_RING(batch->start + batch->used - 4);
497 OUT_RING(0); 500 OUT_RING(0);
498 ADVANCE_LP_RING();
499 } 501 }
502 ADVANCE_LP_RING();
500 } 503 }
501 504
502 505
503 if (IS_G4X(dev) || IS_GEN5(dev)) { 506 if (IS_G4X(dev) || IS_GEN5(dev)) {
504 BEGIN_LP_RING(2); 507 if (BEGIN_LP_RING(2) == 0) {
505 OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP); 508 OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
506 OUT_RING(MI_NOOP); 509 OUT_RING(MI_NOOP);
507 ADVANCE_LP_RING(); 510 ADVANCE_LP_RING();
511 }
508 } 512 }
509 i915_emit_breadcrumb(dev);
510 513
514 i915_emit_breadcrumb(dev);
511 return 0; 515 return 0;
512} 516}
513 517
@@ -516,6 +520,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
516 drm_i915_private_t *dev_priv = dev->dev_private; 520 drm_i915_private_t *dev_priv = dev->dev_private;
517 struct drm_i915_master_private *master_priv = 521 struct drm_i915_master_private *master_priv =
518 dev->primary->master->driver_priv; 522 dev->primary->master->driver_priv;
523 int ret;
519 524
520 if (!master_priv->sarea_priv) 525 if (!master_priv->sarea_priv)
521 return -EINVAL; 526 return -EINVAL;
@@ -527,12 +532,13 @@ static int i915_dispatch_flip(struct drm_device * dev)
527 532
528 i915_kernel_lost_context(dev); 533 i915_kernel_lost_context(dev);
529 534
530 BEGIN_LP_RING(2); 535 ret = BEGIN_LP_RING(10);
536 if (ret)
537 return ret;
538
531 OUT_RING(MI_FLUSH | MI_READ_FLUSH); 539 OUT_RING(MI_FLUSH | MI_READ_FLUSH);
532 OUT_RING(0); 540 OUT_RING(0);
533 ADVANCE_LP_RING();
534 541
535 BEGIN_LP_RING(6);
536 OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); 542 OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
537 OUT_RING(0); 543 OUT_RING(0);
538 if (dev_priv->current_page == 0) { 544 if (dev_priv->current_page == 0) {
@@ -543,33 +549,32 @@ static int i915_dispatch_flip(struct drm_device * dev)
543 dev_priv->current_page = 0; 549 dev_priv->current_page = 0;
544 } 550 }
545 OUT_RING(0); 551 OUT_RING(0);
546 ADVANCE_LP_RING();
547 552
548 BEGIN_LP_RING(2);
549 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); 553 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
550 OUT_RING(0); 554 OUT_RING(0);
555
551 ADVANCE_LP_RING(); 556 ADVANCE_LP_RING();
552 557
553 master_priv->sarea_priv->last_enqueue = dev_priv->counter++; 558 master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
554 559
555 BEGIN_LP_RING(4); 560 if (BEGIN_LP_RING(4) == 0) {
556 OUT_RING(MI_STORE_DWORD_INDEX); 561 OUT_RING(MI_STORE_DWORD_INDEX);
557 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 562 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
558 OUT_RING(dev_priv->counter); 563 OUT_RING(dev_priv->counter);
559 OUT_RING(0); 564 OUT_RING(0);
560 ADVANCE_LP_RING(); 565 ADVANCE_LP_RING();
566 }
561 567
562 master_priv->sarea_priv->pf_current_page = dev_priv->current_page; 568 master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
563 return 0; 569 return 0;
564} 570}
565 571
566static int i915_quiescent(struct drm_device * dev) 572static int i915_quiescent(struct drm_device *dev)
567{ 573{
568 drm_i915_private_t *dev_priv = dev->dev_private; 574 struct intel_ring_buffer *ring = LP_RING(dev->dev_private);
569 575
570 i915_kernel_lost_context(dev); 576 i915_kernel_lost_context(dev);
571 return intel_wait_ring_buffer(dev, &dev_priv->render_ring, 577 return intel_wait_ring_buffer(ring, ring->size - 8);
572 dev_priv->render_ring.size - 8);
573} 578}
574 579
575static int i915_flush_ioctl(struct drm_device *dev, void *data, 580static int i915_flush_ioctl(struct drm_device *dev, void *data,
@@ -768,9 +773,15 @@ static int i915_getparam(struct drm_device *dev, void *data,
768 case I915_PARAM_HAS_BLT: 773 case I915_PARAM_HAS_BLT:
769 value = HAS_BLT(dev); 774 value = HAS_BLT(dev);
770 break; 775 break;
776 case I915_PARAM_HAS_RELAXED_FENCING:
777 value = 1;
778 break;
771 case I915_PARAM_HAS_COHERENT_RINGS: 779 case I915_PARAM_HAS_COHERENT_RINGS:
772 value = 1; 780 value = 1;
773 break; 781 break;
782 case I915_PARAM_HAS_EXEC_CONSTANTS:
783 value = INTEL_INFO(dev)->gen >= 4;
784 break;
774 default: 785 default:
775 DRM_DEBUG_DRIVER("Unknown parameter %d\n", 786 DRM_DEBUG_DRIVER("Unknown parameter %d\n",
776 param->param); 787 param->param);
@@ -826,7 +837,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
826{ 837{
827 drm_i915_private_t *dev_priv = dev->dev_private; 838 drm_i915_private_t *dev_priv = dev->dev_private;
828 drm_i915_hws_addr_t *hws = data; 839 drm_i915_hws_addr_t *hws = data;
829 struct intel_ring_buffer *ring = &dev_priv->render_ring; 840 struct intel_ring_buffer *ring = LP_RING(dev_priv);
830 841
831 if (!I915_NEED_GFX_HWS(dev)) 842 if (!I915_NEED_GFX_HWS(dev))
832 return -EINVAL; 843 return -EINVAL;
@@ -1005,73 +1016,47 @@ intel_teardown_mchbar(struct drm_device *dev)
1005#define PTE_VALID (1 << 0) 1016#define PTE_VALID (1 << 0)
1006 1017
1007/** 1018/**
1008 * i915_gtt_to_phys - take a GTT address and turn it into a physical one 1019 * i915_stolen_to_phys - take an offset into stolen memory and turn it into
1020 * a physical one
1009 * @dev: drm device 1021 * @dev: drm device
1010 * @gtt_addr: address to translate 1022 * @offset: address to translate
1011 * 1023 *
1012 * Some chip functions require allocations from stolen space but need the 1024 * Some chip functions require allocations from stolen space and need the
1013 * physical address of the memory in question. We use this routine 1025 * physical address of the memory in question.
1014 * to get a physical address suitable for register programming from a given
1015 * GTT address.
1016 */ 1026 */
1017static unsigned long i915_gtt_to_phys(struct drm_device *dev, 1027static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
1018 unsigned long gtt_addr)
1019{ 1028{
1020 unsigned long *gtt; 1029 struct drm_i915_private *dev_priv = dev->dev_private;
1021 unsigned long entry, phys; 1030 struct pci_dev *pdev = dev_priv->bridge_dev;
1022 int gtt_bar = IS_GEN2(dev) ? 1 : 0; 1031 u32 base;
1023 int gtt_offset, gtt_size; 1032
1024 1033#if 0
1025 if (INTEL_INFO(dev)->gen >= 4) { 1034 /* On the machines I have tested the Graphics Base of Stolen Memory
1026 if (IS_G4X(dev) || INTEL_INFO(dev)->gen > 4) { 1035 * is unreliable, so compute the base by subtracting the stolen memory
1027 gtt_offset = 2*1024*1024; 1036 * from the Top of Low Usable DRAM which is where the BIOS places
1028 gtt_size = 2*1024*1024; 1037 * the graphics stolen memory.
1029 } else { 1038 */
1030 gtt_offset = 512*1024; 1039 if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
1031 gtt_size = 512*1024; 1040 /* top 32bits are reserved = 0 */
1032 } 1041 pci_read_config_dword(pdev, 0xA4, &base);
1033 } else { 1042 } else {
1034 gtt_bar = 3; 1043 /* XXX presume 8xx is the same as i915 */
1035 gtt_offset = 0; 1044 pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
1036 gtt_size = pci_resource_len(dev->pdev, gtt_bar); 1045 }
1037 } 1046#else
1038 1047 if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
1039 gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset, 1048 u16 val;
1040 gtt_size); 1049 pci_read_config_word(pdev, 0xb0, &val);
1041 if (!gtt) { 1050 base = val >> 4 << 20;
1042 DRM_ERROR("ioremap of GTT failed\n"); 1051 } else {
1043 return 0; 1052 u8 val;
1044 } 1053 pci_read_config_byte(pdev, 0x9c, &val);
1045 1054 base = val >> 3 << 27;
1046 entry = *(volatile u32 *)(gtt + (gtt_addr / 1024));
1047
1048 DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry);
1049
1050 /* Mask out these reserved bits on this hardware. */
1051 if (INTEL_INFO(dev)->gen < 4 && !IS_G33(dev))
1052 entry &= ~PTE_ADDRESS_MASK_HIGH;
1053
1054 /* If it's not a mapping type we know, then bail. */
1055 if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED &&
1056 (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) {
1057 iounmap(gtt);
1058 return 0;
1059 }
1060
1061 if (!(entry & PTE_VALID)) {
1062 DRM_ERROR("bad GTT entry in stolen space\n");
1063 iounmap(gtt);
1064 return 0;
1065 } 1055 }
1056 base -= dev_priv->mm.gtt->stolen_size;
1057#endif
1066 1058
1067 iounmap(gtt); 1059 return base + offset;
1068
1069 phys =(entry & PTE_ADDRESS_MASK) |
1070 ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4));
1071
1072 DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys);
1073
1074 return phys;
1075} 1060}
1076 1061
1077static void i915_warn_stolen(struct drm_device *dev) 1062static void i915_warn_stolen(struct drm_device *dev)
@@ -1087,54 +1072,35 @@ static void i915_setup_compression(struct drm_device *dev, int size)
1087 unsigned long cfb_base; 1072 unsigned long cfb_base;
1088 unsigned long ll_base = 0; 1073 unsigned long ll_base = 0;
1089 1074
1090 /* Leave 1M for line length buffer & misc. */ 1075 compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
1091 compressed_fb = drm_mm_search_free(&dev_priv->mm.vram, size, 4096, 0); 1076 if (compressed_fb)
1092 if (!compressed_fb) { 1077 compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
1093 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; 1078 if (!compressed_fb)
1094 i915_warn_stolen(dev); 1079 goto err;
1095 return;
1096 }
1097 1080
1098 compressed_fb = drm_mm_get_block(compressed_fb, size, 4096); 1081 cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
1099 if (!compressed_fb) { 1082 if (!cfb_base)
1100 i915_warn_stolen(dev); 1083 goto err_fb;
1101 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
1102 return;
1103 }
1104 1084
1105 cfb_base = i915_gtt_to_phys(dev, compressed_fb->start); 1085 if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) {
1106 if (!cfb_base) { 1086 compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
1107 DRM_ERROR("failed to get stolen phys addr, disabling FBC\n"); 1087 4096, 4096, 0);
1108 drm_mm_put_block(compressed_fb); 1088 if (compressed_llb)
1109 } 1089 compressed_llb = drm_mm_get_block(compressed_llb,
1090 4096, 4096);
1091 if (!compressed_llb)
1092 goto err_fb;
1110 1093
1111 if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) { 1094 ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
1112 compressed_llb = drm_mm_search_free(&dev_priv->mm.vram, 4096, 1095 if (!ll_base)
1113 4096, 0); 1096 goto err_llb;
1114 if (!compressed_llb) {
1115 i915_warn_stolen(dev);
1116 return;
1117 }
1118
1119 compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096);
1120 if (!compressed_llb) {
1121 i915_warn_stolen(dev);
1122 return;
1123 }
1124
1125 ll_base = i915_gtt_to_phys(dev, compressed_llb->start);
1126 if (!ll_base) {
1127 DRM_ERROR("failed to get stolen phys addr, disabling FBC\n");
1128 drm_mm_put_block(compressed_fb);
1129 drm_mm_put_block(compressed_llb);
1130 }
1131 } 1097 }
1132 1098
1133 dev_priv->cfb_size = size; 1099 dev_priv->cfb_size = size;
1134 1100
1135 intel_disable_fbc(dev); 1101 intel_disable_fbc(dev);
1136 dev_priv->compressed_fb = compressed_fb; 1102 dev_priv->compressed_fb = compressed_fb;
1137 if (IS_IRONLAKE_M(dev)) 1103 if (HAS_PCH_SPLIT(dev))
1138 I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start); 1104 I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
1139 else if (IS_GM45(dev)) { 1105 else if (IS_GM45(dev)) {
1140 I915_WRITE(DPFC_CB_BASE, compressed_fb->start); 1106 I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
@@ -1144,8 +1110,17 @@ static void i915_setup_compression(struct drm_device *dev, int size)
1144 dev_priv->compressed_llb = compressed_llb; 1110 dev_priv->compressed_llb = compressed_llb;
1145 } 1111 }
1146 1112
1147 DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base, 1113 DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
1148 ll_base, size >> 20); 1114 cfb_base, ll_base, size >> 20);
1115 return;
1116
1117err_llb:
1118 drm_mm_put_block(compressed_llb);
1119err_fb:
1120 drm_mm_put_block(compressed_fb);
1121err:
1122 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
1123 i915_warn_stolen(dev);
1149} 1124}
1150 1125
1151static void i915_cleanup_compression(struct drm_device *dev) 1126static void i915_cleanup_compression(struct drm_device *dev)
@@ -1176,12 +1151,16 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_
1176 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 1151 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
1177 if (state == VGA_SWITCHEROO_ON) { 1152 if (state == VGA_SWITCHEROO_ON) {
1178 printk(KERN_INFO "i915: switched on\n"); 1153 printk(KERN_INFO "i915: switched on\n");
1154 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1179 /* i915 resume handler doesn't set to D0 */ 1155 /* i915 resume handler doesn't set to D0 */
1180 pci_set_power_state(dev->pdev, PCI_D0); 1156 pci_set_power_state(dev->pdev, PCI_D0);
1181 i915_resume(dev); 1157 i915_resume(dev);
1158 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1182 } else { 1159 } else {
1183 printk(KERN_ERR "i915: switched off\n"); 1160 printk(KERN_ERR "i915: switched off\n");
1161 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1184 i915_suspend(dev, pmm); 1162 i915_suspend(dev, pmm);
1163 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1185 } 1164 }
1186} 1165}
1187 1166
@@ -1196,17 +1175,20 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
1196 return can_switch; 1175 return can_switch;
1197} 1176}
1198 1177
1199static int i915_load_modeset_init(struct drm_device *dev, 1178static int i915_load_modeset_init(struct drm_device *dev)
1200 unsigned long prealloc_size,
1201 unsigned long agp_size)
1202{ 1179{
1203 struct drm_i915_private *dev_priv = dev->dev_private; 1180 struct drm_i915_private *dev_priv = dev->dev_private;
1181 unsigned long prealloc_size, gtt_size, mappable_size;
1204 int ret = 0; 1182 int ret = 0;
1205 1183
1206 /* Basic memrange allocator for stolen space (aka mm.vram) */ 1184 prealloc_size = dev_priv->mm.gtt->stolen_size;
1207 drm_mm_init(&dev_priv->mm.vram, 0, prealloc_size); 1185 gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
1186 mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1208 1187
1209 /* Let GEM Manage from end of prealloc space to end of aperture. 1188 /* Basic memrange allocator for stolen space */
1189 drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
1190
1191 /* Let GEM Manage all of the aperture.
1210 * 1192 *
1211 * However, leave one page at the end still bound to the scratch page. 1193 * However, leave one page at the end still bound to the scratch page.
1212 * There are a number of places where the hardware apparently 1194 * There are a number of places where the hardware apparently
@@ -1215,7 +1197,7 @@ static int i915_load_modeset_init(struct drm_device *dev,
1215 * at the last page of the aperture. One page should be enough to 1197 * at the last page of the aperture. One page should be enough to
1216 * keep any prefetching inside of the aperture. 1198 * keep any prefetching inside of the aperture.
1217 */ 1199 */
1218 i915_gem_do_init(dev, prealloc_size, agp_size - 4096); 1200 i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
1219 1201
1220 mutex_lock(&dev->struct_mutex); 1202 mutex_lock(&dev->struct_mutex);
1221 ret = i915_gem_init_ringbuffer(dev); 1203 ret = i915_gem_init_ringbuffer(dev);
@@ -1227,16 +1209,17 @@ static int i915_load_modeset_init(struct drm_device *dev,
1227 if (I915_HAS_FBC(dev) && i915_powersave) { 1209 if (I915_HAS_FBC(dev) && i915_powersave) {
1228 int cfb_size; 1210 int cfb_size;
1229 1211
1230 /* Try to get an 8M buffer... */ 1212 /* Leave 1M for line length buffer & misc. */
1231 if (prealloc_size > (9*1024*1024)) 1213
1232 cfb_size = 8*1024*1024; 1214 /* Try to get a 32M buffer... */
1215 if (prealloc_size > (36*1024*1024))
1216 cfb_size = 32*1024*1024;
1233 else /* fall back to 7/8 of the stolen space */ 1217 else /* fall back to 7/8 of the stolen space */
1234 cfb_size = prealloc_size * 7 / 8; 1218 cfb_size = prealloc_size * 7 / 8;
1235 i915_setup_compression(dev, cfb_size); 1219 i915_setup_compression(dev, cfb_size);
1236 } 1220 }
1237 1221
1238 /* Allow hardware batchbuffers unless told otherwise. 1222 /* Allow hardware batchbuffers unless told otherwise. */
1239 */
1240 dev_priv->allow_batchbuffer = 1; 1223 dev_priv->allow_batchbuffer = 1;
1241 1224
1242 ret = intel_parse_bios(dev); 1225 ret = intel_parse_bios(dev);
@@ -1252,6 +1235,7 @@ static int i915_load_modeset_init(struct drm_device *dev,
1252 1235
1253 ret = vga_switcheroo_register_client(dev->pdev, 1236 ret = vga_switcheroo_register_client(dev->pdev,
1254 i915_switcheroo_set_state, 1237 i915_switcheroo_set_state,
1238 NULL,
1255 i915_switcheroo_can_switch); 1239 i915_switcheroo_can_switch);
1256 if (ret) 1240 if (ret)
1257 goto cleanup_vga_client; 1241 goto cleanup_vga_client;
@@ -1426,152 +1410,12 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
1426 } 1410 }
1427} 1411}
1428 1412
1429struct v_table { 1413static const struct cparams {
1430 u8 vid; 1414 u16 i;
1431 unsigned long vd; /* in .1 mil */ 1415 u16 t;
1432 unsigned long vm; /* in .1 mil */ 1416 u16 m;
1433 u8 pvid; 1417 u16 c;
1434}; 1418} cparams[] = {
1435
1436static struct v_table v_table[] = {
1437 { 0, 16125, 15000, 0x7f, },
1438 { 1, 16000, 14875, 0x7e, },
1439 { 2, 15875, 14750, 0x7d, },
1440 { 3, 15750, 14625, 0x7c, },
1441 { 4, 15625, 14500, 0x7b, },
1442 { 5, 15500, 14375, 0x7a, },
1443 { 6, 15375, 14250, 0x79, },
1444 { 7, 15250, 14125, 0x78, },
1445 { 8, 15125, 14000, 0x77, },
1446 { 9, 15000, 13875, 0x76, },
1447 { 10, 14875, 13750, 0x75, },
1448 { 11, 14750, 13625, 0x74, },
1449 { 12, 14625, 13500, 0x73, },
1450 { 13, 14500, 13375, 0x72, },
1451 { 14, 14375, 13250, 0x71, },
1452 { 15, 14250, 13125, 0x70, },
1453 { 16, 14125, 13000, 0x6f, },
1454 { 17, 14000, 12875, 0x6e, },
1455 { 18, 13875, 12750, 0x6d, },
1456 { 19, 13750, 12625, 0x6c, },
1457 { 20, 13625, 12500, 0x6b, },
1458 { 21, 13500, 12375, 0x6a, },
1459 { 22, 13375, 12250, 0x69, },
1460 { 23, 13250, 12125, 0x68, },
1461 { 24, 13125, 12000, 0x67, },
1462 { 25, 13000, 11875, 0x66, },
1463 { 26, 12875, 11750, 0x65, },
1464 { 27, 12750, 11625, 0x64, },
1465 { 28, 12625, 11500, 0x63, },
1466 { 29, 12500, 11375, 0x62, },
1467 { 30, 12375, 11250, 0x61, },
1468 { 31, 12250, 11125, 0x60, },
1469 { 32, 12125, 11000, 0x5f, },
1470 { 33, 12000, 10875, 0x5e, },
1471 { 34, 11875, 10750, 0x5d, },
1472 { 35, 11750, 10625, 0x5c, },
1473 { 36, 11625, 10500, 0x5b, },
1474 { 37, 11500, 10375, 0x5a, },
1475 { 38, 11375, 10250, 0x59, },
1476 { 39, 11250, 10125, 0x58, },
1477 { 40, 11125, 10000, 0x57, },
1478 { 41, 11000, 9875, 0x56, },
1479 { 42, 10875, 9750, 0x55, },
1480 { 43, 10750, 9625, 0x54, },
1481 { 44, 10625, 9500, 0x53, },
1482 { 45, 10500, 9375, 0x52, },
1483 { 46, 10375, 9250, 0x51, },
1484 { 47, 10250, 9125, 0x50, },
1485 { 48, 10125, 9000, 0x4f, },
1486 { 49, 10000, 8875, 0x4e, },
1487 { 50, 9875, 8750, 0x4d, },
1488 { 51, 9750, 8625, 0x4c, },
1489 { 52, 9625, 8500, 0x4b, },
1490 { 53, 9500, 8375, 0x4a, },
1491 { 54, 9375, 8250, 0x49, },
1492 { 55, 9250, 8125, 0x48, },
1493 { 56, 9125, 8000, 0x47, },
1494 { 57, 9000, 7875, 0x46, },
1495 { 58, 8875, 7750, 0x45, },
1496 { 59, 8750, 7625, 0x44, },
1497 { 60, 8625, 7500, 0x43, },
1498 { 61, 8500, 7375, 0x42, },
1499 { 62, 8375, 7250, 0x41, },
1500 { 63, 8250, 7125, 0x40, },
1501 { 64, 8125, 7000, 0x3f, },
1502 { 65, 8000, 6875, 0x3e, },
1503 { 66, 7875, 6750, 0x3d, },
1504 { 67, 7750, 6625, 0x3c, },
1505 { 68, 7625, 6500, 0x3b, },
1506 { 69, 7500, 6375, 0x3a, },
1507 { 70, 7375, 6250, 0x39, },
1508 { 71, 7250, 6125, 0x38, },
1509 { 72, 7125, 6000, 0x37, },
1510 { 73, 7000, 5875, 0x36, },
1511 { 74, 6875, 5750, 0x35, },
1512 { 75, 6750, 5625, 0x34, },
1513 { 76, 6625, 5500, 0x33, },
1514 { 77, 6500, 5375, 0x32, },
1515 { 78, 6375, 5250, 0x31, },
1516 { 79, 6250, 5125, 0x30, },
1517 { 80, 6125, 5000, 0x2f, },
1518 { 81, 6000, 4875, 0x2e, },
1519 { 82, 5875, 4750, 0x2d, },
1520 { 83, 5750, 4625, 0x2c, },
1521 { 84, 5625, 4500, 0x2b, },
1522 { 85, 5500, 4375, 0x2a, },
1523 { 86, 5375, 4250, 0x29, },
1524 { 87, 5250, 4125, 0x28, },
1525 { 88, 5125, 4000, 0x27, },
1526 { 89, 5000, 3875, 0x26, },
1527 { 90, 4875, 3750, 0x25, },
1528 { 91, 4750, 3625, 0x24, },
1529 { 92, 4625, 3500, 0x23, },
1530 { 93, 4500, 3375, 0x22, },
1531 { 94, 4375, 3250, 0x21, },
1532 { 95, 4250, 3125, 0x20, },
1533 { 96, 4125, 3000, 0x1f, },
1534 { 97, 4125, 3000, 0x1e, },
1535 { 98, 4125, 3000, 0x1d, },
1536 { 99, 4125, 3000, 0x1c, },
1537 { 100, 4125, 3000, 0x1b, },
1538 { 101, 4125, 3000, 0x1a, },
1539 { 102, 4125, 3000, 0x19, },
1540 { 103, 4125, 3000, 0x18, },
1541 { 104, 4125, 3000, 0x17, },
1542 { 105, 4125, 3000, 0x16, },
1543 { 106, 4125, 3000, 0x15, },
1544 { 107, 4125, 3000, 0x14, },
1545 { 108, 4125, 3000, 0x13, },
1546 { 109, 4125, 3000, 0x12, },
1547 { 110, 4125, 3000, 0x11, },
1548 { 111, 4125, 3000, 0x10, },
1549 { 112, 4125, 3000, 0x0f, },
1550 { 113, 4125, 3000, 0x0e, },
1551 { 114, 4125, 3000, 0x0d, },
1552 { 115, 4125, 3000, 0x0c, },
1553 { 116, 4125, 3000, 0x0b, },
1554 { 117, 4125, 3000, 0x0a, },
1555 { 118, 4125, 3000, 0x09, },
1556 { 119, 4125, 3000, 0x08, },
1557 { 120, 1125, 0, 0x07, },
1558 { 121, 1000, 0, 0x06, },
1559 { 122, 875, 0, 0x05, },
1560 { 123, 750, 0, 0x04, },
1561 { 124, 625, 0, 0x03, },
1562 { 125, 500, 0, 0x02, },
1563 { 126, 375, 0, 0x01, },
1564 { 127, 0, 0, 0x00, },
1565};
1566
1567struct cparams {
1568 int i;
1569 int t;
1570 int m;
1571 int c;
1572};
1573
1574static struct cparams cparams[] = {
1575 { 1, 1333, 301, 28664 }, 1419 { 1, 1333, 301, 28664 },
1576 { 1, 1066, 294, 24460 }, 1420 { 1, 1066, 294, 24460 },
1577 { 1, 800, 294, 25192 }, 1421 { 1, 800, 294, 25192 },
@@ -1637,21 +1481,145 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
1637 return ((m * x) / 127) - b; 1481 return ((m * x) / 127) - b;
1638} 1482}
1639 1483
1640static unsigned long pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) 1484static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
1641{ 1485{
1642 unsigned long val = 0; 1486 static const struct v_table {
1643 int i; 1487 u16 vd; /* in .1 mil */
1644 1488 u16 vm; /* in .1 mil */
1645 for (i = 0; i < ARRAY_SIZE(v_table); i++) { 1489 } v_table[] = {
1646 if (v_table[i].pvid == pxvid) { 1490 { 0, 0, },
1647 if (IS_MOBILE(dev_priv->dev)) 1491 { 375, 0, },
1648 val = v_table[i].vm; 1492 { 500, 0, },
1649 else 1493 { 625, 0, },
1650 val = v_table[i].vd; 1494 { 750, 0, },
1651 } 1495 { 875, 0, },
1652 } 1496 { 1000, 0, },
1653 1497 { 1125, 0, },
1654 return val; 1498 { 4125, 3000, },
1499 { 4125, 3000, },
1500 { 4125, 3000, },
1501 { 4125, 3000, },
1502 { 4125, 3000, },
1503 { 4125, 3000, },
1504 { 4125, 3000, },
1505 { 4125, 3000, },
1506 { 4125, 3000, },
1507 { 4125, 3000, },
1508 { 4125, 3000, },
1509 { 4125, 3000, },
1510 { 4125, 3000, },
1511 { 4125, 3000, },
1512 { 4125, 3000, },
1513 { 4125, 3000, },
1514 { 4125, 3000, },
1515 { 4125, 3000, },
1516 { 4125, 3000, },
1517 { 4125, 3000, },
1518 { 4125, 3000, },
1519 { 4125, 3000, },
1520 { 4125, 3000, },
1521 { 4125, 3000, },
1522 { 4250, 3125, },
1523 { 4375, 3250, },
1524 { 4500, 3375, },
1525 { 4625, 3500, },
1526 { 4750, 3625, },
1527 { 4875, 3750, },
1528 { 5000, 3875, },
1529 { 5125, 4000, },
1530 { 5250, 4125, },
1531 { 5375, 4250, },
1532 { 5500, 4375, },
1533 { 5625, 4500, },
1534 { 5750, 4625, },
1535 { 5875, 4750, },
1536 { 6000, 4875, },
1537 { 6125, 5000, },
1538 { 6250, 5125, },
1539 { 6375, 5250, },
1540 { 6500, 5375, },
1541 { 6625, 5500, },
1542 { 6750, 5625, },
1543 { 6875, 5750, },
1544 { 7000, 5875, },
1545 { 7125, 6000, },
1546 { 7250, 6125, },
1547 { 7375, 6250, },
1548 { 7500, 6375, },
1549 { 7625, 6500, },
1550 { 7750, 6625, },
1551 { 7875, 6750, },
1552 { 8000, 6875, },
1553 { 8125, 7000, },
1554 { 8250, 7125, },
1555 { 8375, 7250, },
1556 { 8500, 7375, },
1557 { 8625, 7500, },
1558 { 8750, 7625, },
1559 { 8875, 7750, },
1560 { 9000, 7875, },
1561 { 9125, 8000, },
1562 { 9250, 8125, },
1563 { 9375, 8250, },
1564 { 9500, 8375, },
1565 { 9625, 8500, },
1566 { 9750, 8625, },
1567 { 9875, 8750, },
1568 { 10000, 8875, },
1569 { 10125, 9000, },
1570 { 10250, 9125, },
1571 { 10375, 9250, },
1572 { 10500, 9375, },
1573 { 10625, 9500, },
1574 { 10750, 9625, },
1575 { 10875, 9750, },
1576 { 11000, 9875, },
1577 { 11125, 10000, },
1578 { 11250, 10125, },
1579 { 11375, 10250, },
1580 { 11500, 10375, },
1581 { 11625, 10500, },
1582 { 11750, 10625, },
1583 { 11875, 10750, },
1584 { 12000, 10875, },
1585 { 12125, 11000, },
1586 { 12250, 11125, },
1587 { 12375, 11250, },
1588 { 12500, 11375, },
1589 { 12625, 11500, },
1590 { 12750, 11625, },
1591 { 12875, 11750, },
1592 { 13000, 11875, },
1593 { 13125, 12000, },
1594 { 13250, 12125, },
1595 { 13375, 12250, },
1596 { 13500, 12375, },
1597 { 13625, 12500, },
1598 { 13750, 12625, },
1599 { 13875, 12750, },
1600 { 14000, 12875, },
1601 { 14125, 13000, },
1602 { 14250, 13125, },
1603 { 14375, 13250, },
1604 { 14500, 13375, },
1605 { 14625, 13500, },
1606 { 14750, 13625, },
1607 { 14875, 13750, },
1608 { 15000, 13875, },
1609 { 15125, 14000, },
1610 { 15250, 14125, },
1611 { 15375, 14250, },
1612 { 15500, 14375, },
1613 { 15625, 14500, },
1614 { 15750, 14625, },
1615 { 15875, 14750, },
1616 { 16000, 14875, },
1617 { 16125, 15000, },
1618 };
1619 if (dev_priv->info->is_mobile)
1620 return v_table[pxvid].vm;
1621 else
1622 return v_table[pxvid].vd;
1655} 1623}
1656 1624
1657void i915_update_gfx_val(struct drm_i915_private *dev_priv) 1625void i915_update_gfx_val(struct drm_i915_private *dev_priv)
@@ -1905,9 +1873,9 @@ ips_ping_for_i915_load(void)
1905int i915_driver_load(struct drm_device *dev, unsigned long flags) 1873int i915_driver_load(struct drm_device *dev, unsigned long flags)
1906{ 1874{
1907 struct drm_i915_private *dev_priv; 1875 struct drm_i915_private *dev_priv;
1908 resource_size_t base, size;
1909 int ret = 0, mmio_bar; 1876 int ret = 0, mmio_bar;
1910 uint32_t agp_size, prealloc_size; 1877 uint32_t agp_size;
1878
1911 /* i915 has 4 more counters */ 1879 /* i915 has 4 more counters */
1912 dev->counters += 4; 1880 dev->counters += 4;
1913 dev->types[6] = _DRM_STAT_IRQ; 1881 dev->types[6] = _DRM_STAT_IRQ;
@@ -1923,11 +1891,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1923 dev_priv->dev = dev; 1891 dev_priv->dev = dev;
1924 dev_priv->info = (struct intel_device_info *) flags; 1892 dev_priv->info = (struct intel_device_info *) flags;
1925 1893
1926 /* Add register map (needed for suspend/resume) */
1927 mmio_bar = IS_GEN2(dev) ? 1 : 0;
1928 base = pci_resource_start(dev->pdev, mmio_bar);
1929 size = pci_resource_len(dev->pdev, mmio_bar);
1930
1931 if (i915_get_bridge_dev(dev)) { 1894 if (i915_get_bridge_dev(dev)) {
1932 ret = -EIO; 1895 ret = -EIO;
1933 goto free_priv; 1896 goto free_priv;
@@ -1937,16 +1900,25 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1937 if (IS_GEN2(dev)) 1900 if (IS_GEN2(dev))
1938 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30)); 1901 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
1939 1902
1940 dev_priv->regs = ioremap(base, size); 1903 mmio_bar = IS_GEN2(dev) ? 1 : 0;
1904 dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
1941 if (!dev_priv->regs) { 1905 if (!dev_priv->regs) {
1942 DRM_ERROR("failed to map registers\n"); 1906 DRM_ERROR("failed to map registers\n");
1943 ret = -EIO; 1907 ret = -EIO;
1944 goto put_bridge; 1908 goto put_bridge;
1945 } 1909 }
1946 1910
1911 dev_priv->mm.gtt = intel_gtt_get();
1912 if (!dev_priv->mm.gtt) {
1913 DRM_ERROR("Failed to initialize GTT\n");
1914 ret = -ENODEV;
1915 goto out_iomapfree;
1916 }
1917
1918 agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1919
1947 dev_priv->mm.gtt_mapping = 1920 dev_priv->mm.gtt_mapping =
1948 io_mapping_create_wc(dev->agp->base, 1921 io_mapping_create_wc(dev->agp->base, agp_size);
1949 dev->agp->agp_info.aper_size * 1024*1024);
1950 if (dev_priv->mm.gtt_mapping == NULL) { 1922 if (dev_priv->mm.gtt_mapping == NULL) {
1951 ret = -EIO; 1923 ret = -EIO;
1952 goto out_rmmap; 1924 goto out_rmmap;
@@ -1958,24 +1930,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1958 * MTRR if present. Even if a UC MTRR isn't present. 1930 * MTRR if present. Even if a UC MTRR isn't present.
1959 */ 1931 */
1960 dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base, 1932 dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
1961 dev->agp->agp_info.aper_size * 1933 agp_size,
1962 1024 * 1024,
1963 MTRR_TYPE_WRCOMB, 1); 1934 MTRR_TYPE_WRCOMB, 1);
1964 if (dev_priv->mm.gtt_mtrr < 0) { 1935 if (dev_priv->mm.gtt_mtrr < 0) {
1965 DRM_INFO("MTRR allocation failed. Graphics " 1936 DRM_INFO("MTRR allocation failed. Graphics "
1966 "performance may suffer.\n"); 1937 "performance may suffer.\n");
1967 } 1938 }
1968 1939
1969 dev_priv->mm.gtt = intel_gtt_get();
1970 if (!dev_priv->mm.gtt) {
1971 DRM_ERROR("Failed to initialize GTT\n");
1972 ret = -ENODEV;
1973 goto out_iomapfree;
1974 }
1975
1976 prealloc_size = dev_priv->mm.gtt->gtt_stolen_entries << PAGE_SHIFT;
1977 agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
1978
1979 /* The i915 workqueue is primarily used for batched retirement of 1940 /* The i915 workqueue is primarily used for batched retirement of
1980 * requests (and thus managing bo) once the task has been completed 1941 * requests (and thus managing bo) once the task has been completed
1981 * by the GPU. i915_gem_retire_requests() is called directly when we 1942 * by the GPU. i915_gem_retire_requests() is called directly when we
@@ -1983,7 +1944,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
1983 * bo. 1944 * bo.
1984 * 1945 *
1985 * It is also used for periodic low-priority events, such as 1946 * It is also used for periodic low-priority events, such as
1986 * idle-timers and hangcheck. 1947 * idle-timers and recording error state.
1987 * 1948 *
1988 * All tasks on the workqueue are expected to acquire the dev mutex 1949 * All tasks on the workqueue are expected to acquire the dev mutex
1989 * so there is no point in running more than one instance of the 1950 * so there is no point in running more than one instance of the
@@ -2001,20 +1962,11 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2001 /* enable GEM by default */ 1962 /* enable GEM by default */
2002 dev_priv->has_gem = 1; 1963 dev_priv->has_gem = 1;
2003 1964
2004 if (prealloc_size > agp_size * 3 / 4) {
2005 DRM_ERROR("Detected broken video BIOS with %d/%dkB of video "
2006 "memory stolen.\n",
2007 prealloc_size / 1024, agp_size / 1024);
2008 DRM_ERROR("Disabling GEM. (try reducing stolen memory or "
2009 "updating the BIOS to fix).\n");
2010 dev_priv->has_gem = 0;
2011 }
2012
2013 if (dev_priv->has_gem == 0 && 1965 if (dev_priv->has_gem == 0 &&
2014 drm_core_check_feature(dev, DRIVER_MODESET)) { 1966 drm_core_check_feature(dev, DRIVER_MODESET)) {
2015 DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n"); 1967 DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n");
2016 ret = -ENODEV; 1968 ret = -ENODEV;
2017 goto out_iomapfree; 1969 goto out_workqueue_free;
2018 } 1970 }
2019 1971
2020 dev->driver->get_vblank_counter = i915_get_vblank_counter; 1972 dev->driver->get_vblank_counter = i915_get_vblank_counter;
@@ -2037,8 +1989,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2037 /* Init HWS */ 1989 /* Init HWS */
2038 if (!I915_NEED_GFX_HWS(dev)) { 1990 if (!I915_NEED_GFX_HWS(dev)) {
2039 ret = i915_init_phys_hws(dev); 1991 ret = i915_init_phys_hws(dev);
2040 if (ret != 0) 1992 if (ret)
2041 goto out_workqueue_free; 1993 goto out_gem_unload;
2042 } 1994 }
2043 1995
2044 if (IS_PINEVIEW(dev)) 1996 if (IS_PINEVIEW(dev))
@@ -2060,16 +2012,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2060 if (!IS_I945G(dev) && !IS_I945GM(dev)) 2012 if (!IS_I945G(dev) && !IS_I945GM(dev))
2061 pci_enable_msi(dev->pdev); 2013 pci_enable_msi(dev->pdev);
2062 2014
2063 spin_lock_init(&dev_priv->user_irq_lock); 2015 spin_lock_init(&dev_priv->irq_lock);
2064 spin_lock_init(&dev_priv->error_lock); 2016 spin_lock_init(&dev_priv->error_lock);
2065 dev_priv->trace_irq_seqno = 0; 2017 dev_priv->trace_irq_seqno = 0;
2066 2018
2067 ret = drm_vblank_init(dev, I915_NUM_PIPE); 2019 ret = drm_vblank_init(dev, I915_NUM_PIPE);
2068 2020 if (ret)
2069 if (ret) { 2021 goto out_gem_unload;
2070 (void) i915_driver_unload(dev);
2071 return ret;
2072 }
2073 2022
2074 /* Start out suspended */ 2023 /* Start out suspended */
2075 dev_priv->mm.suspended = 1; 2024 dev_priv->mm.suspended = 1;
@@ -2077,10 +2026,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2077 intel_detect_pch(dev); 2026 intel_detect_pch(dev);
2078 2027
2079 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 2028 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
2080 ret = i915_load_modeset_init(dev, prealloc_size, agp_size); 2029 ret = i915_load_modeset_init(dev);
2081 if (ret < 0) { 2030 if (ret < 0) {
2082 DRM_ERROR("failed to init modeset\n"); 2031 DRM_ERROR("failed to init modeset\n");
2083 goto out_workqueue_free; 2032 goto out_gem_unload;
2084 } 2033 }
2085 } 2034 }
2086 2035
@@ -2100,12 +2049,18 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
2100 2049
2101 return 0; 2050 return 0;
2102 2051
2052out_gem_unload:
2053 if (dev->pdev->msi_enabled)
2054 pci_disable_msi(dev->pdev);
2055
2056 intel_teardown_gmbus(dev);
2057 intel_teardown_mchbar(dev);
2103out_workqueue_free: 2058out_workqueue_free:
2104 destroy_workqueue(dev_priv->wq); 2059 destroy_workqueue(dev_priv->wq);
2105out_iomapfree: 2060out_iomapfree:
2106 io_mapping_free(dev_priv->mm.gtt_mapping); 2061 io_mapping_free(dev_priv->mm.gtt_mapping);
2107out_rmmap: 2062out_rmmap:
2108 iounmap(dev_priv->regs); 2063 pci_iounmap(dev->pdev, dev_priv->regs);
2109put_bridge: 2064put_bridge:
2110 pci_dev_put(dev_priv->bridge_dev); 2065 pci_dev_put(dev_priv->bridge_dev);
2111free_priv: 2066free_priv:
@@ -2122,6 +2077,9 @@ int i915_driver_unload(struct drm_device *dev)
2122 i915_mch_dev = NULL; 2077 i915_mch_dev = NULL;
2123 spin_unlock(&mchdev_lock); 2078 spin_unlock(&mchdev_lock);
2124 2079
2080 if (dev_priv->mm.inactive_shrinker.shrink)
2081 unregister_shrinker(&dev_priv->mm.inactive_shrinker);
2082
2125 mutex_lock(&dev->struct_mutex); 2083 mutex_lock(&dev->struct_mutex);
2126 ret = i915_gpu_idle(dev); 2084 ret = i915_gpu_idle(dev);
2127 if (ret) 2085 if (ret)
@@ -2179,7 +2137,7 @@ int i915_driver_unload(struct drm_device *dev)
2179 mutex_unlock(&dev->struct_mutex); 2137 mutex_unlock(&dev->struct_mutex);
2180 if (I915_HAS_FBC(dev) && i915_powersave) 2138 if (I915_HAS_FBC(dev) && i915_powersave)
2181 i915_cleanup_compression(dev); 2139 i915_cleanup_compression(dev);
2182 drm_mm_takedown(&dev_priv->mm.vram); 2140 drm_mm_takedown(&dev_priv->mm.stolen);
2183 2141
2184 intel_cleanup_overlay(dev); 2142 intel_cleanup_overlay(dev);
2185 2143
@@ -2188,7 +2146,7 @@ int i915_driver_unload(struct drm_device *dev)
2188 } 2146 }
2189 2147
2190 if (dev_priv->regs != NULL) 2148 if (dev_priv->regs != NULL)
2191 iounmap(dev_priv->regs); 2149 pci_iounmap(dev->pdev, dev_priv->regs);
2192 2150
2193 intel_teardown_gmbus(dev); 2151 intel_teardown_gmbus(dev);
2194 intel_teardown_mchbar(dev); 2152 intel_teardown_mchbar(dev);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index f737960712e6..872493331988 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -111,7 +111,7 @@ static const struct intel_device_info intel_i965g_info = {
111 111
112static const struct intel_device_info intel_i965gm_info = { 112static const struct intel_device_info intel_i965gm_info = {
113 .gen = 4, .is_crestline = 1, 113 .gen = 4, .is_crestline = 1,
114 .is_mobile = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1, 114 .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
115 .has_overlay = 1, 115 .has_overlay = 1,
116 .supports_tv = 1, 116 .supports_tv = 1,
117}; 117};
@@ -130,7 +130,7 @@ static const struct intel_device_info intel_g45_info = {
130 130
131static const struct intel_device_info intel_gm45_info = { 131static const struct intel_device_info intel_gm45_info = {
132 .gen = 4, .is_g4x = 1, 132 .gen = 4, .is_g4x = 1,
133 .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1, 133 .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
134 .has_pipe_cxsr = 1, .has_hotplug = 1, 134 .has_pipe_cxsr = 1, .has_hotplug = 1,
135 .supports_tv = 1, 135 .supports_tv = 1,
136 .has_bsd_ring = 1, 136 .has_bsd_ring = 1,
@@ -150,7 +150,7 @@ static const struct intel_device_info intel_ironlake_d_info = {
150 150
151static const struct intel_device_info intel_ironlake_m_info = { 151static const struct intel_device_info intel_ironlake_m_info = {
152 .gen = 5, .is_mobile = 1, 152 .gen = 5, .is_mobile = 1,
153 .need_gfx_hws = 1, .has_rc6 = 1, .has_hotplug = 1, 153 .need_gfx_hws = 1, .has_hotplug = 1,
154 .has_fbc = 0, /* disabled due to buggy hardware */ 154 .has_fbc = 0, /* disabled due to buggy hardware */
155 .has_bsd_ring = 1, 155 .has_bsd_ring = 1,
156}; 156};
@@ -165,6 +165,7 @@ static const struct intel_device_info intel_sandybridge_d_info = {
165static const struct intel_device_info intel_sandybridge_m_info = { 165static const struct intel_device_info intel_sandybridge_m_info = {
166 .gen = 6, .is_mobile = 1, 166 .gen = 6, .is_mobile = 1,
167 .need_gfx_hws = 1, .has_hotplug = 1, 167 .need_gfx_hws = 1, .has_hotplug = 1,
168 .has_fbc = 1,
168 .has_bsd_ring = 1, 169 .has_bsd_ring = 1,
169 .has_blt_ring = 1, 170 .has_blt_ring = 1,
170}; 171};
@@ -244,10 +245,34 @@ void intel_detect_pch (struct drm_device *dev)
244 } 245 }
245} 246}
246 247
248void __gen6_force_wake_get(struct drm_i915_private *dev_priv)
249{
250 int count;
251
252 count = 0;
253 while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1))
254 udelay(10);
255
256 I915_WRITE_NOTRACE(FORCEWAKE, 1);
257 POSTING_READ(FORCEWAKE);
258
259 count = 0;
260 while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0)
261 udelay(10);
262}
263
264void __gen6_force_wake_put(struct drm_i915_private *dev_priv)
265{
266 I915_WRITE_NOTRACE(FORCEWAKE, 0);
267 POSTING_READ(FORCEWAKE);
268}
269
247static int i915_drm_freeze(struct drm_device *dev) 270static int i915_drm_freeze(struct drm_device *dev)
248{ 271{
249 struct drm_i915_private *dev_priv = dev->dev_private; 272 struct drm_i915_private *dev_priv = dev->dev_private;
250 273
274 drm_kms_helper_poll_disable(dev);
275
251 pci_save_state(dev->pdev); 276 pci_save_state(dev->pdev);
252 277
253 /* If KMS is active, we do the leavevt stuff here */ 278 /* If KMS is active, we do the leavevt stuff here */
@@ -284,7 +309,9 @@ int i915_suspend(struct drm_device *dev, pm_message_t state)
284 if (state.event == PM_EVENT_PRETHAW) 309 if (state.event == PM_EVENT_PRETHAW)
285 return 0; 310 return 0;
286 311
287 drm_kms_helper_poll_disable(dev); 312
313 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
314 return 0;
288 315
289 error = i915_drm_freeze(dev); 316 error = i915_drm_freeze(dev);
290 if (error) 317 if (error)
@@ -304,6 +331,12 @@ static int i915_drm_thaw(struct drm_device *dev)
304 struct drm_i915_private *dev_priv = dev->dev_private; 331 struct drm_i915_private *dev_priv = dev->dev_private;
305 int error = 0; 332 int error = 0;
306 333
334 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
335 mutex_lock(&dev->struct_mutex);
336 i915_gem_restore_gtt_mappings(dev);
337 mutex_unlock(&dev->struct_mutex);
338 }
339
307 i915_restore_state(dev); 340 i915_restore_state(dev);
308 intel_opregion_setup(dev); 341 intel_opregion_setup(dev);
309 342
@@ -332,6 +365,9 @@ int i915_resume(struct drm_device *dev)
332{ 365{
333 int ret; 366 int ret;
334 367
368 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
369 return 0;
370
335 if (pci_enable_device(dev->pdev)) 371 if (pci_enable_device(dev->pdev))
336 return -EIO; 372 return -EIO;
337 373
@@ -405,6 +441,14 @@ static int ironlake_do_reset(struct drm_device *dev, u8 flags)
405 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 441 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
406} 442}
407 443
444static int gen6_do_reset(struct drm_device *dev, u8 flags)
445{
446 struct drm_i915_private *dev_priv = dev->dev_private;
447
448 I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL);
449 return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
450}
451
408/** 452/**
409 * i965_reset - reset chip after a hang 453 * i965_reset - reset chip after a hang
410 * @dev: drm device to reset 454 * @dev: drm device to reset
@@ -431,7 +475,8 @@ int i915_reset(struct drm_device *dev, u8 flags)
431 bool need_display = true; 475 bool need_display = true;
432 int ret; 476 int ret;
433 477
434 mutex_lock(&dev->struct_mutex); 478 if (!mutex_trylock(&dev->struct_mutex))
479 return -EBUSY;
435 480
436 i915_gem_reset(dev); 481 i915_gem_reset(dev);
437 482
@@ -439,6 +484,9 @@ int i915_reset(struct drm_device *dev, u8 flags)
439 if (get_seconds() - dev_priv->last_gpu_reset < 5) { 484 if (get_seconds() - dev_priv->last_gpu_reset < 5) {
440 DRM_ERROR("GPU hanging too fast, declaring wedged!\n"); 485 DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
441 } else switch (INTEL_INFO(dev)->gen) { 486 } else switch (INTEL_INFO(dev)->gen) {
487 case 6:
488 ret = gen6_do_reset(dev, flags);
489 break;
442 case 5: 490 case 5:
443 ret = ironlake_do_reset(dev, flags); 491 ret = ironlake_do_reset(dev, flags);
444 break; 492 break;
@@ -472,9 +520,14 @@ int i915_reset(struct drm_device *dev, u8 flags)
472 */ 520 */
473 if (drm_core_check_feature(dev, DRIVER_MODESET) || 521 if (drm_core_check_feature(dev, DRIVER_MODESET) ||
474 !dev_priv->mm.suspended) { 522 !dev_priv->mm.suspended) {
475 struct intel_ring_buffer *ring = &dev_priv->render_ring;
476 dev_priv->mm.suspended = 0; 523 dev_priv->mm.suspended = 0;
477 ring->init(dev, ring); 524
525 dev_priv->ring[RCS].init(&dev_priv->ring[RCS]);
526 if (HAS_BSD(dev))
527 dev_priv->ring[VCS].init(&dev_priv->ring[VCS]);
528 if (HAS_BLT(dev))
529 dev_priv->ring[BCS].init(&dev_priv->ring[BCS]);
530
478 mutex_unlock(&dev->struct_mutex); 531 mutex_unlock(&dev->struct_mutex);
479 drm_irq_uninstall(dev); 532 drm_irq_uninstall(dev);
480 drm_irq_install(dev); 533 drm_irq_install(dev);
@@ -523,6 +576,9 @@ static int i915_pm_suspend(struct device *dev)
523 return -ENODEV; 576 return -ENODEV;
524 } 577 }
525 578
579 if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
580 return 0;
581
526 error = i915_drm_freeze(drm_dev); 582 error = i915_drm_freeze(drm_dev);
527 if (error) 583 if (error)
528 return error; 584 return error;
@@ -606,6 +662,8 @@ static struct drm_driver driver = {
606 .device_is_agp = i915_driver_device_is_agp, 662 .device_is_agp = i915_driver_device_is_agp,
607 .enable_vblank = i915_enable_vblank, 663 .enable_vblank = i915_enable_vblank,
608 .disable_vblank = i915_disable_vblank, 664 .disable_vblank = i915_disable_vblank,
665 .get_vblank_timestamp = i915_get_vblank_timestamp,
666 .get_scanout_position = i915_get_crtc_scanoutpos,
609 .irq_preinstall = i915_driver_irq_preinstall, 667 .irq_preinstall = i915_driver_irq_preinstall,
610 .irq_postinstall = i915_driver_irq_postinstall, 668 .irq_postinstall = i915_driver_irq_postinstall,
611 .irq_uninstall = i915_driver_irq_uninstall, 669 .irq_uninstall = i915_driver_irq_uninstall,
@@ -661,8 +719,6 @@ static int __init i915_init(void)
661 719
662 driver.num_ioctls = i915_max_ioctl; 720 driver.num_ioctls = i915_max_ioctl;
663 721
664 i915_gem_shrinker_init();
665
666 /* 722 /*
667 * If CONFIG_DRM_I915_KMS is set, default to KMS unless 723 * If CONFIG_DRM_I915_KMS is set, default to KMS unless
668 * explicitly disabled with the module pararmeter. 724 * explicitly disabled with the module pararmeter.
@@ -684,17 +740,11 @@ static int __init i915_init(void)
684 driver.driver_features &= ~DRIVER_MODESET; 740 driver.driver_features &= ~DRIVER_MODESET;
685#endif 741#endif
686 742
687 if (!(driver.driver_features & DRIVER_MODESET)) {
688 driver.suspend = i915_suspend;
689 driver.resume = i915_resume;
690 }
691
692 return drm_init(&driver); 743 return drm_init(&driver);
693} 744}
694 745
695static void __exit i915_exit(void) 746static void __exit i915_exit(void)
696{ 747{
697 i915_gem_shrinker_exit();
698 drm_exit(&driver); 748 drm_exit(&driver);
699} 749}
700 750
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 409826da3099..aac1bf332f75 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -89,7 +89,7 @@ struct drm_i915_gem_phys_object {
89 int id; 89 int id;
90 struct page **page_list; 90 struct page **page_list;
91 drm_dma_handle_t *handle; 91 drm_dma_handle_t *handle;
92 struct drm_gem_object *cur_obj; 92 struct drm_i915_gem_object *cur_obj;
93}; 93};
94 94
95struct mem_block { 95struct mem_block {
@@ -124,9 +124,9 @@ struct drm_i915_master_private {
124#define I915_FENCE_REG_NONE -1 124#define I915_FENCE_REG_NONE -1
125 125
126struct drm_i915_fence_reg { 126struct drm_i915_fence_reg {
127 struct drm_gem_object *obj;
128 struct list_head lru_list; 127 struct list_head lru_list;
129 bool gpu; 128 struct drm_i915_gem_object *obj;
129 uint32_t setup_seqno;
130}; 130};
131 131
132struct sdvo_device_mapping { 132struct sdvo_device_mapping {
@@ -139,6 +139,8 @@ struct sdvo_device_mapping {
139 u8 ddc_pin; 139 u8 ddc_pin;
140}; 140};
141 141
142struct intel_display_error_state;
143
142struct drm_i915_error_state { 144struct drm_i915_error_state {
143 u32 eir; 145 u32 eir;
144 u32 pgtbl_er; 146 u32 pgtbl_er;
@@ -148,11 +150,23 @@ struct drm_i915_error_state {
148 u32 ipehr; 150 u32 ipehr;
149 u32 instdone; 151 u32 instdone;
150 u32 acthd; 152 u32 acthd;
153 u32 error; /* gen6+ */
154 u32 bcs_acthd; /* gen6+ blt engine */
155 u32 bcs_ipehr;
156 u32 bcs_ipeir;
157 u32 bcs_instdone;
158 u32 bcs_seqno;
159 u32 vcs_acthd; /* gen6+ bsd engine */
160 u32 vcs_ipehr;
161 u32 vcs_ipeir;
162 u32 vcs_instdone;
163 u32 vcs_seqno;
151 u32 instpm; 164 u32 instpm;
152 u32 instps; 165 u32 instps;
153 u32 instdone1; 166 u32 instdone1;
154 u32 seqno; 167 u32 seqno;
155 u64 bbaddr; 168 u64 bbaddr;
169 u64 fence[16];
156 struct timeval time; 170 struct timeval time;
157 struct drm_i915_error_object { 171 struct drm_i915_error_object {
158 int page_count; 172 int page_count;
@@ -171,9 +185,11 @@ struct drm_i915_error_state {
171 u32 tiling:2; 185 u32 tiling:2;
172 u32 dirty:1; 186 u32 dirty:1;
173 u32 purgeable:1; 187 u32 purgeable:1;
174 } *active_bo; 188 u32 ring:4;
175 u32 active_bo_count; 189 } *active_bo, *pinned_bo;
190 u32 active_bo_count, pinned_bo_count;
176 struct intel_overlay_error_state *overlay; 191 struct intel_overlay_error_state *overlay;
192 struct intel_display_error_state *display;
177}; 193};
178 194
179struct drm_i915_display_funcs { 195struct drm_i915_display_funcs {
@@ -207,7 +223,6 @@ struct intel_device_info {
207 u8 is_broadwater : 1; 223 u8 is_broadwater : 1;
208 u8 is_crestline : 1; 224 u8 is_crestline : 1;
209 u8 has_fbc : 1; 225 u8 has_fbc : 1;
210 u8 has_rc6 : 1;
211 u8 has_pipe_cxsr : 1; 226 u8 has_pipe_cxsr : 1;
212 u8 has_hotplug : 1; 227 u8 has_hotplug : 1;
213 u8 cursor_needs_physical : 1; 228 u8 cursor_needs_physical : 1;
@@ -243,6 +258,7 @@ typedef struct drm_i915_private {
243 const struct intel_device_info *info; 258 const struct intel_device_info *info;
244 259
245 int has_gem; 260 int has_gem;
261 int relative_constants_mode;
246 262
247 void __iomem *regs; 263 void __iomem *regs;
248 264
@@ -253,20 +269,15 @@ typedef struct drm_i915_private {
253 } *gmbus; 269 } *gmbus;
254 270
255 struct pci_dev *bridge_dev; 271 struct pci_dev *bridge_dev;
256 struct intel_ring_buffer render_ring; 272 struct intel_ring_buffer ring[I915_NUM_RINGS];
257 struct intel_ring_buffer bsd_ring;
258 struct intel_ring_buffer blt_ring;
259 uint32_t next_seqno; 273 uint32_t next_seqno;
260 274
261 drm_dma_handle_t *status_page_dmah; 275 drm_dma_handle_t *status_page_dmah;
262 void *seqno_page;
263 dma_addr_t dma_status_page; 276 dma_addr_t dma_status_page;
264 uint32_t counter; 277 uint32_t counter;
265 unsigned int seqno_gfx_addr;
266 drm_local_map_t hws_map; 278 drm_local_map_t hws_map;
267 struct drm_gem_object *seqno_obj; 279 struct drm_i915_gem_object *pwrctx;
268 struct drm_gem_object *pwrctx; 280 struct drm_i915_gem_object *renderctx;
269 struct drm_gem_object *renderctx;
270 281
271 struct resource mch_res; 282 struct resource mch_res;
272 283
@@ -275,25 +286,17 @@ typedef struct drm_i915_private {
275 int front_offset; 286 int front_offset;
276 int current_page; 287 int current_page;
277 int page_flipping; 288 int page_flipping;
278#define I915_DEBUG_READ (1<<0)
279#define I915_DEBUG_WRITE (1<<1)
280 unsigned long debug_flags;
281 289
282 wait_queue_head_t irq_queue;
283 atomic_t irq_received; 290 atomic_t irq_received;
284 /** Protects user_irq_refcount and irq_mask_reg */
285 spinlock_t user_irq_lock;
286 u32 trace_irq_seqno; 291 u32 trace_irq_seqno;
292
293 /* protects the irq masks */
294 spinlock_t irq_lock;
287 /** Cached value of IMR to avoid reads in updating the bitfield */ 295 /** Cached value of IMR to avoid reads in updating the bitfield */
288 u32 irq_mask_reg;
289 u32 pipestat[2]; 296 u32 pipestat[2];
290 /** splitted irq regs for graphics and display engine on Ironlake, 297 u32 irq_mask;
291 irq_mask_reg is still used for display irq. */ 298 u32 gt_irq_mask;
292 u32 gt_irq_mask_reg; 299 u32 pch_irq_mask;
293 u32 gt_irq_enable_reg;
294 u32 de_irq_enable_reg;
295 u32 pch_irq_mask_reg;
296 u32 pch_irq_enable_reg;
297 300
298 u32 hotplug_supported_mask; 301 u32 hotplug_supported_mask;
299 struct work_struct hotplug_work; 302 struct work_struct hotplug_work;
@@ -306,7 +309,7 @@ typedef struct drm_i915_private {
306 int num_pipe; 309 int num_pipe;
307 310
308 /* For hangcheck timer */ 311 /* For hangcheck timer */
309#define DRM_I915_HANGCHECK_PERIOD 250 /* in ms */ 312#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
310 struct timer_list hangcheck_timer; 313 struct timer_list hangcheck_timer;
311 int hangcheck_count; 314 int hangcheck_count;
312 uint32_t last_acthd; 315 uint32_t last_acthd;
@@ -530,23 +533,21 @@ typedef struct drm_i915_private {
530 533
531 struct { 534 struct {
532 /** Bridge to intel-gtt-ko */ 535 /** Bridge to intel-gtt-ko */
533 struct intel_gtt *gtt; 536 const struct intel_gtt *gtt;
534 /** Memory allocator for GTT stolen memory */ 537 /** Memory allocator for GTT stolen memory */
535 struct drm_mm vram; 538 struct drm_mm stolen;
536 /** Memory allocator for GTT */ 539 /** Memory allocator for GTT */
537 struct drm_mm gtt_space; 540 struct drm_mm gtt_space;
541 /** List of all objects in gtt_space. Used to restore gtt
542 * mappings on resume */
543 struct list_head gtt_list;
544 /** End of mappable part of GTT */
545 unsigned long gtt_mappable_end;
538 546
539 struct io_mapping *gtt_mapping; 547 struct io_mapping *gtt_mapping;
540 int gtt_mtrr; 548 int gtt_mtrr;
541 549
542 /** 550 struct shrinker inactive_shrinker;
543 * Membership on list of all loaded devices, used to evict
544 * inactive buffers under memory pressure.
545 *
546 * Modifications should only be done whilst holding the
547 * shrink_list_lock spinlock.
548 */
549 struct list_head shrink_list;
550 551
551 /** 552 /**
552 * List of objects currently involved in rendering. 553 * List of objects currently involved in rendering.
@@ -609,16 +610,6 @@ typedef struct drm_i915_private {
609 struct delayed_work retire_work; 610 struct delayed_work retire_work;
610 611
611 /** 612 /**
612 * Waiting sequence number, if any
613 */
614 uint32_t waiting_gem_seqno;
615
616 /**
617 * Last seq seen at irq time
618 */
619 uint32_t irq_gem_seqno;
620
621 /**
622 * Flag if the X Server, and thus DRM, is not currently in 613 * Flag if the X Server, and thus DRM, is not currently in
623 * control of the device. 614 * control of the device.
624 * 615 *
@@ -645,16 +636,11 @@ typedef struct drm_i915_private {
645 /* storage for physical objects */ 636 /* storage for physical objects */
646 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; 637 struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT];
647 638
648 uint32_t flush_rings;
649
650 /* accounting, useful for userland debugging */ 639 /* accounting, useful for userland debugging */
651 size_t object_memory;
652 size_t pin_memory;
653 size_t gtt_memory;
654 size_t gtt_total; 640 size_t gtt_total;
641 size_t mappable_gtt_total;
642 size_t object_memory;
655 u32 object_count; 643 u32 object_count;
656 u32 pin_count;
657 u32 gtt_count;
658 } mm; 644 } mm;
659 struct sdvo_device_mapping sdvo_mappings[2]; 645 struct sdvo_device_mapping sdvo_mappings[2];
660 /* indicate whether the LVDS_BORDER should be enabled or not */ 646 /* indicate whether the LVDS_BORDER should be enabled or not */
@@ -688,14 +674,14 @@ typedef struct drm_i915_private {
688 u8 fmax; 674 u8 fmax;
689 u8 fstart; 675 u8 fstart;
690 676
691 u64 last_count1; 677 u64 last_count1;
692 unsigned long last_time1; 678 unsigned long last_time1;
693 u64 last_count2; 679 u64 last_count2;
694 struct timespec last_time2; 680 struct timespec last_time2;
695 unsigned long gfx_power; 681 unsigned long gfx_power;
696 int c_m; 682 int c_m;
697 int r_t; 683 int r_t;
698 u8 corr; 684 u8 corr;
699 spinlock_t *mchdev_lock; 685 spinlock_t *mchdev_lock;
700 686
701 enum no_fbc_reason no_fbc_reason; 687 enum no_fbc_reason no_fbc_reason;
@@ -709,20 +695,20 @@ typedef struct drm_i915_private {
709 struct intel_fbdev *fbdev; 695 struct intel_fbdev *fbdev;
710} drm_i915_private_t; 696} drm_i915_private_t;
711 697
712/** driver private structure attached to each drm_gem_object */
713struct drm_i915_gem_object { 698struct drm_i915_gem_object {
714 struct drm_gem_object base; 699 struct drm_gem_object base;
715 700
716 /** Current space allocated to this object in the GTT, if any. */ 701 /** Current space allocated to this object in the GTT, if any. */
717 struct drm_mm_node *gtt_space; 702 struct drm_mm_node *gtt_space;
703 struct list_head gtt_list;
718 704
719 /** This object's place on the active/flushing/inactive lists */ 705 /** This object's place on the active/flushing/inactive lists */
720 struct list_head ring_list; 706 struct list_head ring_list;
721 struct list_head mm_list; 707 struct list_head mm_list;
722 /** This object's place on GPU write list */ 708 /** This object's place on GPU write list */
723 struct list_head gpu_write_list; 709 struct list_head gpu_write_list;
724 /** This object's place on eviction list */ 710 /** This object's place in the batchbuffer or on the eviction list */
725 struct list_head evict_list; 711 struct list_head exec_list;
726 712
727 /** 713 /**
728 * This is set if the object is on the active or flushing lists 714 * This is set if the object is on the active or flushing lists
@@ -738,6 +724,12 @@ struct drm_i915_gem_object {
738 unsigned int dirty : 1; 724 unsigned int dirty : 1;
739 725
740 /** 726 /**
727 * This is set if the object has been written to since the last
728 * GPU flush.
729 */
730 unsigned int pending_gpu_write : 1;
731
732 /**
741 * Fence register bits (if any) for this object. Will be set 733 * Fence register bits (if any) for this object. Will be set
742 * as needed when mapped into the GTT. 734 * as needed when mapped into the GTT.
743 * Protected by dev->struct_mutex. 735 * Protected by dev->struct_mutex.
@@ -747,29 +739,15 @@ struct drm_i915_gem_object {
747 signed int fence_reg : 5; 739 signed int fence_reg : 5;
748 740
749 /** 741 /**
750 * Used for checking the object doesn't appear more than once
751 * in an execbuffer object list.
752 */
753 unsigned int in_execbuffer : 1;
754
755 /**
756 * Advice: are the backing pages purgeable? 742 * Advice: are the backing pages purgeable?
757 */ 743 */
758 unsigned int madv : 2; 744 unsigned int madv : 2;
759 745
760 /** 746 /**
761 * Refcount for the pages array. With the current locking scheme, there
762 * are at most two concurrent users: Binding a bo to the gtt and
763 * pwrite/pread using physical addresses. So two bits for a maximum
764 * of two users are enough.
765 */
766 unsigned int pages_refcount : 2;
767#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3
768
769 /**
770 * Current tiling mode for the object. 747 * Current tiling mode for the object.
771 */ 748 */
772 unsigned int tiling_mode : 2; 749 unsigned int tiling_mode : 2;
750 unsigned int tiling_changed : 1;
773 751
774 /** How many users have pinned this object in GTT space. The following 752 /** How many users have pinned this object in GTT space. The following
775 * users can each hold at most one reference: pwrite/pread, pin_ioctl 753 * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -783,28 +761,54 @@ struct drm_i915_gem_object {
783 unsigned int pin_count : 4; 761 unsigned int pin_count : 4;
784#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf 762#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
785 763
786 /** AGP memory structure for our GTT binding. */ 764 /**
787 DRM_AGP_MEM *agp_mem; 765 * Is the object at the current location in the gtt mappable and
766 * fenceable? Used to avoid costly recalculations.
767 */
768 unsigned int map_and_fenceable : 1;
769
770 /**
771 * Whether the current gtt mapping needs to be mappable (and isn't just
772 * mappable by accident). Track pin and fault separate for a more
773 * accurate mappable working set.
774 */
775 unsigned int fault_mappable : 1;
776 unsigned int pin_mappable : 1;
777
778 /*
779 * Is the GPU currently using a fence to access this buffer,
780 */
781 unsigned int pending_fenced_gpu_access:1;
782 unsigned int fenced_gpu_access:1;
788 783
789 struct page **pages; 784 struct page **pages;
790 785
791 /** 786 /**
792 * Current offset of the object in GTT space. 787 * DMAR support
793 *
794 * This is the same as gtt_space->start
795 */ 788 */
796 uint32_t gtt_offset; 789 struct scatterlist *sg_list;
790 int num_sg;
797 791
798 /* Which ring is refering to is this object */ 792 /**
799 struct intel_ring_buffer *ring; 793 * Used for performing relocations during execbuffer insertion.
794 */
795 struct hlist_node exec_node;
796 unsigned long exec_handle;
800 797
801 /** 798 /**
802 * Fake offset for use by mmap(2) 799 * Current offset of the object in GTT space.
800 *
801 * This is the same as gtt_space->start
803 */ 802 */
804 uint64_t mmap_offset; 803 uint32_t gtt_offset;
805 804
806 /** Breadcrumb of last rendering to the buffer. */ 805 /** Breadcrumb of last rendering to the buffer. */
807 uint32_t last_rendering_seqno; 806 uint32_t last_rendering_seqno;
807 struct intel_ring_buffer *ring;
808
809 /** Breadcrumb of last fenced GPU access to the buffer. */
810 uint32_t last_fenced_seqno;
811 struct intel_ring_buffer *last_fenced_ring;
808 812
809 /** Current tiling stride for the object, if it's tiled. */ 813 /** Current tiling stride for the object, if it's tiled. */
810 uint32_t stride; 814 uint32_t stride;
@@ -880,6 +884,68 @@ enum intel_chip_family {
880 CHIP_I965 = 0x08, 884 CHIP_I965 = 0x08,
881}; 885};
882 886
887#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
888
889#define IS_I830(dev) ((dev)->pci_device == 0x3577)
890#define IS_845G(dev) ((dev)->pci_device == 0x2562)
891#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x)
892#define IS_I865G(dev) ((dev)->pci_device == 0x2572)
893#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g)
894#define IS_I915GM(dev) ((dev)->pci_device == 0x2592)
895#define IS_I945G(dev) ((dev)->pci_device == 0x2772)
896#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm)
897#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater)
898#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline)
899#define IS_GM45(dev) ((dev)->pci_device == 0x2A42)
900#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x)
901#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001)
902#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011)
903#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview)
904#define IS_G33(dev) (INTEL_INFO(dev)->is_g33)
905#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042)
906#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046)
907#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile)
908
909#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2)
910#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3)
911#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4)
912#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5)
913#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6)
914
915#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring)
916#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring)
917#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws)
918
919#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
920#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
921
922/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
923 * rows, which changed the alignment requirements and fence programming.
924 */
925#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
926 IS_I915GM(dev)))
927#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
928#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev))
929#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev))
930#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev))
931#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv)
932#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug)
933/* dsparb controlled by hw only */
934#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
935
936#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
937#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
938#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
939
940#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
941#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
942
943#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
944#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
945#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
946
947#include "i915_trace.h"
948
883extern struct drm_ioctl_desc i915_ioctls[]; 949extern struct drm_ioctl_desc i915_ioctls[];
884extern int i915_max_ioctl; 950extern int i915_max_ioctl;
885extern unsigned int i915_fbpercrtc; 951extern unsigned int i915_fbpercrtc;
@@ -907,8 +973,8 @@ extern int i915_driver_device_is_agp(struct drm_device * dev);
907extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, 973extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
908 unsigned long arg); 974 unsigned long arg);
909extern int i915_emit_box(struct drm_device *dev, 975extern int i915_emit_box(struct drm_device *dev,
910 struct drm_clip_rect *boxes, 976 struct drm_clip_rect *box,
911 int i, int DR1, int DR4); 977 int DR1, int DR4);
912extern int i915_reset(struct drm_device *dev, u8 flags); 978extern int i915_reset(struct drm_device *dev, u8 flags);
913extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); 979extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
914extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); 980extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
@@ -918,6 +984,7 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
918 984
919/* i915_irq.c */ 985/* i915_irq.c */
920void i915_hangcheck_elapsed(unsigned long data); 986void i915_hangcheck_elapsed(unsigned long data);
987void i915_handle_error(struct drm_device *dev, bool wedged);
921extern int i915_irq_emit(struct drm_device *dev, void *data, 988extern int i915_irq_emit(struct drm_device *dev, void *data,
922 struct drm_file *file_priv); 989 struct drm_file *file_priv);
923extern int i915_irq_wait(struct drm_device *dev, void *data, 990extern int i915_irq_wait(struct drm_device *dev, void *data,
@@ -953,6 +1020,13 @@ void
953i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); 1020i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
954 1021
955void intel_enable_asle (struct drm_device *dev); 1022void intel_enable_asle (struct drm_device *dev);
1023int i915_get_vblank_timestamp(struct drm_device *dev, int crtc,
1024 int *max_error,
1025 struct timeval *vblank_time,
1026 unsigned flags);
1027
1028int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
1029 int *vpos, int *hpos);
956 1030
957#ifdef CONFIG_DEBUG_FS 1031#ifdef CONFIG_DEBUG_FS
958extern void i915_destroy_error_state(struct drm_device *dev); 1032extern void i915_destroy_error_state(struct drm_device *dev);
@@ -1017,15 +1091,28 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
1017 struct drm_file *file_priv); 1091 struct drm_file *file_priv);
1018void i915_gem_load(struct drm_device *dev); 1092void i915_gem_load(struct drm_device *dev);
1019int i915_gem_init_object(struct drm_gem_object *obj); 1093int i915_gem_init_object(struct drm_gem_object *obj);
1020struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 1094void i915_gem_flush_ring(struct drm_device *dev,
1021 size_t size); 1095 struct intel_ring_buffer *ring,
1096 uint32_t invalidate_domains,
1097 uint32_t flush_domains);
1098struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
1099 size_t size);
1022void i915_gem_free_object(struct drm_gem_object *obj); 1100void i915_gem_free_object(struct drm_gem_object *obj);
1023int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment); 1101int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
1024void i915_gem_object_unpin(struct drm_gem_object *obj); 1102 uint32_t alignment,
1025int i915_gem_object_unbind(struct drm_gem_object *obj); 1103 bool map_and_fenceable);
1026void i915_gem_release_mmap(struct drm_gem_object *obj); 1104void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
1105int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
1106void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
1027void i915_gem_lastclose(struct drm_device *dev); 1107void i915_gem_lastclose(struct drm_device *dev);
1028 1108
1109int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
1110int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1111 bool interruptible);
1112void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1113 struct intel_ring_buffer *ring,
1114 u32 seqno);
1115
1029/** 1116/**
1030 * Returns true if seq1 is later than seq2. 1117 * Returns true if seq1 is later than seq2.
1031 */ 1118 */
@@ -1035,73 +1122,88 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1035 return (int32_t)(seq1 - seq2) >= 0; 1122 return (int32_t)(seq1 - seq2) >= 0;
1036} 1123}
1037 1124
1038int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, 1125static inline u32
1039 bool interruptible); 1126i915_gem_next_request_seqno(struct drm_device *dev,
1040int i915_gem_object_put_fence_reg(struct drm_gem_object *obj, 1127 struct intel_ring_buffer *ring)
1041 bool interruptible); 1128{
1129 drm_i915_private_t *dev_priv = dev->dev_private;
1130 return ring->outstanding_lazy_request = dev_priv->next_seqno;
1131}
1132
1133int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
1134 struct intel_ring_buffer *pipelined,
1135 bool interruptible);
1136int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
1137
1042void i915_gem_retire_requests(struct drm_device *dev); 1138void i915_gem_retire_requests(struct drm_device *dev);
1043void i915_gem_reset(struct drm_device *dev); 1139void i915_gem_reset(struct drm_device *dev);
1044void i915_gem_clflush_object(struct drm_gem_object *obj); 1140void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
1045int i915_gem_object_set_domain(struct drm_gem_object *obj, 1141int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj,
1046 uint32_t read_domains, 1142 uint32_t read_domains,
1047 uint32_t write_domain); 1143 uint32_t write_domain);
1048int i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, 1144int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
1049 bool interruptible); 1145 bool interruptible);
1050int i915_gem_init_ringbuffer(struct drm_device *dev); 1146int __must_check i915_gem_init_ringbuffer(struct drm_device *dev);
1051void i915_gem_cleanup_ringbuffer(struct drm_device *dev); 1147void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
1052int i915_gem_do_init(struct drm_device *dev, unsigned long start, 1148void i915_gem_do_init(struct drm_device *dev,
1053 unsigned long end); 1149 unsigned long start,
1054int i915_gpu_idle(struct drm_device *dev); 1150 unsigned long mappable_end,
1055int i915_gem_idle(struct drm_device *dev); 1151 unsigned long end);
1056uint32_t i915_add_request(struct drm_device *dev, 1152int __must_check i915_gpu_idle(struct drm_device *dev);
1057 struct drm_file *file_priv, 1153int __must_check i915_gem_idle(struct drm_device *dev);
1058 struct drm_i915_gem_request *request, 1154int __must_check i915_add_request(struct drm_device *dev,
1059 struct intel_ring_buffer *ring); 1155 struct drm_file *file_priv,
1060int i915_do_wait_request(struct drm_device *dev, 1156 struct drm_i915_gem_request *request,
1061 uint32_t seqno, 1157 struct intel_ring_buffer *ring);
1062 bool interruptible, 1158int __must_check i915_do_wait_request(struct drm_device *dev,
1063 struct intel_ring_buffer *ring); 1159 uint32_t seqno,
1160 bool interruptible,
1161 struct intel_ring_buffer *ring);
1064int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); 1162int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
1065int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, 1163int __must_check
1066 int write); 1164i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
1067int i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, 1165 bool write);
1068 bool pipelined); 1166int __must_check
1167i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
1168 struct intel_ring_buffer *pipelined);
1069int i915_gem_attach_phys_object(struct drm_device *dev, 1169int i915_gem_attach_phys_object(struct drm_device *dev,
1070 struct drm_gem_object *obj, 1170 struct drm_i915_gem_object *obj,
1071 int id, 1171 int id,
1072 int align); 1172 int align);
1073void i915_gem_detach_phys_object(struct drm_device *dev, 1173void i915_gem_detach_phys_object(struct drm_device *dev,
1074 struct drm_gem_object *obj); 1174 struct drm_i915_gem_object *obj);
1075void i915_gem_free_all_phys_object(struct drm_device *dev); 1175void i915_gem_free_all_phys_object(struct drm_device *dev);
1076void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv); 1176void i915_gem_release(struct drm_device *dev, struct drm_file *file);
1077 1177
1078void i915_gem_shrinker_init(void); 1178/* i915_gem_gtt.c */
1079void i915_gem_shrinker_exit(void); 1179void i915_gem_restore_gtt_mappings(struct drm_device *dev);
1180int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
1181void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
1080 1182
1081/* i915_gem_evict.c */ 1183/* i915_gem_evict.c */
1082int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment); 1184int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
1083int i915_gem_evict_everything(struct drm_device *dev); 1185 unsigned alignment, bool mappable);
1084int i915_gem_evict_inactive(struct drm_device *dev); 1186int __must_check i915_gem_evict_everything(struct drm_device *dev,
1187 bool purgeable_only);
1188int __must_check i915_gem_evict_inactive(struct drm_device *dev,
1189 bool purgeable_only);
1085 1190
1086/* i915_gem_tiling.c */ 1191/* i915_gem_tiling.c */
1087void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); 1192void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
1088void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj); 1193void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
1089void i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj); 1194void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
1090bool i915_tiling_ok(struct drm_device *dev, int stride, int size,
1091 int tiling_mode);
1092bool i915_gem_object_fence_offset_ok(struct drm_gem_object *obj,
1093 int tiling_mode);
1094 1195
1095/* i915_gem_debug.c */ 1196/* i915_gem_debug.c */
1096void i915_gem_dump_object(struct drm_gem_object *obj, int len, 1197void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
1097 const char *where, uint32_t mark); 1198 const char *where, uint32_t mark);
1098#if WATCH_LISTS 1199#if WATCH_LISTS
1099int i915_verify_lists(struct drm_device *dev); 1200int i915_verify_lists(struct drm_device *dev);
1100#else 1201#else
1101#define i915_verify_lists(dev) 0 1202#define i915_verify_lists(dev) 0
1102#endif 1203#endif
1103void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle); 1204void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj,
1104void i915_gem_dump_object(struct drm_gem_object *obj, int len, 1205 int handle);
1206void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
1105 const char *where, uint32_t mark); 1207 const char *where, uint32_t mark);
1106 1208
1107/* i915_debugfs.c */ 1209/* i915_debugfs.c */
@@ -1163,6 +1265,7 @@ extern void intel_disable_fbc(struct drm_device *dev);
1163extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval); 1265extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
1164extern bool intel_fbc_enabled(struct drm_device *dev); 1266extern bool intel_fbc_enabled(struct drm_device *dev);
1165extern bool ironlake_set_drps(struct drm_device *dev, u8 val); 1267extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
1268extern void gen6_set_rps(struct drm_device *dev, u8 val);
1166extern void intel_detect_pch (struct drm_device *dev); 1269extern void intel_detect_pch (struct drm_device *dev);
1167extern int intel_trans_dp_port_sel (struct drm_crtc *crtc); 1270extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
1168 1271
@@ -1170,79 +1273,120 @@ extern int intel_trans_dp_port_sel (struct drm_crtc *crtc);
1170#ifdef CONFIG_DEBUG_FS 1273#ifdef CONFIG_DEBUG_FS
1171extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev); 1274extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
1172extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error); 1275extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error);
1276
1277extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
1278extern void intel_display_print_error_state(struct seq_file *m,
1279 struct drm_device *dev,
1280 struct intel_display_error_state *error);
1173#endif 1281#endif
1174 1282
1283#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
1284
1285#define BEGIN_LP_RING(n) \
1286 intel_ring_begin(LP_RING(dev_priv), (n))
1287
1288#define OUT_RING(x) \
1289 intel_ring_emit(LP_RING(dev_priv), x)
1290
1291#define ADVANCE_LP_RING() \
1292 intel_ring_advance(LP_RING(dev_priv))
1293
1175/** 1294/**
1176 * Lock test for when it's just for synchronization of ring access. 1295 * Lock test for when it's just for synchronization of ring access.
1177 * 1296 *
1178 * In that case, we don't need to do it when GEM is initialized as nobody else 1297 * In that case, we don't need to do it when GEM is initialized as nobody else
1179 * has access to the ring. 1298 * has access to the ring.
1180 */ 1299 */
1181#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do { \ 1300#define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \
1182 if (((drm_i915_private_t *)dev->dev_private)->render_ring.gem_object \ 1301 if (LP_RING(dev->dev_private)->obj == NULL) \
1183 == NULL) \ 1302 LOCK_TEST_WITH_RETURN(dev, file); \
1184 LOCK_TEST_WITH_RETURN(dev, file_priv); \
1185} while (0) 1303} while (0)
1186 1304
1187static inline u32 i915_read(struct drm_i915_private *dev_priv, u32 reg) 1305
1306#define __i915_read(x, y) \
1307static inline u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
1308 u##x val = read##y(dev_priv->regs + reg); \
1309 trace_i915_reg_rw('R', reg, val, sizeof(val)); \
1310 return val; \
1311}
1312__i915_read(8, b)
1313__i915_read(16, w)
1314__i915_read(32, l)
1315__i915_read(64, q)
1316#undef __i915_read
1317
1318#define __i915_write(x, y) \
1319static inline void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
1320 trace_i915_reg_rw('W', reg, val, sizeof(val)); \
1321 write##y(val, dev_priv->regs + reg); \
1322}
1323__i915_write(8, b)
1324__i915_write(16, w)
1325__i915_write(32, l)
1326__i915_write(64, q)
1327#undef __i915_write
1328
1329#define I915_READ8(reg) i915_read8(dev_priv, (reg))
1330#define I915_WRITE8(reg, val) i915_write8(dev_priv, (reg), (val))
1331
1332#define I915_READ16(reg) i915_read16(dev_priv, (reg))
1333#define I915_WRITE16(reg, val) i915_write16(dev_priv, (reg), (val))
1334#define I915_READ16_NOTRACE(reg) readw(dev_priv->regs + (reg))
1335#define I915_WRITE16_NOTRACE(reg, val) writew(val, dev_priv->regs + (reg))
1336
1337#define I915_READ(reg) i915_read32(dev_priv, (reg))
1338#define I915_WRITE(reg, val) i915_write32(dev_priv, (reg), (val))
1339#define I915_READ_NOTRACE(reg) readl(dev_priv->regs + (reg))
1340#define I915_WRITE_NOTRACE(reg, val) writel(val, dev_priv->regs + (reg))
1341
1342#define I915_WRITE64(reg, val) i915_write64(dev_priv, (reg), (val))
1343#define I915_READ64(reg) i915_read64(dev_priv, (reg))
1344
1345#define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg)
1346#define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg)
1347
1348
1349/* On SNB platform, before reading ring registers forcewake bit
1350 * must be set to prevent GT core from power down and stale values being
1351 * returned.
1352 */
1353void __gen6_force_wake_get(struct drm_i915_private *dev_priv);
1354void __gen6_force_wake_put (struct drm_i915_private *dev_priv);
1355static inline u32 i915_safe_read(struct drm_i915_private *dev_priv, u32 reg)
1188{ 1356{
1189 u32 val; 1357 u32 val;
1190 1358
1191 val = readl(dev_priv->regs + reg); 1359 if (dev_priv->info->gen >= 6) {
1192 if (dev_priv->debug_flags & I915_DEBUG_READ) 1360 __gen6_force_wake_get(dev_priv);
1193 printk(KERN_ERR "read 0x%08x from 0x%08x\n", val, reg); 1361 val = I915_READ(reg);
1362 __gen6_force_wake_put(dev_priv);
1363 } else
1364 val = I915_READ(reg);
1365
1194 return val; 1366 return val;
1195} 1367}
1196 1368
1197static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg, 1369static inline void
1198 u32 val) 1370i915_write(struct drm_i915_private *dev_priv, u32 reg, u64 val, int len)
1199{ 1371{
1200 writel(val, dev_priv->regs + reg); 1372 /* Trace down the write operation before the real write */
1201 if (dev_priv->debug_flags & I915_DEBUG_WRITE) 1373 trace_i915_reg_rw('W', reg, val, len);
1202 printk(KERN_ERR "wrote 0x%08x to 0x%08x\n", val, reg); 1374 switch (len) {
1375 case 8:
1376 writeq(val, dev_priv->regs + reg);
1377 break;
1378 case 4:
1379 writel(val, dev_priv->regs + reg);
1380 break;
1381 case 2:
1382 writew(val, dev_priv->regs + reg);
1383 break;
1384 case 1:
1385 writeb(val, dev_priv->regs + reg);
1386 break;
1387 }
1203} 1388}
1204 1389
1205#define I915_READ(reg) i915_read(dev_priv, (reg))
1206#define I915_WRITE(reg, val) i915_write(dev_priv, (reg), (val))
1207#define I915_READ16(reg) readw(dev_priv->regs + (reg))
1208#define I915_WRITE16(reg, val) writel(val, dev_priv->regs + (reg))
1209#define I915_READ8(reg) readb(dev_priv->regs + (reg))
1210#define I915_WRITE8(reg, val) writeb(val, dev_priv->regs + (reg))
1211#define I915_WRITE64(reg, val) writeq(val, dev_priv->regs + (reg))
1212#define I915_READ64(reg) readq(dev_priv->regs + (reg))
1213#define POSTING_READ(reg) (void)I915_READ(reg)
1214#define POSTING_READ16(reg) (void)I915_READ16(reg)
1215
1216#define I915_DEBUG_ENABLE_IO() (dev_priv->debug_flags |= I915_DEBUG_READ | \
1217 I915_DEBUG_WRITE)
1218#define I915_DEBUG_DISABLE_IO() (dev_priv->debug_flags &= ~(I915_DEBUG_READ | \
1219 I915_DEBUG_WRITE))
1220
1221#define I915_VERBOSE 0
1222
1223#define BEGIN_LP_RING(n) do { \
1224 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1225 if (I915_VERBOSE) \
1226 DRM_DEBUG(" BEGIN_LP_RING %x\n", (int)(n)); \
1227 intel_ring_begin(dev, &dev_priv__->render_ring, (n)); \
1228} while (0)
1229
1230
1231#define OUT_RING(x) do { \
1232 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1233 if (I915_VERBOSE) \
1234 DRM_DEBUG(" OUT_RING %x\n", (int)(x)); \
1235 intel_ring_emit(dev, &dev_priv__->render_ring, x); \
1236} while (0)
1237
1238#define ADVANCE_LP_RING() do { \
1239 drm_i915_private_t *dev_priv__ = dev->dev_private; \
1240 if (I915_VERBOSE) \
1241 DRM_DEBUG("ADVANCE_LP_RING %x\n", \
1242 dev_priv__->render_ring.tail); \
1243 intel_ring_advance(dev, &dev_priv__->render_ring); \
1244} while(0)
1245
1246/** 1390/**
1247 * Reads a dword out of the status page, which is written to from the command 1391 * Reads a dword out of the status page, which is written to from the command
1248 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or 1392 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
@@ -1259,72 +1403,9 @@ static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg,
1259 * The area from dword 0x20 to 0x3ff is available for driver usage. 1403 * The area from dword 0x20 to 0x3ff is available for driver usage.
1260 */ 1404 */
1261#define READ_HWSP(dev_priv, reg) (((volatile u32 *)\ 1405#define READ_HWSP(dev_priv, reg) (((volatile u32 *)\
1262 (dev_priv->render_ring.status_page.page_addr))[reg]) 1406 (LP_RING(dev_priv)->status_page.page_addr))[reg])
1263#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) 1407#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
1264#define I915_GEM_HWS_INDEX 0x20 1408#define I915_GEM_HWS_INDEX 0x20
1265#define I915_BREADCRUMB_INDEX 0x21 1409#define I915_BREADCRUMB_INDEX 0x21
1266 1410
1267#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info)
1268
1269#define IS_I830(dev) ((dev)->pci_device == 0x3577)
1270#define IS_845G(dev) ((dev)->pci_device == 0x2562)
1271#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x)
1272#define IS_I865G(dev) ((dev)->pci_device == 0x2572)
1273#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g)
1274#define IS_I915GM(dev) ((dev)->pci_device == 0x2592)
1275#define IS_I945G(dev) ((dev)->pci_device == 0x2772)
1276#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm)
1277#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater)
1278#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline)
1279#define IS_GM45(dev) ((dev)->pci_device == 0x2A42)
1280#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x)
1281#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001)
1282#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011)
1283#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview)
1284#define IS_G33(dev) (INTEL_INFO(dev)->is_g33)
1285#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042)
1286#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046)
1287#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile)
1288
1289#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2)
1290#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3)
1291#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4)
1292#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5)
1293#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6)
1294
1295#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring)
1296#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring)
1297#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws)
1298
1299#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay)
1300#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical)
1301
1302/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
1303 * rows, which changed the alignment requirements and fence programming.
1304 */
1305#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
1306 IS_I915GM(dev)))
1307#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev))
1308#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev))
1309#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev))
1310#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev))
1311#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv)
1312#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug)
1313/* dsparb controlled by hw only */
1314#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev))
1315
1316#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
1317#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
1318#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
1319#define I915_HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6)
1320
1321#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev))
1322#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev))
1323
1324#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
1325#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
1326#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
1327
1328#define PRIMARY_RINGBUFFER_SIZE (128*1024)
1329
1330#endif 1411#endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 275ec6ed43ae..c79c0b62ef60 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,38 +34,31 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/swap.h> 35#include <linux/swap.h>
36#include <linux/pci.h> 36#include <linux/pci.h>
37#include <linux/intel-gtt.h>
38 37
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); 38static void i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 41static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 42 bool write);
44static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 43static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45 int write);
46static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
47 uint64_t offset, 44 uint64_t offset,
48 uint64_t size); 45 uint64_t size);
49static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
50static int i915_gem_object_wait_rendering(struct drm_gem_object *obj, 47static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
51 bool interruptible); 48 unsigned alignment,
52static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 49 bool map_and_fenceable);
53 unsigned alignment); 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
54static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 51 struct drm_i915_fence_reg *reg);
55static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
56 struct drm_i915_gem_pwrite *args, 54 struct drm_i915_gem_pwrite *args,
57 struct drm_file *file_priv); 55 struct drm_file *file);
58static void i915_gem_free_object_tail(struct drm_gem_object *obj); 56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
59 57
60static int 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
61i915_gem_object_get_pages(struct drm_gem_object *obj, 59 int nr_to_scan,
62 gfp_t gfpmask); 60 gfp_t gfp_mask);
63 61
64static void
65i915_gem_object_put_pages(struct drm_gem_object *obj);
66
67static LIST_HEAD(shrink_list);
68static DEFINE_SPINLOCK(shrink_list_lock);
69 62
70/* some bookkeeping */ 63/* some bookkeeping */
71static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 64static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
@@ -82,34 +75,6 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
82 dev_priv->mm.object_memory -= size; 75 dev_priv->mm.object_memory -= size;
83} 76}
84 77
85static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
86 size_t size)
87{
88 dev_priv->mm.gtt_count++;
89 dev_priv->mm.gtt_memory += size;
90}
91
92static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
93 size_t size)
94{
95 dev_priv->mm.gtt_count--;
96 dev_priv->mm.gtt_memory -= size;
97}
98
99static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
100 size_t size)
101{
102 dev_priv->mm.pin_count++;
103 dev_priv->mm.pin_memory += size;
104}
105
106static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
107 size_t size)
108{
109 dev_priv->mm.pin_count--;
110 dev_priv->mm.pin_memory -= size;
111}
112
113int 78int
114i915_gem_check_is_wedged(struct drm_device *dev) 79i915_gem_check_is_wedged(struct drm_device *dev)
115{ 80{
@@ -140,7 +105,7 @@ i915_gem_check_is_wedged(struct drm_device *dev)
140 return -EIO; 105 return -EIO;
141} 106}
142 107
143static int i915_mutex_lock_interruptible(struct drm_device *dev) 108int i915_mutex_lock_interruptible(struct drm_device *dev)
144{ 109{
145 struct drm_i915_private *dev_priv = dev->dev_private; 110 struct drm_i915_private *dev_priv = dev->dev_private;
146 int ret; 111 int ret;
@@ -163,75 +128,76 @@ static int i915_mutex_lock_interruptible(struct drm_device *dev)
163} 128}
164 129
165static inline bool 130static inline bool
166i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) 131i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
167{ 132{
168 return obj_priv->gtt_space && 133 return obj->gtt_space && !obj->active && obj->pin_count == 0;
169 !obj_priv->active &&
170 obj_priv->pin_count == 0;
171} 134}
172 135
173int i915_gem_do_init(struct drm_device *dev, 136void i915_gem_do_init(struct drm_device *dev,
174 unsigned long start, 137 unsigned long start,
175 unsigned long end) 138 unsigned long mappable_end,
139 unsigned long end)
176{ 140{
177 drm_i915_private_t *dev_priv = dev->dev_private; 141 drm_i915_private_t *dev_priv = dev->dev_private;
178 142
179 if (start >= end ||
180 (start & (PAGE_SIZE - 1)) != 0 ||
181 (end & (PAGE_SIZE - 1)) != 0) {
182 return -EINVAL;
183 }
184
185 drm_mm_init(&dev_priv->mm.gtt_space, start, 143 drm_mm_init(&dev_priv->mm.gtt_space, start,
186 end - start); 144 end - start);
187 145
188 dev_priv->mm.gtt_total = end - start; 146 dev_priv->mm.gtt_total = end - start;
189 147 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
190 return 0; 148 dev_priv->mm.gtt_mappable_end = mappable_end;
191} 149}
192 150
193int 151int
194i915_gem_init_ioctl(struct drm_device *dev, void *data, 152i915_gem_init_ioctl(struct drm_device *dev, void *data,
195 struct drm_file *file_priv) 153 struct drm_file *file)
196{ 154{
197 struct drm_i915_gem_init *args = data; 155 struct drm_i915_gem_init *args = data;
198 int ret; 156
157 if (args->gtt_start >= args->gtt_end ||
158 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159 return -EINVAL;
199 160
200 mutex_lock(&dev->struct_mutex); 161 mutex_lock(&dev->struct_mutex);
201 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); 162 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
202 mutex_unlock(&dev->struct_mutex); 163 mutex_unlock(&dev->struct_mutex);
203 164
204 return ret; 165 return 0;
205} 166}
206 167
207int 168int
208i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 169i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
209 struct drm_file *file_priv) 170 struct drm_file *file)
210{ 171{
211 struct drm_i915_private *dev_priv = dev->dev_private; 172 struct drm_i915_private *dev_priv = dev->dev_private;
212 struct drm_i915_gem_get_aperture *args = data; 173 struct drm_i915_gem_get_aperture *args = data;
174 struct drm_i915_gem_object *obj;
175 size_t pinned;
213 176
214 if (!(dev->driver->driver_features & DRIVER_GEM)) 177 if (!(dev->driver->driver_features & DRIVER_GEM))
215 return -ENODEV; 178 return -ENODEV;
216 179
180 pinned = 0;
217 mutex_lock(&dev->struct_mutex); 181 mutex_lock(&dev->struct_mutex);
218 args->aper_size = dev_priv->mm.gtt_total; 182 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
219 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory; 183 pinned += obj->gtt_space->size;
220 mutex_unlock(&dev->struct_mutex); 184 mutex_unlock(&dev->struct_mutex);
221 185
186 args->aper_size = dev_priv->mm.gtt_total;
187 args->aper_available_size = args->aper_size -pinned;
188
222 return 0; 189 return 0;
223} 190}
224 191
225
226/** 192/**
227 * Creates a new mm object and returns a handle to it. 193 * Creates a new mm object and returns a handle to it.
228 */ 194 */
229int 195int
230i915_gem_create_ioctl(struct drm_device *dev, void *data, 196i915_gem_create_ioctl(struct drm_device *dev, void *data,
231 struct drm_file *file_priv) 197 struct drm_file *file)
232{ 198{
233 struct drm_i915_gem_create *args = data; 199 struct drm_i915_gem_create *args = data;
234 struct drm_gem_object *obj; 200 struct drm_i915_gem_object *obj;
235 int ret; 201 int ret;
236 u32 handle; 202 u32 handle;
237 203
@@ -242,45 +208,28 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
242 if (obj == NULL) 208 if (obj == NULL)
243 return -ENOMEM; 209 return -ENOMEM;
244 210
245 ret = drm_gem_handle_create(file_priv, obj, &handle); 211 ret = drm_gem_handle_create(file, &obj->base, &handle);
246 if (ret) { 212 if (ret) {
247 drm_gem_object_release(obj); 213 drm_gem_object_release(&obj->base);
248 i915_gem_info_remove_obj(dev->dev_private, obj->size); 214 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
249 kfree(obj); 215 kfree(obj);
250 return ret; 216 return ret;
251 } 217 }
252 218
253 /* drop reference from allocate - handle holds it now */ 219 /* drop reference from allocate - handle holds it now */
254 drm_gem_object_unreference(obj); 220 drm_gem_object_unreference(&obj->base);
255 trace_i915_gem_object_create(obj); 221 trace_i915_gem_object_create(obj);
256 222
257 args->handle = handle; 223 args->handle = handle;
258 return 0; 224 return 0;
259} 225}
260 226
261static inline int 227static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
262fast_shmem_read(struct page **pages,
263 loff_t page_base, int page_offset,
264 char __user *data,
265 int length)
266{
267 char *vaddr;
268 int ret;
269
270 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
271 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
272 kunmap_atomic(vaddr);
273
274 return ret;
275}
276
277static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
278{ 228{
279 drm_i915_private_t *dev_priv = obj->dev->dev_private; 229 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
280 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
281 230
282 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
283 obj_priv->tiling_mode != I915_TILING_NONE; 232 obj->tiling_mode != I915_TILING_NONE;
284} 233}
285 234
286static inline void 235static inline void
@@ -356,38 +305,51 @@ slow_shmem_bit17_copy(struct page *gpu_page,
356 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). 305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
357 */ 306 */
358static int 307static int
359i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, 308i915_gem_shmem_pread_fast(struct drm_device *dev,
309 struct drm_i915_gem_object *obj,
360 struct drm_i915_gem_pread *args, 310 struct drm_i915_gem_pread *args,
361 struct drm_file *file_priv) 311 struct drm_file *file)
362{ 312{
363 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 313 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
364 ssize_t remain; 314 ssize_t remain;
365 loff_t offset, page_base; 315 loff_t offset;
366 char __user *user_data; 316 char __user *user_data;
367 int page_offset, page_length; 317 int page_offset, page_length;
368 318
369 user_data = (char __user *) (uintptr_t) args->data_ptr; 319 user_data = (char __user *) (uintptr_t) args->data_ptr;
370 remain = args->size; 320 remain = args->size;
371 321
372 obj_priv = to_intel_bo(obj);
373 offset = args->offset; 322 offset = args->offset;
374 323
375 while (remain > 0) { 324 while (remain > 0) {
325 struct page *page;
326 char *vaddr;
327 int ret;
328
376 /* Operation in this page 329 /* Operation in this page
377 * 330 *
378 * page_base = page offset within aperture
379 * page_offset = offset within page 331 * page_offset = offset within page
380 * page_length = bytes to copy for this page 332 * page_length = bytes to copy for this page
381 */ 333 */
382 page_base = (offset & ~(PAGE_SIZE-1));
383 page_offset = offset & (PAGE_SIZE-1); 334 page_offset = offset & (PAGE_SIZE-1);
384 page_length = remain; 335 page_length = remain;
385 if ((page_offset + remain) > PAGE_SIZE) 336 if ((page_offset + remain) > PAGE_SIZE)
386 page_length = PAGE_SIZE - page_offset; 337 page_length = PAGE_SIZE - page_offset;
387 338
388 if (fast_shmem_read(obj_priv->pages, 339 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
389 page_base, page_offset, 340 GFP_HIGHUSER | __GFP_RECLAIMABLE);
390 user_data, page_length)) 341 if (IS_ERR(page))
342 return PTR_ERR(page);
343
344 vaddr = kmap_atomic(page);
345 ret = __copy_to_user_inatomic(user_data,
346 vaddr + page_offset,
347 page_length);
348 kunmap_atomic(vaddr);
349
350 mark_page_accessed(page);
351 page_cache_release(page);
352 if (ret)
391 return -EFAULT; 353 return -EFAULT;
392 354
393 remain -= page_length; 355 remain -= page_length;
@@ -398,30 +360,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
398 return 0; 360 return 0;
399} 361}
400 362
401static int
402i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
403{
404 int ret;
405
406 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
407
408 /* If we've insufficient memory to map in the pages, attempt
409 * to make some space by throwing out some old buffers.
410 */
411 if (ret == -ENOMEM) {
412 struct drm_device *dev = obj->dev;
413
414 ret = i915_gem_evict_something(dev, obj->size,
415 i915_gem_get_gtt_alignment(obj));
416 if (ret)
417 return ret;
418
419 ret = i915_gem_object_get_pages(obj, 0);
420 }
421
422 return ret;
423}
424
425/** 363/**
426 * This is the fallback shmem pread path, which allocates temporary storage 364 * This is the fallback shmem pread path, which allocates temporary storage
427 * in kernel space to copy_to_user into outside of the struct_mutex, so we 365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
@@ -429,18 +367,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
429 * and not take page faults. 367 * and not take page faults.
430 */ 368 */
431static int 369static int
432i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, 370i915_gem_shmem_pread_slow(struct drm_device *dev,
371 struct drm_i915_gem_object *obj,
433 struct drm_i915_gem_pread *args, 372 struct drm_i915_gem_pread *args,
434 struct drm_file *file_priv) 373 struct drm_file *file)
435{ 374{
436 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 375 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
437 struct mm_struct *mm = current->mm; 376 struct mm_struct *mm = current->mm;
438 struct page **user_pages; 377 struct page **user_pages;
439 ssize_t remain; 378 ssize_t remain;
440 loff_t offset, pinned_pages, i; 379 loff_t offset, pinned_pages, i;
441 loff_t first_data_page, last_data_page, num_pages; 380 loff_t first_data_page, last_data_page, num_pages;
442 int shmem_page_index, shmem_page_offset; 381 int shmem_page_offset;
443 int data_page_index, data_page_offset; 382 int data_page_index, data_page_offset;
444 int page_length; 383 int page_length;
445 int ret; 384 int ret;
446 uint64_t data_ptr = args->data_ptr; 385 uint64_t data_ptr = args->data_ptr;
@@ -479,19 +418,18 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
479 418
480 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 419 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
481 420
482 obj_priv = to_intel_bo(obj);
483 offset = args->offset; 421 offset = args->offset;
484 422
485 while (remain > 0) { 423 while (remain > 0) {
424 struct page *page;
425
486 /* Operation in this page 426 /* Operation in this page
487 * 427 *
488 * shmem_page_index = page number within shmem file
489 * shmem_page_offset = offset within page in shmem file 428 * shmem_page_offset = offset within page in shmem file
490 * data_page_index = page number in get_user_pages return 429 * data_page_index = page number in get_user_pages return
491 * data_page_offset = offset with data_page_index page. 430 * data_page_offset = offset with data_page_index page.
492 * page_length = bytes to copy for this page 431 * page_length = bytes to copy for this page
493 */ 432 */
494 shmem_page_index = offset / PAGE_SIZE;
495 shmem_page_offset = offset & ~PAGE_MASK; 433 shmem_page_offset = offset & ~PAGE_MASK;
496 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 434 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
497 data_page_offset = data_ptr & ~PAGE_MASK; 435 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -502,8 +440,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
502 if ((data_page_offset + page_length) > PAGE_SIZE) 440 if ((data_page_offset + page_length) > PAGE_SIZE)
503 page_length = PAGE_SIZE - data_page_offset; 441 page_length = PAGE_SIZE - data_page_offset;
504 442
443 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
444 GFP_HIGHUSER | __GFP_RECLAIMABLE);
445 if (IS_ERR(page))
446 return PTR_ERR(page);
447
505 if (do_bit17_swizzling) { 448 if (do_bit17_swizzling) {
506 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 449 slow_shmem_bit17_copy(page,
507 shmem_page_offset, 450 shmem_page_offset,
508 user_pages[data_page_index], 451 user_pages[data_page_index],
509 data_page_offset, 452 data_page_offset,
@@ -512,11 +455,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
512 } else { 455 } else {
513 slow_shmem_copy(user_pages[data_page_index], 456 slow_shmem_copy(user_pages[data_page_index],
514 data_page_offset, 457 data_page_offset,
515 obj_priv->pages[shmem_page_index], 458 page,
516 shmem_page_offset, 459 shmem_page_offset,
517 page_length); 460 page_length);
518 } 461 }
519 462
463 mark_page_accessed(page);
464 page_cache_release(page);
465
520 remain -= page_length; 466 remain -= page_length;
521 data_ptr += page_length; 467 data_ptr += page_length;
522 offset += page_length; 468 offset += page_length;
@@ -525,6 +471,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
525out: 471out:
526 for (i = 0; i < pinned_pages; i++) { 472 for (i = 0; i < pinned_pages; i++) {
527 SetPageDirty(user_pages[i]); 473 SetPageDirty(user_pages[i]);
474 mark_page_accessed(user_pages[i]);
528 page_cache_release(user_pages[i]); 475 page_cache_release(user_pages[i]);
529 } 476 }
530 drm_free_large(user_pages); 477 drm_free_large(user_pages);
@@ -539,11 +486,10 @@ out:
539 */ 486 */
540int 487int
541i915_gem_pread_ioctl(struct drm_device *dev, void *data, 488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
542 struct drm_file *file_priv) 489 struct drm_file *file)
543{ 490{
544 struct drm_i915_gem_pread *args = data; 491 struct drm_i915_gem_pread *args = data;
545 struct drm_gem_object *obj; 492 struct drm_i915_gem_object *obj;
546 struct drm_i915_gem_object *obj_priv;
547 int ret = 0; 493 int ret = 0;
548 494
549 if (args->size == 0) 495 if (args->size == 0)
@@ -563,39 +509,33 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
563 if (ret) 509 if (ret)
564 return ret; 510 return ret;
565 511
566 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 512 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
567 if (obj == NULL) { 513 if (obj == NULL) {
568 ret = -ENOENT; 514 ret = -ENOENT;
569 goto unlock; 515 goto unlock;
570 } 516 }
571 obj_priv = to_intel_bo(obj);
572 517
573 /* Bounds check source. */ 518 /* Bounds check source. */
574 if (args->offset > obj->size || args->size > obj->size - args->offset) { 519 if (args->offset > obj->base.size ||
520 args->size > obj->base.size - args->offset) {
575 ret = -EINVAL; 521 ret = -EINVAL;
576 goto out; 522 goto out;
577 } 523 }
578 524
579 ret = i915_gem_object_get_pages_or_evict(obj);
580 if (ret)
581 goto out;
582
583 ret = i915_gem_object_set_cpu_read_domain_range(obj, 525 ret = i915_gem_object_set_cpu_read_domain_range(obj,
584 args->offset, 526 args->offset,
585 args->size); 527 args->size);
586 if (ret) 528 if (ret)
587 goto out_put; 529 goto out;
588 530
589 ret = -EFAULT; 531 ret = -EFAULT;
590 if (!i915_gem_object_needs_bit17_swizzle(obj)) 532 if (!i915_gem_object_needs_bit17_swizzle(obj))
591 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 533 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
592 if (ret == -EFAULT) 534 if (ret == -EFAULT)
593 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 535 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
594 536
595out_put:
596 i915_gem_object_put_pages(obj);
597out: 537out:
598 drm_gem_object_unreference(obj); 538 drm_gem_object_unreference(&obj->base);
599unlock: 539unlock:
600 mutex_unlock(&dev->struct_mutex); 540 mutex_unlock(&dev->struct_mutex);
601 return ret; 541 return ret;
@@ -645,32 +585,16 @@ slow_kernel_write(struct io_mapping *mapping,
645 io_mapping_unmap(dst_vaddr); 585 io_mapping_unmap(dst_vaddr);
646} 586}
647 587
648static inline int
649fast_shmem_write(struct page **pages,
650 loff_t page_base, int page_offset,
651 char __user *data,
652 int length)
653{
654 char *vaddr;
655 int ret;
656
657 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
658 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
659 kunmap_atomic(vaddr);
660
661 return ret;
662}
663
664/** 588/**
665 * This is the fast pwrite path, where we copy the data directly from the 589 * This is the fast pwrite path, where we copy the data directly from the
666 * user into the GTT, uncached. 590 * user into the GTT, uncached.
667 */ 591 */
668static int 592static int
669i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 593i915_gem_gtt_pwrite_fast(struct drm_device *dev,
594 struct drm_i915_gem_object *obj,
670 struct drm_i915_gem_pwrite *args, 595 struct drm_i915_gem_pwrite *args,
671 struct drm_file *file_priv) 596 struct drm_file *file)
672{ 597{
673 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
674 drm_i915_private_t *dev_priv = dev->dev_private; 598 drm_i915_private_t *dev_priv = dev->dev_private;
675 ssize_t remain; 599 ssize_t remain;
676 loff_t offset, page_base; 600 loff_t offset, page_base;
@@ -680,8 +604,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
680 user_data = (char __user *) (uintptr_t) args->data_ptr; 604 user_data = (char __user *) (uintptr_t) args->data_ptr;
681 remain = args->size; 605 remain = args->size;
682 606
683 obj_priv = to_intel_bo(obj); 607 offset = obj->gtt_offset + args->offset;
684 offset = obj_priv->gtt_offset + args->offset;
685 608
686 while (remain > 0) { 609 while (remain > 0) {
687 /* Operation in this page 610 /* Operation in this page
@@ -721,11 +644,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
721 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). 644 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
722 */ 645 */
723static int 646static int
724i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 647i915_gem_gtt_pwrite_slow(struct drm_device *dev,
648 struct drm_i915_gem_object *obj,
725 struct drm_i915_gem_pwrite *args, 649 struct drm_i915_gem_pwrite *args,
726 struct drm_file *file_priv) 650 struct drm_file *file)
727{ 651{
728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
729 drm_i915_private_t *dev_priv = dev->dev_private; 652 drm_i915_private_t *dev_priv = dev->dev_private;
730 ssize_t remain; 653 ssize_t remain;
731 loff_t gtt_page_base, offset; 654 loff_t gtt_page_base, offset;
@@ -762,12 +685,15 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
762 goto out_unpin_pages; 685 goto out_unpin_pages;
763 } 686 }
764 687
765 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 688 ret = i915_gem_object_set_to_gtt_domain(obj, true);
766 if (ret) 689 if (ret)
767 goto out_unpin_pages; 690 goto out_unpin_pages;
768 691
769 obj_priv = to_intel_bo(obj); 692 ret = i915_gem_object_put_fence(obj);
770 offset = obj_priv->gtt_offset + args->offset; 693 if (ret)
694 goto out_unpin_pages;
695
696 offset = obj->gtt_offset + args->offset;
771 697
772 while (remain > 0) { 698 while (remain > 0) {
773 /* Operation in this page 699 /* Operation in this page
@@ -813,39 +739,58 @@ out_unpin_pages:
813 * copy_from_user into the kmapped pages backing the object. 739 * copy_from_user into the kmapped pages backing the object.
814 */ 740 */
815static int 741static int
816i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 742i915_gem_shmem_pwrite_fast(struct drm_device *dev,
743 struct drm_i915_gem_object *obj,
817 struct drm_i915_gem_pwrite *args, 744 struct drm_i915_gem_pwrite *args,
818 struct drm_file *file_priv) 745 struct drm_file *file)
819{ 746{
820 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 747 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
821 ssize_t remain; 748 ssize_t remain;
822 loff_t offset, page_base; 749 loff_t offset;
823 char __user *user_data; 750 char __user *user_data;
824 int page_offset, page_length; 751 int page_offset, page_length;
825 752
826 user_data = (char __user *) (uintptr_t) args->data_ptr; 753 user_data = (char __user *) (uintptr_t) args->data_ptr;
827 remain = args->size; 754 remain = args->size;
828 755
829 obj_priv = to_intel_bo(obj);
830 offset = args->offset; 756 offset = args->offset;
831 obj_priv->dirty = 1; 757 obj->dirty = 1;
832 758
833 while (remain > 0) { 759 while (remain > 0) {
760 struct page *page;
761 char *vaddr;
762 int ret;
763
834 /* Operation in this page 764 /* Operation in this page
835 * 765 *
836 * page_base = page offset within aperture
837 * page_offset = offset within page 766 * page_offset = offset within page
838 * page_length = bytes to copy for this page 767 * page_length = bytes to copy for this page
839 */ 768 */
840 page_base = (offset & ~(PAGE_SIZE-1));
841 page_offset = offset & (PAGE_SIZE-1); 769 page_offset = offset & (PAGE_SIZE-1);
842 page_length = remain; 770 page_length = remain;
843 if ((page_offset + remain) > PAGE_SIZE) 771 if ((page_offset + remain) > PAGE_SIZE)
844 page_length = PAGE_SIZE - page_offset; 772 page_length = PAGE_SIZE - page_offset;
845 773
846 if (fast_shmem_write(obj_priv->pages, 774 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
847 page_base, page_offset, 775 GFP_HIGHUSER | __GFP_RECLAIMABLE);
848 user_data, page_length)) 776 if (IS_ERR(page))
777 return PTR_ERR(page);
778
779 vaddr = kmap_atomic(page, KM_USER0);
780 ret = __copy_from_user_inatomic(vaddr + page_offset,
781 user_data,
782 page_length);
783 kunmap_atomic(vaddr, KM_USER0);
784
785 set_page_dirty(page);
786 mark_page_accessed(page);
787 page_cache_release(page);
788
789 /* If we get a fault while copying data, then (presumably) our
790 * source page isn't available. Return the error and we'll
791 * retry in the slow path.
792 */
793 if (ret)
849 return -EFAULT; 794 return -EFAULT;
850 795
851 remain -= page_length; 796 remain -= page_length;
@@ -864,17 +809,18 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
864 * struct_mutex is held. 809 * struct_mutex is held.
865 */ 810 */
866static int 811static int
867i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 812i915_gem_shmem_pwrite_slow(struct drm_device *dev,
813 struct drm_i915_gem_object *obj,
868 struct drm_i915_gem_pwrite *args, 814 struct drm_i915_gem_pwrite *args,
869 struct drm_file *file_priv) 815 struct drm_file *file)
870{ 816{
871 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 817 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
872 struct mm_struct *mm = current->mm; 818 struct mm_struct *mm = current->mm;
873 struct page **user_pages; 819 struct page **user_pages;
874 ssize_t remain; 820 ssize_t remain;
875 loff_t offset, pinned_pages, i; 821 loff_t offset, pinned_pages, i;
876 loff_t first_data_page, last_data_page, num_pages; 822 loff_t first_data_page, last_data_page, num_pages;
877 int shmem_page_index, shmem_page_offset; 823 int shmem_page_offset;
878 int data_page_index, data_page_offset; 824 int data_page_index, data_page_offset;
879 int page_length; 825 int page_length;
880 int ret; 826 int ret;
@@ -912,20 +858,19 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
912 858
913 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 859 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
914 860
915 obj_priv = to_intel_bo(obj);
916 offset = args->offset; 861 offset = args->offset;
917 obj_priv->dirty = 1; 862 obj->dirty = 1;
918 863
919 while (remain > 0) { 864 while (remain > 0) {
865 struct page *page;
866
920 /* Operation in this page 867 /* Operation in this page
921 * 868 *
922 * shmem_page_index = page number within shmem file
923 * shmem_page_offset = offset within page in shmem file 869 * shmem_page_offset = offset within page in shmem file
924 * data_page_index = page number in get_user_pages return 870 * data_page_index = page number in get_user_pages return
925 * data_page_offset = offset with data_page_index page. 871 * data_page_offset = offset with data_page_index page.
926 * page_length = bytes to copy for this page 872 * page_length = bytes to copy for this page
927 */ 873 */
928 shmem_page_index = offset / PAGE_SIZE;
929 shmem_page_offset = offset & ~PAGE_MASK; 874 shmem_page_offset = offset & ~PAGE_MASK;
930 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 875 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
931 data_page_offset = data_ptr & ~PAGE_MASK; 876 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -936,21 +881,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
936 if ((data_page_offset + page_length) > PAGE_SIZE) 881 if ((data_page_offset + page_length) > PAGE_SIZE)
937 page_length = PAGE_SIZE - data_page_offset; 882 page_length = PAGE_SIZE - data_page_offset;
938 883
884 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
885 GFP_HIGHUSER | __GFP_RECLAIMABLE);
886 if (IS_ERR(page)) {
887 ret = PTR_ERR(page);
888 goto out;
889 }
890
939 if (do_bit17_swizzling) { 891 if (do_bit17_swizzling) {
940 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 892 slow_shmem_bit17_copy(page,
941 shmem_page_offset, 893 shmem_page_offset,
942 user_pages[data_page_index], 894 user_pages[data_page_index],
943 data_page_offset, 895 data_page_offset,
944 page_length, 896 page_length,
945 0); 897 0);
946 } else { 898 } else {
947 slow_shmem_copy(obj_priv->pages[shmem_page_index], 899 slow_shmem_copy(page,
948 shmem_page_offset, 900 shmem_page_offset,
949 user_pages[data_page_index], 901 user_pages[data_page_index],
950 data_page_offset, 902 data_page_offset,
951 page_length); 903 page_length);
952 } 904 }
953 905
906 set_page_dirty(page);
907 mark_page_accessed(page);
908 page_cache_release(page);
909
954 remain -= page_length; 910 remain -= page_length;
955 data_ptr += page_length; 911 data_ptr += page_length;
956 offset += page_length; 912 offset += page_length;
@@ -974,8 +930,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
974 struct drm_file *file) 930 struct drm_file *file)
975{ 931{
976 struct drm_i915_gem_pwrite *args = data; 932 struct drm_i915_gem_pwrite *args = data;
977 struct drm_gem_object *obj; 933 struct drm_i915_gem_object *obj;
978 struct drm_i915_gem_object *obj_priv;
979 int ret; 934 int ret;
980 935
981 if (args->size == 0) 936 if (args->size == 0)
@@ -995,15 +950,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
995 if (ret) 950 if (ret)
996 return ret; 951 return ret;
997 952
998 obj = drm_gem_object_lookup(dev, file, args->handle); 953 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
999 if (obj == NULL) { 954 if (obj == NULL) {
1000 ret = -ENOENT; 955 ret = -ENOENT;
1001 goto unlock; 956 goto unlock;
1002 } 957 }
1003 obj_priv = to_intel_bo(obj);
1004 958
1005 /* Bounds check destination. */ 959 /* Bounds check destination. */
1006 if (args->offset > obj->size || args->size > obj->size - args->offset) { 960 if (args->offset > obj->base.size ||
961 args->size > obj->base.size - args->offset) {
1007 ret = -EINVAL; 962 ret = -EINVAL;
1008 goto out; 963 goto out;
1009 } 964 }
@@ -1014,16 +969,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1014 * pread/pwrite currently are reading and writing from the CPU 969 * pread/pwrite currently are reading and writing from the CPU
1015 * perspective, requiring manual detiling by the client. 970 * perspective, requiring manual detiling by the client.
1016 */ 971 */
1017 if (obj_priv->phys_obj) 972 if (obj->phys_obj)
1018 ret = i915_gem_phys_pwrite(dev, obj, args, file); 973 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1019 else if (obj_priv->tiling_mode == I915_TILING_NONE && 974 else if (obj->gtt_space &&
1020 obj_priv->gtt_space && 975 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1021 obj->write_domain != I915_GEM_DOMAIN_CPU) { 976 ret = i915_gem_object_pin(obj, 0, true);
1022 ret = i915_gem_object_pin(obj, 0);
1023 if (ret) 977 if (ret)
1024 goto out; 978 goto out;
1025 979
1026 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 980 ret = i915_gem_object_set_to_gtt_domain(obj, true);
981 if (ret)
982 goto out_unpin;
983
984 ret = i915_gem_object_put_fence(obj);
1027 if (ret) 985 if (ret)
1028 goto out_unpin; 986 goto out_unpin;
1029 987
@@ -1034,26 +992,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1034out_unpin: 992out_unpin:
1035 i915_gem_object_unpin(obj); 993 i915_gem_object_unpin(obj);
1036 } else { 994 } else {
1037 ret = i915_gem_object_get_pages_or_evict(obj);
1038 if (ret)
1039 goto out;
1040
1041 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 995 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1042 if (ret) 996 if (ret)
1043 goto out_put; 997 goto out;
1044 998
1045 ret = -EFAULT; 999 ret = -EFAULT;
1046 if (!i915_gem_object_needs_bit17_swizzle(obj)) 1000 if (!i915_gem_object_needs_bit17_swizzle(obj))
1047 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); 1001 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1048 if (ret == -EFAULT) 1002 if (ret == -EFAULT)
1049 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); 1003 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1050
1051out_put:
1052 i915_gem_object_put_pages(obj);
1053 } 1004 }
1054 1005
1055out: 1006out:
1056 drm_gem_object_unreference(obj); 1007 drm_gem_object_unreference(&obj->base);
1057unlock: 1008unlock:
1058 mutex_unlock(&dev->struct_mutex); 1009 mutex_unlock(&dev->struct_mutex);
1059 return ret; 1010 return ret;
@@ -1065,12 +1016,10 @@ unlock:
1065 */ 1016 */
1066int 1017int
1067i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1018i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1068 struct drm_file *file_priv) 1019 struct drm_file *file)
1069{ 1020{
1070 struct drm_i915_private *dev_priv = dev->dev_private;
1071 struct drm_i915_gem_set_domain *args = data; 1021 struct drm_i915_gem_set_domain *args = data;
1072 struct drm_gem_object *obj; 1022 struct drm_i915_gem_object *obj;
1073 struct drm_i915_gem_object *obj_priv;
1074 uint32_t read_domains = args->read_domains; 1023 uint32_t read_domains = args->read_domains;
1075 uint32_t write_domain = args->write_domain; 1024 uint32_t write_domain = args->write_domain;
1076 int ret; 1025 int ret;
@@ -1095,28 +1044,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1095 if (ret) 1044 if (ret)
1096 return ret; 1045 return ret;
1097 1046
1098 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1047 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1099 if (obj == NULL) { 1048 if (obj == NULL) {
1100 ret = -ENOENT; 1049 ret = -ENOENT;
1101 goto unlock; 1050 goto unlock;
1102 } 1051 }
1103 obj_priv = to_intel_bo(obj);
1104
1105 intel_mark_busy(dev, obj);
1106 1052
1107 if (read_domains & I915_GEM_DOMAIN_GTT) { 1053 if (read_domains & I915_GEM_DOMAIN_GTT) {
1108 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1054 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1109 1055
1110 /* Update the LRU on the fence for the CPU access that's
1111 * about to occur.
1112 */
1113 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1114 struct drm_i915_fence_reg *reg =
1115 &dev_priv->fence_regs[obj_priv->fence_reg];
1116 list_move_tail(&reg->lru_list,
1117 &dev_priv->mm.fence_list);
1118 }
1119
1120 /* Silently promote "you're not bound, there was nothing to do" 1056 /* Silently promote "you're not bound, there was nothing to do"
1121 * to success, since the client was just asking us to 1057 * to success, since the client was just asking us to
1122 * make sure everything was done. 1058 * make sure everything was done.
@@ -1127,11 +1063,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1127 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1063 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1128 } 1064 }
1129 1065
1130 /* Maintain LRU order of "inactive" objects */ 1066 drm_gem_object_unreference(&obj->base);
1131 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1132 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1133
1134 drm_gem_object_unreference(obj);
1135unlock: 1067unlock:
1136 mutex_unlock(&dev->struct_mutex); 1068 mutex_unlock(&dev->struct_mutex);
1137 return ret; 1069 return ret;
@@ -1142,10 +1074,10 @@ unlock:
1142 */ 1074 */
1143int 1075int
1144i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1145 struct drm_file *file_priv) 1077 struct drm_file *file)
1146{ 1078{
1147 struct drm_i915_gem_sw_finish *args = data; 1079 struct drm_i915_gem_sw_finish *args = data;
1148 struct drm_gem_object *obj; 1080 struct drm_i915_gem_object *obj;
1149 int ret = 0; 1081 int ret = 0;
1150 1082
1151 if (!(dev->driver->driver_features & DRIVER_GEM)) 1083 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1155,17 +1087,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1155 if (ret) 1087 if (ret)
1156 return ret; 1088 return ret;
1157 1089
1158 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1159 if (obj == NULL) { 1091 if (obj == NULL) {
1160 ret = -ENOENT; 1092 ret = -ENOENT;
1161 goto unlock; 1093 goto unlock;
1162 } 1094 }
1163 1095
1164 /* Pinned buffers may be scanout, so flush the cache */ 1096 /* Pinned buffers may be scanout, so flush the cache */
1165 if (to_intel_bo(obj)->pin_count) 1097 if (obj->pin_count)
1166 i915_gem_object_flush_cpu_write_domain(obj); 1098 i915_gem_object_flush_cpu_write_domain(obj);
1167 1099
1168 drm_gem_object_unreference(obj); 1100 drm_gem_object_unreference(&obj->base);
1169unlock: 1101unlock:
1170 mutex_unlock(&dev->struct_mutex); 1102 mutex_unlock(&dev->struct_mutex);
1171 return ret; 1103 return ret;
@@ -1180,8 +1112,9 @@ unlock:
1180 */ 1112 */
1181int 1113int
1182i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1114i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1183 struct drm_file *file_priv) 1115 struct drm_file *file)
1184{ 1116{
1117 struct drm_i915_private *dev_priv = dev->dev_private;
1185 struct drm_i915_gem_mmap *args = data; 1118 struct drm_i915_gem_mmap *args = data;
1186 struct drm_gem_object *obj; 1119 struct drm_gem_object *obj;
1187 loff_t offset; 1120 loff_t offset;
@@ -1190,10 +1123,15 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1190 if (!(dev->driver->driver_features & DRIVER_GEM)) 1123 if (!(dev->driver->driver_features & DRIVER_GEM))
1191 return -ENODEV; 1124 return -ENODEV;
1192 1125
1193 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1126 obj = drm_gem_object_lookup(dev, file, args->handle);
1194 if (obj == NULL) 1127 if (obj == NULL)
1195 return -ENOENT; 1128 return -ENOENT;
1196 1129
1130 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1131 drm_gem_object_unreference_unlocked(obj);
1132 return -E2BIG;
1133 }
1134
1197 offset = args->offset; 1135 offset = args->offset;
1198 1136
1199 down_write(&current->mm->mmap_sem); 1137 down_write(&current->mm->mmap_sem);
@@ -1228,10 +1166,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1228 */ 1166 */
1229int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1167int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1230{ 1168{
1231 struct drm_gem_object *obj = vma->vm_private_data; 1169 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1232 struct drm_device *dev = obj->dev; 1170 struct drm_device *dev = obj->base.dev;
1233 drm_i915_private_t *dev_priv = dev->dev_private; 1171 drm_i915_private_t *dev_priv = dev->dev_private;
1234 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1235 pgoff_t page_offset; 1172 pgoff_t page_offset;
1236 unsigned long pfn; 1173 unsigned long pfn;
1237 int ret = 0; 1174 int ret = 0;
@@ -1243,27 +1180,35 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1243 1180
1244 /* Now bind it into the GTT if needed */ 1181 /* Now bind it into the GTT if needed */
1245 mutex_lock(&dev->struct_mutex); 1182 mutex_lock(&dev->struct_mutex);
1246 if (!obj_priv->gtt_space) {
1247 ret = i915_gem_object_bind_to_gtt(obj, 0);
1248 if (ret)
1249 goto unlock;
1250 1183
1251 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1184 if (!obj->map_and_fenceable) {
1185 ret = i915_gem_object_unbind(obj);
1252 if (ret) 1186 if (ret)
1253 goto unlock; 1187 goto unlock;
1254 } 1188 }
1255 1189 if (!obj->gtt_space) {
1256 /* Need a new fence register? */ 1190 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1257 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1258 ret = i915_gem_object_get_fence_reg(obj, true);
1259 if (ret) 1191 if (ret)
1260 goto unlock; 1192 goto unlock;
1261 } 1193 }
1262 1194
1263 if (i915_gem_object_is_inactive(obj_priv)) 1195 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1264 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 1196 if (ret)
1197 goto unlock;
1198
1199 if (obj->tiling_mode == I915_TILING_NONE)
1200 ret = i915_gem_object_put_fence(obj);
1201 else
1202 ret = i915_gem_object_get_fence(obj, NULL, true);
1203 if (ret)
1204 goto unlock;
1205
1206 if (i915_gem_object_is_inactive(obj))
1207 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1208
1209 obj->fault_mappable = true;
1265 1210
1266 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1211 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1267 page_offset; 1212 page_offset;
1268 1213
1269 /* Finally, remap it using the new GTT offset */ 1214 /* Finally, remap it using the new GTT offset */
@@ -1272,11 +1217,12 @@ unlock:
1272 mutex_unlock(&dev->struct_mutex); 1217 mutex_unlock(&dev->struct_mutex);
1273 1218
1274 switch (ret) { 1219 switch (ret) {
1220 case -EAGAIN:
1221 set_need_resched();
1275 case 0: 1222 case 0:
1276 case -ERESTARTSYS: 1223 case -ERESTARTSYS:
1277 return VM_FAULT_NOPAGE; 1224 return VM_FAULT_NOPAGE;
1278 case -ENOMEM: 1225 case -ENOMEM:
1279 case -EAGAIN:
1280 return VM_FAULT_OOM; 1226 return VM_FAULT_OOM;
1281 default: 1227 default:
1282 return VM_FAULT_SIGBUS; 1228 return VM_FAULT_SIGBUS;
@@ -1295,37 +1241,39 @@ unlock:
1295 * This routine allocates and attaches a fake offset for @obj. 1241 * This routine allocates and attaches a fake offset for @obj.
1296 */ 1242 */
1297static int 1243static int
1298i915_gem_create_mmap_offset(struct drm_gem_object *obj) 1244i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1299{ 1245{
1300 struct drm_device *dev = obj->dev; 1246 struct drm_device *dev = obj->base.dev;
1301 struct drm_gem_mm *mm = dev->mm_private; 1247 struct drm_gem_mm *mm = dev->mm_private;
1302 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1303 struct drm_map_list *list; 1248 struct drm_map_list *list;
1304 struct drm_local_map *map; 1249 struct drm_local_map *map;
1305 int ret = 0; 1250 int ret = 0;
1306 1251
1307 /* Set the object up for mmap'ing */ 1252 /* Set the object up for mmap'ing */
1308 list = &obj->map_list; 1253 list = &obj->base.map_list;
1309 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); 1254 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1310 if (!list->map) 1255 if (!list->map)
1311 return -ENOMEM; 1256 return -ENOMEM;
1312 1257
1313 map = list->map; 1258 map = list->map;
1314 map->type = _DRM_GEM; 1259 map->type = _DRM_GEM;
1315 map->size = obj->size; 1260 map->size = obj->base.size;
1316 map->handle = obj; 1261 map->handle = obj;
1317 1262
1318 /* Get a DRM GEM mmap offset allocated... */ 1263 /* Get a DRM GEM mmap offset allocated... */
1319 list->file_offset_node = drm_mm_search_free(&mm->offset_manager, 1264 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1320 obj->size / PAGE_SIZE, 0, 0); 1265 obj->base.size / PAGE_SIZE,
1266 0, 0);
1321 if (!list->file_offset_node) { 1267 if (!list->file_offset_node) {
1322 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); 1268 DRM_ERROR("failed to allocate offset for bo %d\n",
1269 obj->base.name);
1323 ret = -ENOSPC; 1270 ret = -ENOSPC;
1324 goto out_free_list; 1271 goto out_free_list;
1325 } 1272 }
1326 1273
1327 list->file_offset_node = drm_mm_get_block(list->file_offset_node, 1274 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1328 obj->size / PAGE_SIZE, 0); 1275 obj->base.size / PAGE_SIZE,
1276 0);
1329 if (!list->file_offset_node) { 1277 if (!list->file_offset_node) {
1330 ret = -ENOMEM; 1278 ret = -ENOMEM;
1331 goto out_free_list; 1279 goto out_free_list;
@@ -1338,16 +1286,13 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1338 goto out_free_mm; 1286 goto out_free_mm;
1339 } 1287 }
1340 1288
1341 /* By now we should be all set, any drm_mmap request on the offset
1342 * below will get to our mmap & fault handler */
1343 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1344
1345 return 0; 1289 return 0;
1346 1290
1347out_free_mm: 1291out_free_mm:
1348 drm_mm_put_block(list->file_offset_node); 1292 drm_mm_put_block(list->file_offset_node);
1349out_free_list: 1293out_free_list:
1350 kfree(list->map); 1294 kfree(list->map);
1295 list->map = NULL;
1351 1296
1352 return ret; 1297 return ret;
1353} 1298}
@@ -1367,38 +1312,51 @@ out_free_list:
1367 * fixup by i915_gem_fault(). 1312 * fixup by i915_gem_fault().
1368 */ 1313 */
1369void 1314void
1370i915_gem_release_mmap(struct drm_gem_object *obj) 1315i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1371{ 1316{
1372 struct drm_device *dev = obj->dev; 1317 if (!obj->fault_mappable)
1373 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1318 return;
1374 1319
1375 if (dev->dev_mapping) 1320 unmap_mapping_range(obj->base.dev->dev_mapping,
1376 unmap_mapping_range(dev->dev_mapping, 1321 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1377 obj_priv->mmap_offset, obj->size, 1); 1322 obj->base.size, 1);
1323
1324 obj->fault_mappable = false;
1378} 1325}
1379 1326
1380static void 1327static void
1381i915_gem_free_mmap_offset(struct drm_gem_object *obj) 1328i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1382{ 1329{
1383 struct drm_device *dev = obj->dev; 1330 struct drm_device *dev = obj->base.dev;
1384 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1385 struct drm_gem_mm *mm = dev->mm_private; 1331 struct drm_gem_mm *mm = dev->mm_private;
1386 struct drm_map_list *list; 1332 struct drm_map_list *list = &obj->base.map_list;
1387 1333
1388 list = &obj->map_list;
1389 drm_ht_remove_item(&mm->offset_hash, &list->hash); 1334 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1335 drm_mm_put_block(list->file_offset_node);
1336 kfree(list->map);
1337 list->map = NULL;
1338}
1390 1339
1391 if (list->file_offset_node) { 1340static uint32_t
1392 drm_mm_put_block(list->file_offset_node); 1341i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1393 list->file_offset_node = NULL; 1342{
1394 } 1343 struct drm_device *dev = obj->base.dev;
1344 uint32_t size;
1395 1345
1396 if (list->map) { 1346 if (INTEL_INFO(dev)->gen >= 4 ||
1397 kfree(list->map); 1347 obj->tiling_mode == I915_TILING_NONE)
1398 list->map = NULL; 1348 return obj->base.size;
1399 } 1349
1350 /* Previous chips need a power-of-two fence region when tiling */
1351 if (INTEL_INFO(dev)->gen == 3)
1352 size = 1024*1024;
1353 else
1354 size = 512*1024;
1355
1356 while (size < obj->base.size)
1357 size <<= 1;
1400 1358
1401 obj_priv->mmap_offset = 0; 1359 return size;
1402} 1360}
1403 1361
1404/** 1362/**
@@ -1406,42 +1364,68 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1406 * @obj: object to check 1364 * @obj: object to check
1407 * 1365 *
1408 * Return the required GTT alignment for an object, taking into account 1366 * Return the required GTT alignment for an object, taking into account
1409 * potential fence register mapping if needed. 1367 * potential fence register mapping.
1410 */ 1368 */
1411static uint32_t 1369static uint32_t
1412i915_gem_get_gtt_alignment(struct drm_gem_object *obj) 1370i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
1413{ 1371{
1414 struct drm_device *dev = obj->dev; 1372 struct drm_device *dev = obj->base.dev;
1415 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1416 int start, i;
1417 1373
1418 /* 1374 /*
1419 * Minimum alignment is 4k (GTT page size), but might be greater 1375 * Minimum alignment is 4k (GTT page size), but might be greater
1420 * if a fence register is needed for the object. 1376 * if a fence register is needed for the object.
1421 */ 1377 */
1422 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE) 1378 if (INTEL_INFO(dev)->gen >= 4 ||
1379 obj->tiling_mode == I915_TILING_NONE)
1423 return 4096; 1380 return 4096;
1424 1381
1425 /* 1382 /*
1426 * Previous chips need to be aligned to the size of the smallest 1383 * Previous chips need to be aligned to the size of the smallest
1427 * fence register that can contain the object. 1384 * fence register that can contain the object.
1428 */ 1385 */
1429 if (INTEL_INFO(dev)->gen == 3) 1386 return i915_gem_get_gtt_size(obj);
1430 start = 1024*1024; 1387}
1431 else 1388
1432 start = 512*1024; 1389/**
1390 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1391 * unfenced object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, only taking into account
1395 * unfenced tiled surface requirements.
1396 */
1397static uint32_t
1398i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
1399{
1400 struct drm_device *dev = obj->base.dev;
1401 int tile_height;
1433 1402
1434 for (i = start; i < obj->size; i <<= 1) 1403 /*
1435 ; 1404 * Minimum alignment is 4k (GTT page size) for sane hw.
1405 */
1406 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1407 obj->tiling_mode == I915_TILING_NONE)
1408 return 4096;
1436 1409
1437 return i; 1410 /*
1411 * Older chips need unfenced tiled buffers to be aligned to the left
1412 * edge of an even tile row (where tile rows are counted as if the bo is
1413 * placed in a fenced gtt region).
1414 */
1415 if (IS_GEN2(dev) ||
1416 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
1417 tile_height = 32;
1418 else
1419 tile_height = 8;
1420
1421 return tile_height * obj->stride * 2;
1438} 1422}
1439 1423
1440/** 1424/**
1441 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1425 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1442 * @dev: DRM device 1426 * @dev: DRM device
1443 * @data: GTT mapping ioctl data 1427 * @data: GTT mapping ioctl data
1444 * @file_priv: GEM object info 1428 * @file: GEM object info
1445 * 1429 *
1446 * Simply returns the fake offset to userspace so it can mmap it. 1430 * Simply returns the fake offset to userspace so it can mmap it.
1447 * The mmap call will end up in drm_gem_mmap(), which will set things 1431 * The mmap call will end up in drm_gem_mmap(), which will set things
@@ -1454,11 +1438,11 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1454 */ 1438 */
1455int 1439int
1456i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1440i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1457 struct drm_file *file_priv) 1441 struct drm_file *file)
1458{ 1442{
1443 struct drm_i915_private *dev_priv = dev->dev_private;
1459 struct drm_i915_gem_mmap_gtt *args = data; 1444 struct drm_i915_gem_mmap_gtt *args = data;
1460 struct drm_gem_object *obj; 1445 struct drm_i915_gem_object *obj;
1461 struct drm_i915_gem_object *obj_priv;
1462 int ret; 1446 int ret;
1463 1447
1464 if (!(dev->driver->driver_features & DRIVER_GEM)) 1448 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1468,130 +1452,196 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1468 if (ret) 1452 if (ret)
1469 return ret; 1453 return ret;
1470 1454
1471 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1455 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1472 if (obj == NULL) { 1456 if (obj == NULL) {
1473 ret = -ENOENT; 1457 ret = -ENOENT;
1474 goto unlock; 1458 goto unlock;
1475 } 1459 }
1476 obj_priv = to_intel_bo(obj);
1477 1460
1478 if (obj_priv->madv != I915_MADV_WILLNEED) { 1461 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1462 ret = -E2BIG;
1463 goto unlock;
1464 }
1465
1466 if (obj->madv != I915_MADV_WILLNEED) {
1479 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1467 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1480 ret = -EINVAL; 1468 ret = -EINVAL;
1481 goto out; 1469 goto out;
1482 } 1470 }
1483 1471
1484 if (!obj_priv->mmap_offset) { 1472 if (!obj->base.map_list.map) {
1485 ret = i915_gem_create_mmap_offset(obj); 1473 ret = i915_gem_create_mmap_offset(obj);
1486 if (ret) 1474 if (ret)
1487 goto out; 1475 goto out;
1488 } 1476 }
1489 1477
1490 args->offset = obj_priv->mmap_offset; 1478 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1491
1492 /*
1493 * Pull it into the GTT so that we have a page list (makes the
1494 * initial fault faster and any subsequent flushing possible).
1495 */
1496 if (!obj_priv->agp_mem) {
1497 ret = i915_gem_object_bind_to_gtt(obj, 0);
1498 if (ret)
1499 goto out;
1500 }
1501 1479
1502out: 1480out:
1503 drm_gem_object_unreference(obj); 1481 drm_gem_object_unreference(&obj->base);
1504unlock: 1482unlock:
1505 mutex_unlock(&dev->struct_mutex); 1483 mutex_unlock(&dev->struct_mutex);
1506 return ret; 1484 return ret;
1507} 1485}
1508 1486
1487static int
1488i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1489 gfp_t gfpmask)
1490{
1491 int page_count, i;
1492 struct address_space *mapping;
1493 struct inode *inode;
1494 struct page *page;
1495
1496 /* Get the list of pages out of our struct file. They'll be pinned
1497 * at this point until we release them.
1498 */
1499 page_count = obj->base.size / PAGE_SIZE;
1500 BUG_ON(obj->pages != NULL);
1501 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1502 if (obj->pages == NULL)
1503 return -ENOMEM;
1504
1505 inode = obj->base.filp->f_path.dentry->d_inode;
1506 mapping = inode->i_mapping;
1507 for (i = 0; i < page_count; i++) {
1508 page = read_cache_page_gfp(mapping, i,
1509 GFP_HIGHUSER |
1510 __GFP_COLD |
1511 __GFP_RECLAIMABLE |
1512 gfpmask);
1513 if (IS_ERR(page))
1514 goto err_pages;
1515
1516 obj->pages[i] = page;
1517 }
1518
1519 if (obj->tiling_mode != I915_TILING_NONE)
1520 i915_gem_object_do_bit_17_swizzle(obj);
1521
1522 return 0;
1523
1524err_pages:
1525 while (i--)
1526 page_cache_release(obj->pages[i]);
1527
1528 drm_free_large(obj->pages);
1529 obj->pages = NULL;
1530 return PTR_ERR(page);
1531}
1532
1509static void 1533static void
1510i915_gem_object_put_pages(struct drm_gem_object *obj) 1534i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1511{ 1535{
1512 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1536 int page_count = obj->base.size / PAGE_SIZE;
1513 int page_count = obj->size / PAGE_SIZE;
1514 int i; 1537 int i;
1515 1538
1516 BUG_ON(obj_priv->pages_refcount == 0); 1539 BUG_ON(obj->madv == __I915_MADV_PURGED);
1517 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1518
1519 if (--obj_priv->pages_refcount != 0)
1520 return;
1521 1540
1522 if (obj_priv->tiling_mode != I915_TILING_NONE) 1541 if (obj->tiling_mode != I915_TILING_NONE)
1523 i915_gem_object_save_bit_17_swizzle(obj); 1542 i915_gem_object_save_bit_17_swizzle(obj);
1524 1543
1525 if (obj_priv->madv == I915_MADV_DONTNEED) 1544 if (obj->madv == I915_MADV_DONTNEED)
1526 obj_priv->dirty = 0; 1545 obj->dirty = 0;
1527 1546
1528 for (i = 0; i < page_count; i++) { 1547 for (i = 0; i < page_count; i++) {
1529 if (obj_priv->dirty) 1548 if (obj->dirty)
1530 set_page_dirty(obj_priv->pages[i]); 1549 set_page_dirty(obj->pages[i]);
1531 1550
1532 if (obj_priv->madv == I915_MADV_WILLNEED) 1551 if (obj->madv == I915_MADV_WILLNEED)
1533 mark_page_accessed(obj_priv->pages[i]); 1552 mark_page_accessed(obj->pages[i]);
1534 1553
1535 page_cache_release(obj_priv->pages[i]); 1554 page_cache_release(obj->pages[i]);
1536 } 1555 }
1537 obj_priv->dirty = 0; 1556 obj->dirty = 0;
1538 1557
1539 drm_free_large(obj_priv->pages); 1558 drm_free_large(obj->pages);
1540 obj_priv->pages = NULL; 1559 obj->pages = NULL;
1541} 1560}
1542 1561
1543static uint32_t 1562void
1544i915_gem_next_request_seqno(struct drm_device *dev, 1563i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1545 struct intel_ring_buffer *ring) 1564 struct intel_ring_buffer *ring,
1546{ 1565 u32 seqno)
1547 drm_i915_private_t *dev_priv = dev->dev_private;
1548
1549 ring->outstanding_lazy_request = true;
1550 return dev_priv->next_seqno;
1551}
1552
1553static void
1554i915_gem_object_move_to_active(struct drm_gem_object *obj,
1555 struct intel_ring_buffer *ring)
1556{ 1566{
1557 struct drm_device *dev = obj->dev; 1567 struct drm_device *dev = obj->base.dev;
1558 struct drm_i915_private *dev_priv = dev->dev_private; 1568 struct drm_i915_private *dev_priv = dev->dev_private;
1559 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1560 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1561 1569
1562 BUG_ON(ring == NULL); 1570 BUG_ON(ring == NULL);
1563 obj_priv->ring = ring; 1571 obj->ring = ring;
1564 1572
1565 /* Add a reference if we're newly entering the active list. */ 1573 /* Add a reference if we're newly entering the active list. */
1566 if (!obj_priv->active) { 1574 if (!obj->active) {
1567 drm_gem_object_reference(obj); 1575 drm_gem_object_reference(&obj->base);
1568 obj_priv->active = 1; 1576 obj->active = 1;
1569 } 1577 }
1570 1578
1571 /* Move from whatever list we were on to the tail of execution. */ 1579 /* Move from whatever list we were on to the tail of execution. */
1572 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list); 1580 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1573 list_move_tail(&obj_priv->ring_list, &ring->active_list); 1581 list_move_tail(&obj->ring_list, &ring->active_list);
1574 obj_priv->last_rendering_seqno = seqno; 1582
1583 obj->last_rendering_seqno = seqno;
1584 if (obj->fenced_gpu_access) {
1585 struct drm_i915_fence_reg *reg;
1586
1587 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1588
1589 obj->last_fenced_seqno = seqno;
1590 obj->last_fenced_ring = ring;
1591
1592 reg = &dev_priv->fence_regs[obj->fence_reg];
1593 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1594 }
1575} 1595}
1576 1596
1577static void 1597static void
1578i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 1598i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1579{ 1599{
1580 struct drm_device *dev = obj->dev; 1600 list_del_init(&obj->ring_list);
1601 obj->last_rendering_seqno = 0;
1602}
1603
1604static void
1605i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1606{
1607 struct drm_device *dev = obj->base.dev;
1581 drm_i915_private_t *dev_priv = dev->dev_private; 1608 drm_i915_private_t *dev_priv = dev->dev_private;
1582 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1583 1609
1584 BUG_ON(!obj_priv->active); 1610 BUG_ON(!obj->active);
1585 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list); 1611 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1586 list_del_init(&obj_priv->ring_list); 1612
1587 obj_priv->last_rendering_seqno = 0; 1613 i915_gem_object_move_off_active(obj);
1614}
1615
1616static void
1617i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1618{
1619 struct drm_device *dev = obj->base.dev;
1620 struct drm_i915_private *dev_priv = dev->dev_private;
1621
1622 if (obj->pin_count != 0)
1623 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1624 else
1625 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1626
1627 BUG_ON(!list_empty(&obj->gpu_write_list));
1628 BUG_ON(!obj->active);
1629 obj->ring = NULL;
1630
1631 i915_gem_object_move_off_active(obj);
1632 obj->fenced_gpu_access = false;
1633
1634 obj->active = 0;
1635 obj->pending_gpu_write = false;
1636 drm_gem_object_unreference(&obj->base);
1637
1638 WARN_ON(i915_verify_lists(dev));
1588} 1639}
1589 1640
1590/* Immediately discard the backing storage */ 1641/* Immediately discard the backing storage */
1591static void 1642static void
1592i915_gem_object_truncate(struct drm_gem_object *obj) 1643i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1593{ 1644{
1594 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1595 struct inode *inode; 1645 struct inode *inode;
1596 1646
1597 /* Our goal here is to return as much of the memory as 1647 /* Our goal here is to return as much of the memory as
@@ -1600,42 +1650,18 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
1600 * backing pages, *now*. Here we mirror the actions taken 1650 * backing pages, *now*. Here we mirror the actions taken
1601 * when by shmem_delete_inode() to release the backing store. 1651 * when by shmem_delete_inode() to release the backing store.
1602 */ 1652 */
1603 inode = obj->filp->f_path.dentry->d_inode; 1653 inode = obj->base.filp->f_path.dentry->d_inode;
1604 truncate_inode_pages(inode->i_mapping, 0); 1654 truncate_inode_pages(inode->i_mapping, 0);
1605 if (inode->i_op->truncate_range) 1655 if (inode->i_op->truncate_range)
1606 inode->i_op->truncate_range(inode, 0, (loff_t)-1); 1656 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1607 1657
1608 obj_priv->madv = __I915_MADV_PURGED; 1658 obj->madv = __I915_MADV_PURGED;
1609} 1659}
1610 1660
1611static inline int 1661static inline int
1612i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) 1662i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1613{ 1663{
1614 return obj_priv->madv == I915_MADV_DONTNEED; 1664 return obj->madv == I915_MADV_DONTNEED;
1615}
1616
1617static void
1618i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1619{
1620 struct drm_device *dev = obj->dev;
1621 drm_i915_private_t *dev_priv = dev->dev_private;
1622 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1623
1624 if (obj_priv->pin_count != 0)
1625 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1626 else
1627 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1628 list_del_init(&obj_priv->ring_list);
1629
1630 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1631
1632 obj_priv->last_rendering_seqno = 0;
1633 obj_priv->ring = NULL;
1634 if (obj_priv->active) {
1635 obj_priv->active = 0;
1636 drm_gem_object_unreference(obj);
1637 }
1638 WARN_ON(i915_verify_lists(dev));
1639} 1665}
1640 1666
1641static void 1667static void
@@ -1643,37 +1669,27 @@ i915_gem_process_flushing_list(struct drm_device *dev,
1643 uint32_t flush_domains, 1669 uint32_t flush_domains,
1644 struct intel_ring_buffer *ring) 1670 struct intel_ring_buffer *ring)
1645{ 1671{
1646 drm_i915_private_t *dev_priv = dev->dev_private; 1672 struct drm_i915_gem_object *obj, *next;
1647 struct drm_i915_gem_object *obj_priv, *next;
1648 1673
1649 list_for_each_entry_safe(obj_priv, next, 1674 list_for_each_entry_safe(obj, next,
1650 &ring->gpu_write_list, 1675 &ring->gpu_write_list,
1651 gpu_write_list) { 1676 gpu_write_list) {
1652 struct drm_gem_object *obj = &obj_priv->base; 1677 if (obj->base.write_domain & flush_domains) {
1678 uint32_t old_write_domain = obj->base.write_domain;
1653 1679
1654 if (obj->write_domain & flush_domains) { 1680 obj->base.write_domain = 0;
1655 uint32_t old_write_domain = obj->write_domain; 1681 list_del_init(&obj->gpu_write_list);
1656 1682 i915_gem_object_move_to_active(obj, ring,
1657 obj->write_domain = 0; 1683 i915_gem_next_request_seqno(dev, ring));
1658 list_del_init(&obj_priv->gpu_write_list);
1659 i915_gem_object_move_to_active(obj, ring);
1660
1661 /* update the fence lru list */
1662 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1663 struct drm_i915_fence_reg *reg =
1664 &dev_priv->fence_regs[obj_priv->fence_reg];
1665 list_move_tail(&reg->lru_list,
1666 &dev_priv->mm.fence_list);
1667 }
1668 1684
1669 trace_i915_gem_object_change_domain(obj, 1685 trace_i915_gem_object_change_domain(obj,
1670 obj->read_domains, 1686 obj->base.read_domains,
1671 old_write_domain); 1687 old_write_domain);
1672 } 1688 }
1673 } 1689 }
1674} 1690}
1675 1691
1676uint32_t 1692int
1677i915_add_request(struct drm_device *dev, 1693i915_add_request(struct drm_device *dev,
1678 struct drm_file *file, 1694 struct drm_file *file,
1679 struct drm_i915_gem_request *request, 1695 struct drm_i915_gem_request *request,
@@ -1683,17 +1699,17 @@ i915_add_request(struct drm_device *dev,
1683 struct drm_i915_file_private *file_priv = NULL; 1699 struct drm_i915_file_private *file_priv = NULL;
1684 uint32_t seqno; 1700 uint32_t seqno;
1685 int was_empty; 1701 int was_empty;
1702 int ret;
1703
1704 BUG_ON(request == NULL);
1686 1705
1687 if (file != NULL) 1706 if (file != NULL)
1688 file_priv = file->driver_priv; 1707 file_priv = file->driver_priv;
1689 1708
1690 if (request == NULL) { 1709 ret = ring->add_request(ring, &seqno);
1691 request = kzalloc(sizeof(*request), GFP_KERNEL); 1710 if (ret)
1692 if (request == NULL) 1711 return ret;
1693 return 0;
1694 }
1695 1712
1696 seqno = ring->add_request(dev, ring, 0);
1697 ring->outstanding_lazy_request = false; 1713 ring->outstanding_lazy_request = false;
1698 1714
1699 request->seqno = seqno; 1715 request->seqno = seqno;
@@ -1717,26 +1733,7 @@ i915_add_request(struct drm_device *dev,
1717 queue_delayed_work(dev_priv->wq, 1733 queue_delayed_work(dev_priv->wq,
1718 &dev_priv->mm.retire_work, HZ); 1734 &dev_priv->mm.retire_work, HZ);
1719 } 1735 }
1720 return seqno; 1736 return 0;
1721}
1722
1723/**
1724 * Command execution barrier
1725 *
1726 * Ensures that all commands in the ring are finished
1727 * before signalling the CPU
1728 */
1729static void
1730i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1731{
1732 uint32_t flush_domains = 0;
1733
1734 /* The sampler always gets flushed on i965 (sigh) */
1735 if (INTEL_INFO(dev)->gen >= 4)
1736 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1737
1738 ring->flush(dev, ring,
1739 I915_GEM_DOMAIN_COMMAND, flush_domains);
1740} 1737}
1741 1738
1742static inline void 1739static inline void
@@ -1769,62 +1766,76 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1769 } 1766 }
1770 1767
1771 while (!list_empty(&ring->active_list)) { 1768 while (!list_empty(&ring->active_list)) {
1772 struct drm_i915_gem_object *obj_priv; 1769 struct drm_i915_gem_object *obj;
1773 1770
1774 obj_priv = list_first_entry(&ring->active_list, 1771 obj = list_first_entry(&ring->active_list,
1775 struct drm_i915_gem_object, 1772 struct drm_i915_gem_object,
1776 ring_list); 1773 ring_list);
1777 1774
1778 obj_priv->base.write_domain = 0; 1775 obj->base.write_domain = 0;
1779 list_del_init(&obj_priv->gpu_write_list); 1776 list_del_init(&obj->gpu_write_list);
1780 i915_gem_object_move_to_inactive(&obj_priv->base); 1777 i915_gem_object_move_to_inactive(obj);
1778 }
1779}
1780
1781static void i915_gem_reset_fences(struct drm_device *dev)
1782{
1783 struct drm_i915_private *dev_priv = dev->dev_private;
1784 int i;
1785
1786 for (i = 0; i < 16; i++) {
1787 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1788 struct drm_i915_gem_object *obj = reg->obj;
1789
1790 if (!obj)
1791 continue;
1792
1793 if (obj->tiling_mode)
1794 i915_gem_release_mmap(obj);
1795
1796 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1797 reg->obj->fenced_gpu_access = false;
1798 reg->obj->last_fenced_seqno = 0;
1799 reg->obj->last_fenced_ring = NULL;
1800 i915_gem_clear_fence_reg(dev, reg);
1781 } 1801 }
1782} 1802}
1783 1803
1784void i915_gem_reset(struct drm_device *dev) 1804void i915_gem_reset(struct drm_device *dev)
1785{ 1805{
1786 struct drm_i915_private *dev_priv = dev->dev_private; 1806 struct drm_i915_private *dev_priv = dev->dev_private;
1787 struct drm_i915_gem_object *obj_priv; 1807 struct drm_i915_gem_object *obj;
1788 int i; 1808 int i;
1789 1809
1790 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); 1810 for (i = 0; i < I915_NUM_RINGS; i++)
1791 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); 1811 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1792 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1793 1812
1794 /* Remove anything from the flushing lists. The GPU cache is likely 1813 /* Remove anything from the flushing lists. The GPU cache is likely
1795 * to be lost on reset along with the data, so simply move the 1814 * to be lost on reset along with the data, so simply move the
1796 * lost bo to the inactive list. 1815 * lost bo to the inactive list.
1797 */ 1816 */
1798 while (!list_empty(&dev_priv->mm.flushing_list)) { 1817 while (!list_empty(&dev_priv->mm.flushing_list)) {
1799 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 1818 obj= list_first_entry(&dev_priv->mm.flushing_list,
1800 struct drm_i915_gem_object, 1819 struct drm_i915_gem_object,
1801 mm_list); 1820 mm_list);
1802 1821
1803 obj_priv->base.write_domain = 0; 1822 obj->base.write_domain = 0;
1804 list_del_init(&obj_priv->gpu_write_list); 1823 list_del_init(&obj->gpu_write_list);
1805 i915_gem_object_move_to_inactive(&obj_priv->base); 1824 i915_gem_object_move_to_inactive(obj);
1806 } 1825 }
1807 1826
1808 /* Move everything out of the GPU domains to ensure we do any 1827 /* Move everything out of the GPU domains to ensure we do any
1809 * necessary invalidation upon reuse. 1828 * necessary invalidation upon reuse.
1810 */ 1829 */
1811 list_for_each_entry(obj_priv, 1830 list_for_each_entry(obj,
1812 &dev_priv->mm.inactive_list, 1831 &dev_priv->mm.inactive_list,
1813 mm_list) 1832 mm_list)
1814 { 1833 {
1815 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1834 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1816 } 1835 }
1817 1836
1818 /* The fence registers are invalidated so clear them out */ 1837 /* The fence registers are invalidated so clear them out */
1819 for (i = 0; i < 16; i++) { 1838 i915_gem_reset_fences(dev);
1820 struct drm_i915_fence_reg *reg;
1821
1822 reg = &dev_priv->fence_regs[i];
1823 if (!reg->obj)
1824 continue;
1825
1826 i915_gem_clear_fence_reg(reg->obj);
1827 }
1828} 1839}
1829 1840
1830/** 1841/**
@@ -1836,6 +1847,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1836{ 1847{
1837 drm_i915_private_t *dev_priv = dev->dev_private; 1848 drm_i915_private_t *dev_priv = dev->dev_private;
1838 uint32_t seqno; 1849 uint32_t seqno;
1850 int i;
1839 1851
1840 if (!ring->status_page.page_addr || 1852 if (!ring->status_page.page_addr ||
1841 list_empty(&ring->request_list)) 1853 list_empty(&ring->request_list))
@@ -1843,7 +1855,12 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1843 1855
1844 WARN_ON(i915_verify_lists(dev)); 1856 WARN_ON(i915_verify_lists(dev));
1845 1857
1846 seqno = ring->get_seqno(dev, ring); 1858 seqno = ring->get_seqno(ring);
1859
1860 for (i = 0; i < I915_NUM_RINGS; i++)
1861 if (seqno >= ring->sync_seqno[i])
1862 ring->sync_seqno[i] = 0;
1863
1847 while (!list_empty(&ring->request_list)) { 1864 while (!list_empty(&ring->request_list)) {
1848 struct drm_i915_gem_request *request; 1865 struct drm_i915_gem_request *request;
1849 1866
@@ -1865,18 +1882,16 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1865 * by the ringbuffer to the flushing/inactive lists as appropriate. 1882 * by the ringbuffer to the flushing/inactive lists as appropriate.
1866 */ 1883 */
1867 while (!list_empty(&ring->active_list)) { 1884 while (!list_empty(&ring->active_list)) {
1868 struct drm_gem_object *obj; 1885 struct drm_i915_gem_object *obj;
1869 struct drm_i915_gem_object *obj_priv;
1870 1886
1871 obj_priv = list_first_entry(&ring->active_list, 1887 obj= list_first_entry(&ring->active_list,
1872 struct drm_i915_gem_object, 1888 struct drm_i915_gem_object,
1873 ring_list); 1889 ring_list);
1874 1890
1875 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) 1891 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1876 break; 1892 break;
1877 1893
1878 obj = &obj_priv->base; 1894 if (obj->base.write_domain != 0)
1879 if (obj->write_domain != 0)
1880 i915_gem_object_move_to_flushing(obj); 1895 i915_gem_object_move_to_flushing(obj);
1881 else 1896 else
1882 i915_gem_object_move_to_inactive(obj); 1897 i915_gem_object_move_to_inactive(obj);
@@ -1884,7 +1899,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1884 1899
1885 if (unlikely (dev_priv->trace_irq_seqno && 1900 if (unlikely (dev_priv->trace_irq_seqno &&
1886 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { 1901 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1887 ring->user_irq_put(dev, ring); 1902 ring->irq_put(ring);
1888 dev_priv->trace_irq_seqno = 0; 1903 dev_priv->trace_irq_seqno = 0;
1889 } 1904 }
1890 1905
@@ -1895,24 +1910,24 @@ void
1895i915_gem_retire_requests(struct drm_device *dev) 1910i915_gem_retire_requests(struct drm_device *dev)
1896{ 1911{
1897 drm_i915_private_t *dev_priv = dev->dev_private; 1912 drm_i915_private_t *dev_priv = dev->dev_private;
1913 int i;
1898 1914
1899 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 1915 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1900 struct drm_i915_gem_object *obj_priv, *tmp; 1916 struct drm_i915_gem_object *obj, *next;
1901 1917
1902 /* We must be careful that during unbind() we do not 1918 /* We must be careful that during unbind() we do not
1903 * accidentally infinitely recurse into retire requests. 1919 * accidentally infinitely recurse into retire requests.
1904 * Currently: 1920 * Currently:
1905 * retire -> free -> unbind -> wait -> retire_ring 1921 * retire -> free -> unbind -> wait -> retire_ring
1906 */ 1922 */
1907 list_for_each_entry_safe(obj_priv, tmp, 1923 list_for_each_entry_safe(obj, next,
1908 &dev_priv->mm.deferred_free_list, 1924 &dev_priv->mm.deferred_free_list,
1909 mm_list) 1925 mm_list)
1910 i915_gem_free_object_tail(&obj_priv->base); 1926 i915_gem_free_object_tail(obj);
1911 } 1927 }
1912 1928
1913 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); 1929 for (i = 0; i < I915_NUM_RINGS; i++)
1914 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); 1930 i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
1915 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1916} 1931}
1917 1932
1918static void 1933static void
@@ -1934,9 +1949,9 @@ i915_gem_retire_work_handler(struct work_struct *work)
1934 i915_gem_retire_requests(dev); 1949 i915_gem_retire_requests(dev);
1935 1950
1936 if (!dev_priv->mm.suspended && 1951 if (!dev_priv->mm.suspended &&
1937 (!list_empty(&dev_priv->render_ring.request_list) || 1952 (!list_empty(&dev_priv->ring[RCS].request_list) ||
1938 !list_empty(&dev_priv->bsd_ring.request_list) || 1953 !list_empty(&dev_priv->ring[VCS].request_list) ||
1939 !list_empty(&dev_priv->blt_ring.request_list))) 1954 !list_empty(&dev_priv->ring[BCS].request_list)))
1940 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 1955 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1941 mutex_unlock(&dev->struct_mutex); 1956 mutex_unlock(&dev->struct_mutex);
1942} 1957}
@@ -1954,14 +1969,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1954 if (atomic_read(&dev_priv->mm.wedged)) 1969 if (atomic_read(&dev_priv->mm.wedged))
1955 return -EAGAIN; 1970 return -EAGAIN;
1956 1971
1957 if (ring->outstanding_lazy_request) { 1972 if (seqno == ring->outstanding_lazy_request) {
1958 seqno = i915_add_request(dev, NULL, NULL, ring); 1973 struct drm_i915_gem_request *request;
1959 if (seqno == 0) 1974
1975 request = kzalloc(sizeof(*request), GFP_KERNEL);
1976 if (request == NULL)
1960 return -ENOMEM; 1977 return -ENOMEM;
1978
1979 ret = i915_add_request(dev, NULL, request, ring);
1980 if (ret) {
1981 kfree(request);
1982 return ret;
1983 }
1984
1985 seqno = request->seqno;
1961 } 1986 }
1962 BUG_ON(seqno == dev_priv->next_seqno);
1963 1987
1964 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 1988 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
1965 if (HAS_PCH_SPLIT(dev)) 1989 if (HAS_PCH_SPLIT(dev))
1966 ier = I915_READ(DEIER) | I915_READ(GTIER); 1990 ier = I915_READ(DEIER) | I915_READ(GTIER);
1967 else 1991 else
@@ -1975,21 +1999,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1975 1999
1976 trace_i915_gem_request_wait_begin(dev, seqno); 2000 trace_i915_gem_request_wait_begin(dev, seqno);
1977 2001
1978 ring->waiting_gem_seqno = seqno; 2002 ring->waiting_seqno = seqno;
1979 ring->user_irq_get(dev, ring); 2003 if (ring->irq_get(ring)) {
1980 if (interruptible) 2004 if (interruptible)
1981 ret = wait_event_interruptible(ring->irq_queue, 2005 ret = wait_event_interruptible(ring->irq_queue,
1982 i915_seqno_passed( 2006 i915_seqno_passed(ring->get_seqno(ring), seqno)
1983 ring->get_seqno(dev, ring), seqno) 2007 || atomic_read(&dev_priv->mm.wedged));
1984 || atomic_read(&dev_priv->mm.wedged)); 2008 else
1985 else 2009 wait_event(ring->irq_queue,
1986 wait_event(ring->irq_queue, 2010 i915_seqno_passed(ring->get_seqno(ring), seqno)
1987 i915_seqno_passed( 2011 || atomic_read(&dev_priv->mm.wedged));
1988 ring->get_seqno(dev, ring), seqno)
1989 || atomic_read(&dev_priv->mm.wedged));
1990 2012
1991 ring->user_irq_put(dev, ring); 2013 ring->irq_put(ring);
1992 ring->waiting_gem_seqno = 0; 2014 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2015 seqno) ||
2016 atomic_read(&dev_priv->mm.wedged), 3000))
2017 ret = -EBUSY;
2018 ring->waiting_seqno = 0;
1993 2019
1994 trace_i915_gem_request_wait_end(dev, seqno); 2020 trace_i915_gem_request_wait_end(dev, seqno);
1995 } 2021 }
@@ -1998,7 +2024,7 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1998 2024
1999 if (ret && ret != -ERESTARTSYS) 2025 if (ret && ret != -ERESTARTSYS)
2000 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", 2026 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2001 __func__, ret, seqno, ring->get_seqno(dev, ring), 2027 __func__, ret, seqno, ring->get_seqno(ring),
2002 dev_priv->next_seqno); 2028 dev_priv->next_seqno);
2003 2029
2004 /* Directly dispatch request retiring. While we have the work queue 2030 /* Directly dispatch request retiring. While we have the work queue
@@ -2023,70 +2049,30 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno,
2023 return i915_do_wait_request(dev, seqno, 1, ring); 2049 return i915_do_wait_request(dev, seqno, 1, ring);
2024} 2050}
2025 2051
2026static void
2027i915_gem_flush_ring(struct drm_device *dev,
2028 struct drm_file *file_priv,
2029 struct intel_ring_buffer *ring,
2030 uint32_t invalidate_domains,
2031 uint32_t flush_domains)
2032{
2033 ring->flush(dev, ring, invalidate_domains, flush_domains);
2034 i915_gem_process_flushing_list(dev, flush_domains, ring);
2035}
2036
2037static void
2038i915_gem_flush(struct drm_device *dev,
2039 struct drm_file *file_priv,
2040 uint32_t invalidate_domains,
2041 uint32_t flush_domains,
2042 uint32_t flush_rings)
2043{
2044 drm_i915_private_t *dev_priv = dev->dev_private;
2045
2046 if (flush_domains & I915_GEM_DOMAIN_CPU)
2047 drm_agp_chipset_flush(dev);
2048
2049 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2050 if (flush_rings & RING_RENDER)
2051 i915_gem_flush_ring(dev, file_priv,
2052 &dev_priv->render_ring,
2053 invalidate_domains, flush_domains);
2054 if (flush_rings & RING_BSD)
2055 i915_gem_flush_ring(dev, file_priv,
2056 &dev_priv->bsd_ring,
2057 invalidate_domains, flush_domains);
2058 if (flush_rings & RING_BLT)
2059 i915_gem_flush_ring(dev, file_priv,
2060 &dev_priv->blt_ring,
2061 invalidate_domains, flush_domains);
2062 }
2063}
2064
2065/** 2052/**
2066 * Ensures that all rendering to the object has completed and the object is 2053 * Ensures that all rendering to the object has completed and the object is
2067 * safe to unbind from the GTT or access from the CPU. 2054 * safe to unbind from the GTT or access from the CPU.
2068 */ 2055 */
2069static int 2056int
2070i915_gem_object_wait_rendering(struct drm_gem_object *obj, 2057i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2071 bool interruptible) 2058 bool interruptible)
2072{ 2059{
2073 struct drm_device *dev = obj->dev; 2060 struct drm_device *dev = obj->base.dev;
2074 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2075 int ret; 2061 int ret;
2076 2062
2077 /* This function only exists to support waiting for existing rendering, 2063 /* This function only exists to support waiting for existing rendering,
2078 * not for emitting required flushes. 2064 * not for emitting required flushes.
2079 */ 2065 */
2080 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); 2066 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2081 2067
2082 /* If there is rendering queued on the buffer being evicted, wait for 2068 /* If there is rendering queued on the buffer being evicted, wait for
2083 * it. 2069 * it.
2084 */ 2070 */
2085 if (obj_priv->active) { 2071 if (obj->active) {
2086 ret = i915_do_wait_request(dev, 2072 ret = i915_do_wait_request(dev,
2087 obj_priv->last_rendering_seqno, 2073 obj->last_rendering_seqno,
2088 interruptible, 2074 interruptible,
2089 obj_priv->ring); 2075 obj->ring);
2090 if (ret) 2076 if (ret)
2091 return ret; 2077 return ret;
2092 } 2078 }
@@ -2098,17 +2084,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2098 * Unbinds an object from the GTT aperture. 2084 * Unbinds an object from the GTT aperture.
2099 */ 2085 */
2100int 2086int
2101i915_gem_object_unbind(struct drm_gem_object *obj) 2087i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2102{ 2088{
2103 struct drm_device *dev = obj->dev;
2104 struct drm_i915_private *dev_priv = dev->dev_private;
2105 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2106 int ret = 0; 2089 int ret = 0;
2107 2090
2108 if (obj_priv->gtt_space == NULL) 2091 if (obj->gtt_space == NULL)
2109 return 0; 2092 return 0;
2110 2093
2111 if (obj_priv->pin_count != 0) { 2094 if (obj->pin_count != 0) {
2112 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2095 DRM_ERROR("Attempting to unbind pinned buffer\n");
2113 return -EINVAL; 2096 return -EINVAL;
2114 } 2097 }
@@ -2131,27 +2114,27 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2131 */ 2114 */
2132 if (ret) { 2115 if (ret) {
2133 i915_gem_clflush_object(obj); 2116 i915_gem_clflush_object(obj);
2134 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; 2117 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2135 } 2118 }
2136 2119
2137 /* release the fence reg _after_ flushing */ 2120 /* release the fence reg _after_ flushing */
2138 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 2121 ret = i915_gem_object_put_fence(obj);
2139 i915_gem_clear_fence_reg(obj); 2122 if (ret == -ERESTARTSYS)
2140 2123 return ret;
2141 drm_unbind_agp(obj_priv->agp_mem);
2142 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2143 2124
2144 i915_gem_object_put_pages(obj); 2125 i915_gem_gtt_unbind_object(obj);
2145 BUG_ON(obj_priv->pages_refcount); 2126 i915_gem_object_put_pages_gtt(obj);
2146 2127
2147 i915_gem_info_remove_gtt(dev_priv, obj->size); 2128 list_del_init(&obj->gtt_list);
2148 list_del_init(&obj_priv->mm_list); 2129 list_del_init(&obj->mm_list);
2130 /* Avoid an unnecessary call to unbind on rebind. */
2131 obj->map_and_fenceable = true;
2149 2132
2150 drm_mm_put_block(obj_priv->gtt_space); 2133 drm_mm_put_block(obj->gtt_space);
2151 obj_priv->gtt_space = NULL; 2134 obj->gtt_space = NULL;
2152 obj_priv->gtt_offset = 0; 2135 obj->gtt_offset = 0;
2153 2136
2154 if (i915_gem_object_is_purgeable(obj_priv)) 2137 if (i915_gem_object_is_purgeable(obj))
2155 i915_gem_object_truncate(obj); 2138 i915_gem_object_truncate(obj);
2156 2139
2157 trace_i915_gem_object_unbind(obj); 2140 trace_i915_gem_object_unbind(obj);
@@ -2159,14 +2142,25 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2159 return ret; 2142 return ret;
2160} 2143}
2161 2144
2145void
2146i915_gem_flush_ring(struct drm_device *dev,
2147 struct intel_ring_buffer *ring,
2148 uint32_t invalidate_domains,
2149 uint32_t flush_domains)
2150{
2151 ring->flush(ring, invalidate_domains, flush_domains);
2152 i915_gem_process_flushing_list(dev, flush_domains, ring);
2153}
2154
2162static int i915_ring_idle(struct drm_device *dev, 2155static int i915_ring_idle(struct drm_device *dev,
2163 struct intel_ring_buffer *ring) 2156 struct intel_ring_buffer *ring)
2164{ 2157{
2165 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2158 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2166 return 0; 2159 return 0;
2167 2160
2168 i915_gem_flush_ring(dev, NULL, ring, 2161 if (!list_empty(&ring->gpu_write_list))
2169 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 2162 i915_gem_flush_ring(dev, ring,
2163 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2170 return i915_wait_request(dev, 2164 return i915_wait_request(dev,
2171 i915_gem_next_request_seqno(dev, ring), 2165 i915_gem_next_request_seqno(dev, ring),
2172 ring); 2166 ring);
@@ -2177,7 +2171,7 @@ i915_gpu_idle(struct drm_device *dev)
2177{ 2171{
2178 drm_i915_private_t *dev_priv = dev->dev_private; 2172 drm_i915_private_t *dev_priv = dev->dev_private;
2179 bool lists_empty; 2173 bool lists_empty;
2180 int ret; 2174 int ret, i;
2181 2175
2182 lists_empty = (list_empty(&dev_priv->mm.flushing_list) && 2176 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2183 list_empty(&dev_priv->mm.active_list)); 2177 list_empty(&dev_priv->mm.active_list));
@@ -2185,258 +2179,296 @@ i915_gpu_idle(struct drm_device *dev)
2185 return 0; 2179 return 0;
2186 2180
2187 /* Flush everything onto the inactive list. */ 2181 /* Flush everything onto the inactive list. */
2188 ret = i915_ring_idle(dev, &dev_priv->render_ring); 2182 for (i = 0; i < I915_NUM_RINGS; i++) {
2189 if (ret) 2183 ret = i915_ring_idle(dev, &dev_priv->ring[i]);
2190 return ret; 2184 if (ret)
2191 2185 return ret;
2192 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2193 if (ret)
2194 return ret;
2195
2196 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2197 if (ret)
2198 return ret;
2199
2200 return 0;
2201}
2202
2203static int
2204i915_gem_object_get_pages(struct drm_gem_object *obj,
2205 gfp_t gfpmask)
2206{
2207 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2208 int page_count, i;
2209 struct address_space *mapping;
2210 struct inode *inode;
2211 struct page *page;
2212
2213 BUG_ON(obj_priv->pages_refcount
2214 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2215
2216 if (obj_priv->pages_refcount++ != 0)
2217 return 0;
2218
2219 /* Get the list of pages out of our struct file. They'll be pinned
2220 * at this point until we release them.
2221 */
2222 page_count = obj->size / PAGE_SIZE;
2223 BUG_ON(obj_priv->pages != NULL);
2224 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2225 if (obj_priv->pages == NULL) {
2226 obj_priv->pages_refcount--;
2227 return -ENOMEM;
2228 }
2229
2230 inode = obj->filp->f_path.dentry->d_inode;
2231 mapping = inode->i_mapping;
2232 for (i = 0; i < page_count; i++) {
2233 page = read_cache_page_gfp(mapping, i,
2234 GFP_HIGHUSER |
2235 __GFP_COLD |
2236 __GFP_RECLAIMABLE |
2237 gfpmask);
2238 if (IS_ERR(page))
2239 goto err_pages;
2240
2241 obj_priv->pages[i] = page;
2242 } 2186 }
2243 2187
2244 if (obj_priv->tiling_mode != I915_TILING_NONE)
2245 i915_gem_object_do_bit_17_swizzle(obj);
2246
2247 return 0; 2188 return 0;
2248
2249err_pages:
2250 while (i--)
2251 page_cache_release(obj_priv->pages[i]);
2252
2253 drm_free_large(obj_priv->pages);
2254 obj_priv->pages = NULL;
2255 obj_priv->pages_refcount--;
2256 return PTR_ERR(page);
2257} 2189}
2258 2190
2259static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) 2191static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2192 struct intel_ring_buffer *pipelined)
2260{ 2193{
2261 struct drm_gem_object *obj = reg->obj; 2194 struct drm_device *dev = obj->base.dev;
2262 struct drm_device *dev = obj->dev;
2263 drm_i915_private_t *dev_priv = dev->dev_private; 2195 drm_i915_private_t *dev_priv = dev->dev_private;
2264 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2196 u32 size = obj->gtt_space->size;
2265 int regnum = obj_priv->fence_reg; 2197 int regnum = obj->fence_reg;
2266 uint64_t val; 2198 uint64_t val;
2267 2199
2268 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2200 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2269 0xfffff000) << 32; 2201 0xfffff000) << 32;
2270 val |= obj_priv->gtt_offset & 0xfffff000; 2202 val |= obj->gtt_offset & 0xfffff000;
2271 val |= (uint64_t)((obj_priv->stride / 128) - 1) << 2203 val |= (uint64_t)((obj->stride / 128) - 1) <<
2272 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2204 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2273 2205
2274 if (obj_priv->tiling_mode == I915_TILING_Y) 2206 if (obj->tiling_mode == I915_TILING_Y)
2275 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2207 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2276 val |= I965_FENCE_REG_VALID; 2208 val |= I965_FENCE_REG_VALID;
2277 2209
2278 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val); 2210 if (pipelined) {
2211 int ret = intel_ring_begin(pipelined, 6);
2212 if (ret)
2213 return ret;
2214
2215 intel_ring_emit(pipelined, MI_NOOP);
2216 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2217 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2218 intel_ring_emit(pipelined, (u32)val);
2219 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2220 intel_ring_emit(pipelined, (u32)(val >> 32));
2221 intel_ring_advance(pipelined);
2222 } else
2223 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2224
2225 return 0;
2279} 2226}
2280 2227
2281static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) 2228static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2229 struct intel_ring_buffer *pipelined)
2282{ 2230{
2283 struct drm_gem_object *obj = reg->obj; 2231 struct drm_device *dev = obj->base.dev;
2284 struct drm_device *dev = obj->dev;
2285 drm_i915_private_t *dev_priv = dev->dev_private; 2232 drm_i915_private_t *dev_priv = dev->dev_private;
2286 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2233 u32 size = obj->gtt_space->size;
2287 int regnum = obj_priv->fence_reg; 2234 int regnum = obj->fence_reg;
2288 uint64_t val; 2235 uint64_t val;
2289 2236
2290 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2237 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2291 0xfffff000) << 32; 2238 0xfffff000) << 32;
2292 val |= obj_priv->gtt_offset & 0xfffff000; 2239 val |= obj->gtt_offset & 0xfffff000;
2293 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2240 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2294 if (obj_priv->tiling_mode == I915_TILING_Y) 2241 if (obj->tiling_mode == I915_TILING_Y)
2295 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2242 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2296 val |= I965_FENCE_REG_VALID; 2243 val |= I965_FENCE_REG_VALID;
2297 2244
2298 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); 2245 if (pipelined) {
2246 int ret = intel_ring_begin(pipelined, 6);
2247 if (ret)
2248 return ret;
2249
2250 intel_ring_emit(pipelined, MI_NOOP);
2251 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2252 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2253 intel_ring_emit(pipelined, (u32)val);
2254 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2255 intel_ring_emit(pipelined, (u32)(val >> 32));
2256 intel_ring_advance(pipelined);
2257 } else
2258 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2259
2260 return 0;
2299} 2261}
2300 2262
2301static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) 2263static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2264 struct intel_ring_buffer *pipelined)
2302{ 2265{
2303 struct drm_gem_object *obj = reg->obj; 2266 struct drm_device *dev = obj->base.dev;
2304 struct drm_device *dev = obj->dev;
2305 drm_i915_private_t *dev_priv = dev->dev_private; 2267 drm_i915_private_t *dev_priv = dev->dev_private;
2306 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2268 u32 size = obj->gtt_space->size;
2307 int regnum = obj_priv->fence_reg; 2269 u32 fence_reg, val, pitch_val;
2308 int tile_width; 2270 int tile_width;
2309 uint32_t fence_reg, val;
2310 uint32_t pitch_val;
2311 2271
2312 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || 2272 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2313 (obj_priv->gtt_offset & (obj->size - 1))) { 2273 (size & -size) != size ||
2314 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", 2274 (obj->gtt_offset & (size - 1)),
2315 __func__, obj_priv->gtt_offset, obj->size); 2275 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2316 return; 2276 obj->gtt_offset, obj->map_and_fenceable, size))
2317 } 2277 return -EINVAL;
2318 2278
2319 if (obj_priv->tiling_mode == I915_TILING_Y && 2279 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2320 HAS_128_BYTE_Y_TILING(dev))
2321 tile_width = 128; 2280 tile_width = 128;
2322 else 2281 else
2323 tile_width = 512; 2282 tile_width = 512;
2324 2283
2325 /* Note: pitch better be a power of two tile widths */ 2284 /* Note: pitch better be a power of two tile widths */
2326 pitch_val = obj_priv->stride / tile_width; 2285 pitch_val = obj->stride / tile_width;
2327 pitch_val = ffs(pitch_val) - 1; 2286 pitch_val = ffs(pitch_val) - 1;
2328 2287
2329 if (obj_priv->tiling_mode == I915_TILING_Y && 2288 val = obj->gtt_offset;
2330 HAS_128_BYTE_Y_TILING(dev)) 2289 if (obj->tiling_mode == I915_TILING_Y)
2331 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2332 else
2333 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2334
2335 val = obj_priv->gtt_offset;
2336 if (obj_priv->tiling_mode == I915_TILING_Y)
2337 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2290 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2338 val |= I915_FENCE_SIZE_BITS(obj->size); 2291 val |= I915_FENCE_SIZE_BITS(size);
2339 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2292 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2340 val |= I830_FENCE_REG_VALID; 2293 val |= I830_FENCE_REG_VALID;
2341 2294
2342 if (regnum < 8) 2295 fence_reg = obj->fence_reg;
2343 fence_reg = FENCE_REG_830_0 + (regnum * 4); 2296 if (fence_reg < 8)
2297 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2344 else 2298 else
2345 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); 2299 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2346 I915_WRITE(fence_reg, val); 2300
2301 if (pipelined) {
2302 int ret = intel_ring_begin(pipelined, 4);
2303 if (ret)
2304 return ret;
2305
2306 intel_ring_emit(pipelined, MI_NOOP);
2307 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2308 intel_ring_emit(pipelined, fence_reg);
2309 intel_ring_emit(pipelined, val);
2310 intel_ring_advance(pipelined);
2311 } else
2312 I915_WRITE(fence_reg, val);
2313
2314 return 0;
2347} 2315}
2348 2316
2349static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) 2317static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2318 struct intel_ring_buffer *pipelined)
2350{ 2319{
2351 struct drm_gem_object *obj = reg->obj; 2320 struct drm_device *dev = obj->base.dev;
2352 struct drm_device *dev = obj->dev;
2353 drm_i915_private_t *dev_priv = dev->dev_private; 2321 drm_i915_private_t *dev_priv = dev->dev_private;
2354 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2322 u32 size = obj->gtt_space->size;
2355 int regnum = obj_priv->fence_reg; 2323 int regnum = obj->fence_reg;
2356 uint32_t val; 2324 uint32_t val;
2357 uint32_t pitch_val; 2325 uint32_t pitch_val;
2358 uint32_t fence_size_bits;
2359 2326
2360 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || 2327 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2361 (obj_priv->gtt_offset & (obj->size - 1))) { 2328 (size & -size) != size ||
2362 WARN(1, "%s: object 0x%08x not 512K or size aligned\n", 2329 (obj->gtt_offset & (size - 1)),
2363 __func__, obj_priv->gtt_offset); 2330 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2364 return; 2331 obj->gtt_offset, size))
2365 } 2332 return -EINVAL;
2366 2333
2367 pitch_val = obj_priv->stride / 128; 2334 pitch_val = obj->stride / 128;
2368 pitch_val = ffs(pitch_val) - 1; 2335 pitch_val = ffs(pitch_val) - 1;
2369 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2370 2336
2371 val = obj_priv->gtt_offset; 2337 val = obj->gtt_offset;
2372 if (obj_priv->tiling_mode == I915_TILING_Y) 2338 if (obj->tiling_mode == I915_TILING_Y)
2373 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2339 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2374 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); 2340 val |= I830_FENCE_SIZE_BITS(size);
2375 WARN_ON(fence_size_bits & ~0x00000f00);
2376 val |= fence_size_bits;
2377 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2341 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2378 val |= I830_FENCE_REG_VALID; 2342 val |= I830_FENCE_REG_VALID;
2379 2343
2380 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); 2344 if (pipelined) {
2345 int ret = intel_ring_begin(pipelined, 4);
2346 if (ret)
2347 return ret;
2348
2349 intel_ring_emit(pipelined, MI_NOOP);
2350 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2351 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2352 intel_ring_emit(pipelined, val);
2353 intel_ring_advance(pipelined);
2354 } else
2355 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2356
2357 return 0;
2381} 2358}
2382 2359
2383static int i915_find_fence_reg(struct drm_device *dev, 2360static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2384 bool interruptible) 2361{
2362 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2363}
2364
2365static int
2366i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2367 struct intel_ring_buffer *pipelined,
2368 bool interruptible)
2369{
2370 int ret;
2371
2372 if (obj->fenced_gpu_access) {
2373 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2374 i915_gem_flush_ring(obj->base.dev,
2375 obj->last_fenced_ring,
2376 0, obj->base.write_domain);
2377
2378 obj->fenced_gpu_access = false;
2379 }
2380
2381 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2382 if (!ring_passed_seqno(obj->last_fenced_ring,
2383 obj->last_fenced_seqno)) {
2384 ret = i915_do_wait_request(obj->base.dev,
2385 obj->last_fenced_seqno,
2386 interruptible,
2387 obj->last_fenced_ring);
2388 if (ret)
2389 return ret;
2390 }
2391
2392 obj->last_fenced_seqno = 0;
2393 obj->last_fenced_ring = NULL;
2394 }
2395
2396 return 0;
2397}
2398
2399int
2400i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2401{
2402 int ret;
2403
2404 if (obj->tiling_mode)
2405 i915_gem_release_mmap(obj);
2406
2407 ret = i915_gem_object_flush_fence(obj, NULL, true);
2408 if (ret)
2409 return ret;
2410
2411 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2412 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2413 i915_gem_clear_fence_reg(obj->base.dev,
2414 &dev_priv->fence_regs[obj->fence_reg]);
2415
2416 obj->fence_reg = I915_FENCE_REG_NONE;
2417 }
2418
2419 return 0;
2420}
2421
2422static struct drm_i915_fence_reg *
2423i915_find_fence_reg(struct drm_device *dev,
2424 struct intel_ring_buffer *pipelined)
2385{ 2425{
2386 struct drm_i915_fence_reg *reg = NULL;
2387 struct drm_i915_gem_object *obj_priv = NULL;
2388 struct drm_i915_private *dev_priv = dev->dev_private; 2426 struct drm_i915_private *dev_priv = dev->dev_private;
2389 struct drm_gem_object *obj = NULL; 2427 struct drm_i915_fence_reg *reg, *first, *avail;
2390 int i, avail, ret; 2428 int i;
2391 2429
2392 /* First try to find a free reg */ 2430 /* First try to find a free reg */
2393 avail = 0; 2431 avail = NULL;
2394 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2432 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2395 reg = &dev_priv->fence_regs[i]; 2433 reg = &dev_priv->fence_regs[i];
2396 if (!reg->obj) 2434 if (!reg->obj)
2397 return i; 2435 return reg;
2398 2436
2399 obj_priv = to_intel_bo(reg->obj); 2437 if (!reg->obj->pin_count)
2400 if (!obj_priv->pin_count) 2438 avail = reg;
2401 avail++;
2402 } 2439 }
2403 2440
2404 if (avail == 0) 2441 if (avail == NULL)
2405 return -ENOSPC; 2442 return NULL;
2406 2443
2407 /* None available, try to steal one or wait for a user to finish */ 2444 /* None available, try to steal one or wait for a user to finish */
2408 i = I915_FENCE_REG_NONE; 2445 avail = first = NULL;
2409 list_for_each_entry(reg, &dev_priv->mm.fence_list, 2446 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2410 lru_list) { 2447 if (reg->obj->pin_count)
2411 obj = reg->obj;
2412 obj_priv = to_intel_bo(obj);
2413
2414 if (obj_priv->pin_count)
2415 continue; 2448 continue;
2416 2449
2417 /* found one! */ 2450 if (first == NULL)
2418 i = obj_priv->fence_reg; 2451 first = reg;
2419 break;
2420 }
2421 2452
2422 BUG_ON(i == I915_FENCE_REG_NONE); 2453 if (!pipelined ||
2454 !reg->obj->last_fenced_ring ||
2455 reg->obj->last_fenced_ring == pipelined) {
2456 avail = reg;
2457 break;
2458 }
2459 }
2423 2460
2424 /* We only have a reference on obj from the active list. put_fence_reg 2461 if (avail == NULL)
2425 * might drop that one, causing a use-after-free in it. So hold a 2462 avail = first;
2426 * private reference to obj like the other callers of put_fence_reg
2427 * (set_tiling ioctl) do. */
2428 drm_gem_object_reference(obj);
2429 ret = i915_gem_object_put_fence_reg(obj, interruptible);
2430 drm_gem_object_unreference(obj);
2431 if (ret != 0)
2432 return ret;
2433 2463
2434 return i; 2464 return avail;
2435} 2465}
2436 2466
2437/** 2467/**
2438 * i915_gem_object_get_fence_reg - set up a fence reg for an object 2468 * i915_gem_object_get_fence - set up a fence reg for an object
2439 * @obj: object to map through a fence reg 2469 * @obj: object to map through a fence reg
2470 * @pipelined: ring on which to queue the change, or NULL for CPU access
2471 * @interruptible: must we wait uninterruptibly for the register to retire?
2440 * 2472 *
2441 * When mapping objects through the GTT, userspace wants to be able to write 2473 * When mapping objects through the GTT, userspace wants to be able to write
2442 * to them without having to worry about swizzling if the object is tiled. 2474 * to them without having to worry about swizzling if the object is tiled.
@@ -2448,72 +2480,138 @@ static int i915_find_fence_reg(struct drm_device *dev,
2448 * and tiling format. 2480 * and tiling format.
2449 */ 2481 */
2450int 2482int
2451i915_gem_object_get_fence_reg(struct drm_gem_object *obj, 2483i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2452 bool interruptible) 2484 struct intel_ring_buffer *pipelined,
2485 bool interruptible)
2453{ 2486{
2454 struct drm_device *dev = obj->dev; 2487 struct drm_device *dev = obj->base.dev;
2455 struct drm_i915_private *dev_priv = dev->dev_private; 2488 struct drm_i915_private *dev_priv = dev->dev_private;
2456 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2489 struct drm_i915_fence_reg *reg;
2457 struct drm_i915_fence_reg *reg = NULL;
2458 int ret; 2490 int ret;
2459 2491
2460 /* Just update our place in the LRU if our fence is getting used. */ 2492 /* XXX disable pipelining. There are bugs. Shocking. */
2461 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { 2493 pipelined = NULL;
2462 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2494
2495 /* Just update our place in the LRU if our fence is getting reused. */
2496 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2497 reg = &dev_priv->fence_regs[obj->fence_reg];
2463 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2498 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2499
2500 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2501 pipelined = NULL;
2502
2503 if (!pipelined) {
2504 if (reg->setup_seqno) {
2505 if (!ring_passed_seqno(obj->last_fenced_ring,
2506 reg->setup_seqno)) {
2507 ret = i915_do_wait_request(obj->base.dev,
2508 reg->setup_seqno,
2509 interruptible,
2510 obj->last_fenced_ring);
2511 if (ret)
2512 return ret;
2513 }
2514
2515 reg->setup_seqno = 0;
2516 }
2517 } else if (obj->last_fenced_ring &&
2518 obj->last_fenced_ring != pipelined) {
2519 ret = i915_gem_object_flush_fence(obj,
2520 pipelined,
2521 interruptible);
2522 if (ret)
2523 return ret;
2524 } else if (obj->tiling_changed) {
2525 if (obj->fenced_gpu_access) {
2526 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2527 i915_gem_flush_ring(obj->base.dev, obj->ring,
2528 0, obj->base.write_domain);
2529
2530 obj->fenced_gpu_access = false;
2531 }
2532 }
2533
2534 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2535 pipelined = NULL;
2536 BUG_ON(!pipelined && reg->setup_seqno);
2537
2538 if (obj->tiling_changed) {
2539 if (pipelined) {
2540 reg->setup_seqno =
2541 i915_gem_next_request_seqno(dev, pipelined);
2542 obj->last_fenced_seqno = reg->setup_seqno;
2543 obj->last_fenced_ring = pipelined;
2544 }
2545 goto update;
2546 }
2547
2464 return 0; 2548 return 0;
2465 } 2549 }
2466 2550
2467 switch (obj_priv->tiling_mode) { 2551 reg = i915_find_fence_reg(dev, pipelined);
2468 case I915_TILING_NONE: 2552 if (reg == NULL)
2469 WARN(1, "allocating a fence for non-tiled object?\n"); 2553 return -ENOSPC;
2470 break;
2471 case I915_TILING_X:
2472 if (!obj_priv->stride)
2473 return -EINVAL;
2474 WARN((obj_priv->stride & (512 - 1)),
2475 "object 0x%08x is X tiled but has non-512B pitch\n",
2476 obj_priv->gtt_offset);
2477 break;
2478 case I915_TILING_Y:
2479 if (!obj_priv->stride)
2480 return -EINVAL;
2481 WARN((obj_priv->stride & (128 - 1)),
2482 "object 0x%08x is Y tiled but has non-128B pitch\n",
2483 obj_priv->gtt_offset);
2484 break;
2485 }
2486 2554
2487 ret = i915_find_fence_reg(dev, interruptible); 2555 ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
2488 if (ret < 0) 2556 if (ret)
2489 return ret; 2557 return ret;
2490 2558
2491 obj_priv->fence_reg = ret; 2559 if (reg->obj) {
2492 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2560 struct drm_i915_gem_object *old = reg->obj;
2493 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2561
2562 drm_gem_object_reference(&old->base);
2563
2564 if (old->tiling_mode)
2565 i915_gem_release_mmap(old);
2566
2567 ret = i915_gem_object_flush_fence(old,
2568 pipelined,
2569 interruptible);
2570 if (ret) {
2571 drm_gem_object_unreference(&old->base);
2572 return ret;
2573 }
2574
2575 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2576 pipelined = NULL;
2577
2578 old->fence_reg = I915_FENCE_REG_NONE;
2579 old->last_fenced_ring = pipelined;
2580 old->last_fenced_seqno =
2581 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2582
2583 drm_gem_object_unreference(&old->base);
2584 } else if (obj->last_fenced_seqno == 0)
2585 pipelined = NULL;
2494 2586
2495 reg->obj = obj; 2587 reg->obj = obj;
2588 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2589 obj->fence_reg = reg - dev_priv->fence_regs;
2590 obj->last_fenced_ring = pipelined;
2496 2591
2592 reg->setup_seqno =
2593 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2594 obj->last_fenced_seqno = reg->setup_seqno;
2595
2596update:
2597 obj->tiling_changed = false;
2497 switch (INTEL_INFO(dev)->gen) { 2598 switch (INTEL_INFO(dev)->gen) {
2498 case 6: 2599 case 6:
2499 sandybridge_write_fence_reg(reg); 2600 ret = sandybridge_write_fence_reg(obj, pipelined);
2500 break; 2601 break;
2501 case 5: 2602 case 5:
2502 case 4: 2603 case 4:
2503 i965_write_fence_reg(reg); 2604 ret = i965_write_fence_reg(obj, pipelined);
2504 break; 2605 break;
2505 case 3: 2606 case 3:
2506 i915_write_fence_reg(reg); 2607 ret = i915_write_fence_reg(obj, pipelined);
2507 break; 2608 break;
2508 case 2: 2609 case 2:
2509 i830_write_fence_reg(reg); 2610 ret = i830_write_fence_reg(obj, pipelined);
2510 break; 2611 break;
2511 } 2612 }
2512 2613
2513 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg, 2614 return ret;
2514 obj_priv->tiling_mode);
2515
2516 return 0;
2517} 2615}
2518 2616
2519/** 2617/**
@@ -2521,154 +2619,127 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2521 * @obj: object to clear 2619 * @obj: object to clear
2522 * 2620 *
2523 * Zeroes out the fence register itself and clears out the associated 2621 * Zeroes out the fence register itself and clears out the associated
2524 * data structures in dev_priv and obj_priv. 2622 * data structures in dev_priv and obj.
2525 */ 2623 */
2526static void 2624static void
2527i915_gem_clear_fence_reg(struct drm_gem_object *obj) 2625i915_gem_clear_fence_reg(struct drm_device *dev,
2626 struct drm_i915_fence_reg *reg)
2528{ 2627{
2529 struct drm_device *dev = obj->dev;
2530 drm_i915_private_t *dev_priv = dev->dev_private; 2628 drm_i915_private_t *dev_priv = dev->dev_private;
2531 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2629 uint32_t fence_reg = reg - dev_priv->fence_regs;
2532 struct drm_i915_fence_reg *reg =
2533 &dev_priv->fence_regs[obj_priv->fence_reg];
2534 uint32_t fence_reg;
2535 2630
2536 switch (INTEL_INFO(dev)->gen) { 2631 switch (INTEL_INFO(dev)->gen) {
2537 case 6: 2632 case 6:
2538 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + 2633 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2539 (obj_priv->fence_reg * 8), 0);
2540 break; 2634 break;
2541 case 5: 2635 case 5:
2542 case 4: 2636 case 4:
2543 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); 2637 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2544 break; 2638 break;
2545 case 3: 2639 case 3:
2546 if (obj_priv->fence_reg >= 8) 2640 if (fence_reg >= 8)
2547 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; 2641 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2548 else 2642 else
2549 case 2: 2643 case 2:
2550 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; 2644 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2551 2645
2552 I915_WRITE(fence_reg, 0); 2646 I915_WRITE(fence_reg, 0);
2553 break; 2647 break;
2554 } 2648 }
2555 2649
2556 reg->obj = NULL;
2557 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2558 list_del_init(&reg->lru_list); 2650 list_del_init(&reg->lru_list);
2559} 2651 reg->obj = NULL;
2560 2652 reg->setup_seqno = 0;
2561/**
2562 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2563 * to the buffer to finish, and then resets the fence register.
2564 * @obj: tiled object holding a fence register.
2565 * @bool: whether the wait upon the fence is interruptible
2566 *
2567 * Zeroes out the fence register itself and clears out the associated
2568 * data structures in dev_priv and obj_priv.
2569 */
2570int
2571i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2572 bool interruptible)
2573{
2574 struct drm_device *dev = obj->dev;
2575 struct drm_i915_private *dev_priv = dev->dev_private;
2576 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2577 struct drm_i915_fence_reg *reg;
2578
2579 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2580 return 0;
2581
2582 /* If we've changed tiling, GTT-mappings of the object
2583 * need to re-fault to ensure that the correct fence register
2584 * setup is in place.
2585 */
2586 i915_gem_release_mmap(obj);
2587
2588 /* On the i915, GPU access to tiled buffers is via a fence,
2589 * therefore we must wait for any outstanding access to complete
2590 * before clearing the fence.
2591 */
2592 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2593 if (reg->gpu) {
2594 int ret;
2595
2596 ret = i915_gem_object_flush_gpu_write_domain(obj);
2597 if (ret)
2598 return ret;
2599
2600 ret = i915_gem_object_wait_rendering(obj, interruptible);
2601 if (ret)
2602 return ret;
2603
2604 reg->gpu = false;
2605 }
2606
2607 i915_gem_object_flush_gtt_write_domain(obj);
2608 i915_gem_clear_fence_reg(obj);
2609
2610 return 0;
2611} 2653}
2612 2654
2613/** 2655/**
2614 * Finds free space in the GTT aperture and binds the object there. 2656 * Finds free space in the GTT aperture and binds the object there.
2615 */ 2657 */
2616static int 2658static int
2617i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) 2659i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2660 unsigned alignment,
2661 bool map_and_fenceable)
2618{ 2662{
2619 struct drm_device *dev = obj->dev; 2663 struct drm_device *dev = obj->base.dev;
2620 drm_i915_private_t *dev_priv = dev->dev_private; 2664 drm_i915_private_t *dev_priv = dev->dev_private;
2621 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2622 struct drm_mm_node *free_space; 2665 struct drm_mm_node *free_space;
2623 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; 2666 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2667 u32 size, fence_size, fence_alignment, unfenced_alignment;
2668 bool mappable, fenceable;
2624 int ret; 2669 int ret;
2625 2670
2626 if (obj_priv->madv != I915_MADV_WILLNEED) { 2671 if (obj->madv != I915_MADV_WILLNEED) {
2627 DRM_ERROR("Attempting to bind a purgeable object\n"); 2672 DRM_ERROR("Attempting to bind a purgeable object\n");
2628 return -EINVAL; 2673 return -EINVAL;
2629 } 2674 }
2630 2675
2676 fence_size = i915_gem_get_gtt_size(obj);
2677 fence_alignment = i915_gem_get_gtt_alignment(obj);
2678 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
2679
2631 if (alignment == 0) 2680 if (alignment == 0)
2632 alignment = i915_gem_get_gtt_alignment(obj); 2681 alignment = map_and_fenceable ? fence_alignment :
2633 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { 2682 unfenced_alignment;
2683 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2634 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2684 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2635 return -EINVAL; 2685 return -EINVAL;
2636 } 2686 }
2637 2687
2688 size = map_and_fenceable ? fence_size : obj->base.size;
2689
2638 /* If the object is bigger than the entire aperture, reject it early 2690 /* If the object is bigger than the entire aperture, reject it early
2639 * before evicting everything in a vain attempt to find space. 2691 * before evicting everything in a vain attempt to find space.
2640 */ 2692 */
2641 if (obj->size > dev_priv->mm.gtt_total) { 2693 if (obj->base.size >
2694 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2642 DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 2695 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2643 return -E2BIG; 2696 return -E2BIG;
2644 } 2697 }
2645 2698
2646 search_free: 2699 search_free:
2647 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2700 if (map_and_fenceable)
2648 obj->size, alignment, 0); 2701 free_space =
2649 if (free_space != NULL) 2702 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2650 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2703 size, alignment, 0,
2651 alignment); 2704 dev_priv->mm.gtt_mappable_end,
2652 if (obj_priv->gtt_space == NULL) { 2705 0);
2706 else
2707 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2708 size, alignment, 0);
2709
2710 if (free_space != NULL) {
2711 if (map_and_fenceable)
2712 obj->gtt_space =
2713 drm_mm_get_block_range_generic(free_space,
2714 size, alignment, 0,
2715 dev_priv->mm.gtt_mappable_end,
2716 0);
2717 else
2718 obj->gtt_space =
2719 drm_mm_get_block(free_space, size, alignment);
2720 }
2721 if (obj->gtt_space == NULL) {
2653 /* If the gtt is empty and we're still having trouble 2722 /* If the gtt is empty and we're still having trouble
2654 * fitting our object in, we're out of memory. 2723 * fitting our object in, we're out of memory.
2655 */ 2724 */
2656 ret = i915_gem_evict_something(dev, obj->size, alignment); 2725 ret = i915_gem_evict_something(dev, size, alignment,
2726 map_and_fenceable);
2657 if (ret) 2727 if (ret)
2658 return ret; 2728 return ret;
2659 2729
2660 goto search_free; 2730 goto search_free;
2661 } 2731 }
2662 2732
2663 ret = i915_gem_object_get_pages(obj, gfpmask); 2733 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2664 if (ret) { 2734 if (ret) {
2665 drm_mm_put_block(obj_priv->gtt_space); 2735 drm_mm_put_block(obj->gtt_space);
2666 obj_priv->gtt_space = NULL; 2736 obj->gtt_space = NULL;
2667 2737
2668 if (ret == -ENOMEM) { 2738 if (ret == -ENOMEM) {
2669 /* first try to clear up some space from the GTT */ 2739 /* first try to clear up some space from the GTT */
2670 ret = i915_gem_evict_something(dev, obj->size, 2740 ret = i915_gem_evict_something(dev, size,
2671 alignment); 2741 alignment,
2742 map_and_fenceable);
2672 if (ret) { 2743 if (ret) {
2673 /* now try to shrink everyone else */ 2744 /* now try to shrink everyone else */
2674 if (gfpmask) { 2745 if (gfpmask) {
@@ -2685,122 +2756,113 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2685 return ret; 2756 return ret;
2686 } 2757 }
2687 2758
2688 /* Create an AGP memory structure pointing at our pages, and bind it 2759 ret = i915_gem_gtt_bind_object(obj);
2689 * into the GTT. 2760 if (ret) {
2690 */ 2761 i915_gem_object_put_pages_gtt(obj);
2691 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2762 drm_mm_put_block(obj->gtt_space);
2692 obj_priv->pages, 2763 obj->gtt_space = NULL;
2693 obj->size >> PAGE_SHIFT, 2764
2694 obj_priv->gtt_space->start, 2765 ret = i915_gem_evict_something(dev, size,
2695 obj_priv->agp_type); 2766 alignment, map_and_fenceable);
2696 if (obj_priv->agp_mem == NULL) {
2697 i915_gem_object_put_pages(obj);
2698 drm_mm_put_block(obj_priv->gtt_space);
2699 obj_priv->gtt_space = NULL;
2700
2701 ret = i915_gem_evict_something(dev, obj->size, alignment);
2702 if (ret) 2767 if (ret)
2703 return ret; 2768 return ret;
2704 2769
2705 goto search_free; 2770 goto search_free;
2706 } 2771 }
2707 2772
2708 /* keep track of bounds object by adding it to the inactive list */ 2773 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2709 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 2774 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2710 i915_gem_info_add_gtt(dev_priv, obj->size);
2711 2775
2712 /* Assert that the object is not currently in any GPU domain. As it 2776 /* Assert that the object is not currently in any GPU domain. As it
2713 * wasn't in the GTT, there shouldn't be any way it could have been in 2777 * wasn't in the GTT, there shouldn't be any way it could have been in
2714 * a GPU cache 2778 * a GPU cache
2715 */ 2779 */
2716 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2780 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2717 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2781 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2782
2783 obj->gtt_offset = obj->gtt_space->start;
2718 2784
2719 obj_priv->gtt_offset = obj_priv->gtt_space->start; 2785 fenceable =
2720 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset); 2786 obj->gtt_space->size == fence_size &&
2787 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2721 2788
2789 mappable =
2790 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2791
2792 obj->map_and_fenceable = mappable && fenceable;
2793
2794 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
2722 return 0; 2795 return 0;
2723} 2796}
2724 2797
2725void 2798void
2726i915_gem_clflush_object(struct drm_gem_object *obj) 2799i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2727{ 2800{
2728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2729
2730 /* If we don't have a page list set up, then we're not pinned 2801 /* If we don't have a page list set up, then we're not pinned
2731 * to GPU, and we can ignore the cache flush because it'll happen 2802 * to GPU, and we can ignore the cache flush because it'll happen
2732 * again at bind time. 2803 * again at bind time.
2733 */ 2804 */
2734 if (obj_priv->pages == NULL) 2805 if (obj->pages == NULL)
2735 return; 2806 return;
2736 2807
2737 trace_i915_gem_object_clflush(obj); 2808 trace_i915_gem_object_clflush(obj);
2738 2809
2739 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); 2810 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2740} 2811}
2741 2812
2742/** Flushes any GPU write domain for the object if it's dirty. */ 2813/** Flushes any GPU write domain for the object if it's dirty. */
2743static int 2814static void
2744i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) 2815i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2745{ 2816{
2746 struct drm_device *dev = obj->dev; 2817 struct drm_device *dev = obj->base.dev;
2747 uint32_t old_write_domain;
2748 2818
2749 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) 2819 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2750 return 0; 2820 return;
2751 2821
2752 /* Queue the GPU write cache flushing we need. */ 2822 /* Queue the GPU write cache flushing we need. */
2753 old_write_domain = obj->write_domain; 2823 i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
2754 i915_gem_flush_ring(dev, NULL, 2824 BUG_ON(obj->base.write_domain);
2755 to_intel_bo(obj)->ring,
2756 0, obj->write_domain);
2757 BUG_ON(obj->write_domain);
2758
2759 trace_i915_gem_object_change_domain(obj,
2760 obj->read_domains,
2761 old_write_domain);
2762
2763 return 0;
2764} 2825}
2765 2826
2766/** Flushes the GTT write domain for the object if it's dirty. */ 2827/** Flushes the GTT write domain for the object if it's dirty. */
2767static void 2828static void
2768i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 2829i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2769{ 2830{
2770 uint32_t old_write_domain; 2831 uint32_t old_write_domain;
2771 2832
2772 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 2833 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2773 return; 2834 return;
2774 2835
2775 /* No actual flushing is required for the GTT write domain. Writes 2836 /* No actual flushing is required for the GTT write domain. Writes
2776 * to it immediately go to main memory as far as we know, so there's 2837 * to it immediately go to main memory as far as we know, so there's
2777 * no chipset flush. It also doesn't land in render cache. 2838 * no chipset flush. It also doesn't land in render cache.
2778 */ 2839 */
2779 old_write_domain = obj->write_domain; 2840 i915_gem_release_mmap(obj);
2780 obj->write_domain = 0; 2841
2842 old_write_domain = obj->base.write_domain;
2843 obj->base.write_domain = 0;
2781 2844
2782 trace_i915_gem_object_change_domain(obj, 2845 trace_i915_gem_object_change_domain(obj,
2783 obj->read_domains, 2846 obj->base.read_domains,
2784 old_write_domain); 2847 old_write_domain);
2785} 2848}
2786 2849
2787/** Flushes the CPU write domain for the object if it's dirty. */ 2850/** Flushes the CPU write domain for the object if it's dirty. */
2788static void 2851static void
2789i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 2852i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2790{ 2853{
2791 struct drm_device *dev = obj->dev;
2792 uint32_t old_write_domain; 2854 uint32_t old_write_domain;
2793 2855
2794 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 2856 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2795 return; 2857 return;
2796 2858
2797 i915_gem_clflush_object(obj); 2859 i915_gem_clflush_object(obj);
2798 drm_agp_chipset_flush(dev); 2860 intel_gtt_chipset_flush();
2799 old_write_domain = obj->write_domain; 2861 old_write_domain = obj->base.write_domain;
2800 obj->write_domain = 0; 2862 obj->base.write_domain = 0;
2801 2863
2802 trace_i915_gem_object_change_domain(obj, 2864 trace_i915_gem_object_change_domain(obj,
2803 obj->read_domains, 2865 obj->base.read_domains,
2804 old_write_domain); 2866 old_write_domain);
2805} 2867}
2806 2868
@@ -2811,37 +2873,36 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2811 * flushes to occur. 2873 * flushes to occur.
2812 */ 2874 */
2813int 2875int
2814i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 2876i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2815{ 2877{
2816 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2817 uint32_t old_write_domain, old_read_domains; 2878 uint32_t old_write_domain, old_read_domains;
2818 int ret; 2879 int ret;
2819 2880
2820 /* Not valid to be called on unbound objects. */ 2881 /* Not valid to be called on unbound objects. */
2821 if (obj_priv->gtt_space == NULL) 2882 if (obj->gtt_space == NULL)
2822 return -EINVAL; 2883 return -EINVAL;
2823 2884
2824 ret = i915_gem_object_flush_gpu_write_domain(obj); 2885 i915_gem_object_flush_gpu_write_domain(obj);
2825 if (ret != 0) 2886 if (obj->pending_gpu_write || write) {
2826 return ret; 2887 ret = i915_gem_object_wait_rendering(obj, true);
2827 ret = i915_gem_object_wait_rendering(obj, true); 2888 if (ret)
2828 if (ret) 2889 return ret;
2829 return ret; 2890 }
2830 2891
2831 i915_gem_object_flush_cpu_write_domain(obj); 2892 i915_gem_object_flush_cpu_write_domain(obj);
2832 2893
2833 old_write_domain = obj->write_domain; 2894 old_write_domain = obj->base.write_domain;
2834 old_read_domains = obj->read_domains; 2895 old_read_domains = obj->base.read_domains;
2835 2896
2836 /* It should now be out of any other write domains, and we can update 2897 /* It should now be out of any other write domains, and we can update
2837 * the domain values for our changes. 2898 * the domain values for our changes.
2838 */ 2899 */
2839 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 2900 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2840 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2901 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2841 if (write) { 2902 if (write) {
2842 obj->read_domains = I915_GEM_DOMAIN_GTT; 2903 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2843 obj->write_domain = I915_GEM_DOMAIN_GTT; 2904 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2844 obj_priv->dirty = 1; 2905 obj->dirty = 1;
2845 } 2906 }
2846 2907
2847 trace_i915_gem_object_change_domain(obj, 2908 trace_i915_gem_object_change_domain(obj,
@@ -2856,23 +2917,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2856 * wait, as in modesetting process we're not supposed to be interrupted. 2917 * wait, as in modesetting process we're not supposed to be interrupted.
2857 */ 2918 */
2858int 2919int
2859i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, 2920i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
2860 bool pipelined) 2921 struct intel_ring_buffer *pipelined)
2861{ 2922{
2862 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2863 uint32_t old_read_domains; 2923 uint32_t old_read_domains;
2864 int ret; 2924 int ret;
2865 2925
2866 /* Not valid to be called on unbound objects. */ 2926 /* Not valid to be called on unbound objects. */
2867 if (obj_priv->gtt_space == NULL) 2927 if (obj->gtt_space == NULL)
2868 return -EINVAL; 2928 return -EINVAL;
2869 2929
2870 ret = i915_gem_object_flush_gpu_write_domain(obj); 2930 i915_gem_object_flush_gpu_write_domain(obj);
2871 if (ret)
2872 return ret;
2873 2931
2874 /* Currently, we are always called from an non-interruptible context. */ 2932 /* Currently, we are always called from an non-interruptible context. */
2875 if (!pipelined) { 2933 if (pipelined != obj->ring) {
2876 ret = i915_gem_object_wait_rendering(obj, false); 2934 ret = i915_gem_object_wait_rendering(obj, false);
2877 if (ret) 2935 if (ret)
2878 return ret; 2936 return ret;
@@ -2880,12 +2938,12 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2880 2938
2881 i915_gem_object_flush_cpu_write_domain(obj); 2939 i915_gem_object_flush_cpu_write_domain(obj);
2882 2940
2883 old_read_domains = obj->read_domains; 2941 old_read_domains = obj->base.read_domains;
2884 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2942 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2885 2943
2886 trace_i915_gem_object_change_domain(obj, 2944 trace_i915_gem_object_change_domain(obj,
2887 old_read_domains, 2945 old_read_domains,
2888 obj->write_domain); 2946 obj->base.write_domain);
2889 2947
2890 return 0; 2948 return 0;
2891} 2949}
@@ -2898,10 +2956,10 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2898 return 0; 2956 return 0;
2899 2957
2900 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) 2958 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2901 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring, 2959 i915_gem_flush_ring(obj->base.dev, obj->ring,
2902 0, obj->base.write_domain); 2960 0, obj->base.write_domain);
2903 2961
2904 return i915_gem_object_wait_rendering(&obj->base, interruptible); 2962 return i915_gem_object_wait_rendering(obj, interruptible);
2905} 2963}
2906 2964
2907/** 2965/**
@@ -2911,14 +2969,12 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2911 * flushes to occur. 2969 * flushes to occur.
2912 */ 2970 */
2913static int 2971static int
2914i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 2972i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2915{ 2973{
2916 uint32_t old_write_domain, old_read_domains; 2974 uint32_t old_write_domain, old_read_domains;
2917 int ret; 2975 int ret;
2918 2976
2919 ret = i915_gem_object_flush_gpu_write_domain(obj); 2977 i915_gem_object_flush_gpu_write_domain(obj);
2920 if (ret != 0)
2921 return ret;
2922 ret = i915_gem_object_wait_rendering(obj, true); 2978 ret = i915_gem_object_wait_rendering(obj, true);
2923 if (ret) 2979 if (ret)
2924 return ret; 2980 return ret;
@@ -2930,27 +2986,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2930 */ 2986 */
2931 i915_gem_object_set_to_full_cpu_read_domain(obj); 2987 i915_gem_object_set_to_full_cpu_read_domain(obj);
2932 2988
2933 old_write_domain = obj->write_domain; 2989 old_write_domain = obj->base.write_domain;
2934 old_read_domains = obj->read_domains; 2990 old_read_domains = obj->base.read_domains;
2935 2991
2936 /* Flush the CPU cache if it's still invalid. */ 2992 /* Flush the CPU cache if it's still invalid. */
2937 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2993 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2938 i915_gem_clflush_object(obj); 2994 i915_gem_clflush_object(obj);
2939 2995
2940 obj->read_domains |= I915_GEM_DOMAIN_CPU; 2996 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2941 } 2997 }
2942 2998
2943 /* It should now be out of any other write domains, and we can update 2999 /* It should now be out of any other write domains, and we can update
2944 * the domain values for our changes. 3000 * the domain values for our changes.
2945 */ 3001 */
2946 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3002 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2947 3003
2948 /* If we're writing through the CPU, then the GPU read domains will 3004 /* If we're writing through the CPU, then the GPU read domains will
2949 * need to be invalidated at next use. 3005 * need to be invalidated at next use.
2950 */ 3006 */
2951 if (write) { 3007 if (write) {
2952 obj->read_domains = I915_GEM_DOMAIN_CPU; 3008 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2953 obj->write_domain = I915_GEM_DOMAIN_CPU; 3009 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2954 } 3010 }
2955 3011
2956 trace_i915_gem_object_change_domain(obj, 3012 trace_i915_gem_object_change_domain(obj,
@@ -2960,184 +3016,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2960 return 0; 3016 return 0;
2961} 3017}
2962 3018
2963/*
2964 * Set the next domain for the specified object. This
2965 * may not actually perform the necessary flushing/invaliding though,
2966 * as that may want to be batched with other set_domain operations
2967 *
2968 * This is (we hope) the only really tricky part of gem. The goal
2969 * is fairly simple -- track which caches hold bits of the object
2970 * and make sure they remain coherent. A few concrete examples may
2971 * help to explain how it works. For shorthand, we use the notation
2972 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2973 * a pair of read and write domain masks.
2974 *
2975 * Case 1: the batch buffer
2976 *
2977 * 1. Allocated
2978 * 2. Written by CPU
2979 * 3. Mapped to GTT
2980 * 4. Read by GPU
2981 * 5. Unmapped from GTT
2982 * 6. Freed
2983 *
2984 * Let's take these a step at a time
2985 *
2986 * 1. Allocated
2987 * Pages allocated from the kernel may still have
2988 * cache contents, so we set them to (CPU, CPU) always.
2989 * 2. Written by CPU (using pwrite)
2990 * The pwrite function calls set_domain (CPU, CPU) and
2991 * this function does nothing (as nothing changes)
2992 * 3. Mapped by GTT
2993 * This function asserts that the object is not
2994 * currently in any GPU-based read or write domains
2995 * 4. Read by GPU
2996 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2997 * As write_domain is zero, this function adds in the
2998 * current read domains (CPU+COMMAND, 0).
2999 * flush_domains is set to CPU.
3000 * invalidate_domains is set to COMMAND
3001 * clflush is run to get data out of the CPU caches
3002 * then i915_dev_set_domain calls i915_gem_flush to
3003 * emit an MI_FLUSH and drm_agp_chipset_flush
3004 * 5. Unmapped from GTT
3005 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3006 * flush_domains and invalidate_domains end up both zero
3007 * so no flushing/invalidating happens
3008 * 6. Freed
3009 * yay, done
3010 *
3011 * Case 2: The shared render buffer
3012 *
3013 * 1. Allocated
3014 * 2. Mapped to GTT
3015 * 3. Read/written by GPU
3016 * 4. set_domain to (CPU,CPU)
3017 * 5. Read/written by CPU
3018 * 6. Read/written by GPU
3019 *
3020 * 1. Allocated
3021 * Same as last example, (CPU, CPU)
3022 * 2. Mapped to GTT
3023 * Nothing changes (assertions find that it is not in the GPU)
3024 * 3. Read/written by GPU
3025 * execbuffer calls set_domain (RENDER, RENDER)
3026 * flush_domains gets CPU
3027 * invalidate_domains gets GPU
3028 * clflush (obj)
3029 * MI_FLUSH and drm_agp_chipset_flush
3030 * 4. set_domain (CPU, CPU)
3031 * flush_domains gets GPU
3032 * invalidate_domains gets CPU
3033 * wait_rendering (obj) to make sure all drawing is complete.
3034 * This will include an MI_FLUSH to get the data from GPU
3035 * to memory
3036 * clflush (obj) to invalidate the CPU cache
3037 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3038 * 5. Read/written by CPU
3039 * cache lines are loaded and dirtied
3040 * 6. Read written by GPU
3041 * Same as last GPU access
3042 *
3043 * Case 3: The constant buffer
3044 *
3045 * 1. Allocated
3046 * 2. Written by CPU
3047 * 3. Read by GPU
3048 * 4. Updated (written) by CPU again
3049 * 5. Read by GPU
3050 *
3051 * 1. Allocated
3052 * (CPU, CPU)
3053 * 2. Written by CPU
3054 * (CPU, CPU)
3055 * 3. Read by GPU
3056 * (CPU+RENDER, 0)
3057 * flush_domains = CPU
3058 * invalidate_domains = RENDER
3059 * clflush (obj)
3060 * MI_FLUSH
3061 * drm_agp_chipset_flush
3062 * 4. Updated (written) by CPU again
3063 * (CPU, CPU)
3064 * flush_domains = 0 (no previous write domain)
3065 * invalidate_domains = 0 (no new read domains)
3066 * 5. Read by GPU
3067 * (CPU+RENDER, 0)
3068 * flush_domains = CPU
3069 * invalidate_domains = RENDER
3070 * clflush (obj)
3071 * MI_FLUSH
3072 * drm_agp_chipset_flush
3073 */
3074static void
3075i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3076 struct intel_ring_buffer *ring)
3077{
3078 struct drm_device *dev = obj->dev;
3079 struct drm_i915_private *dev_priv = dev->dev_private;
3080 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3081 uint32_t invalidate_domains = 0;
3082 uint32_t flush_domains = 0;
3083 uint32_t old_read_domains;
3084
3085 intel_mark_busy(dev, obj);
3086
3087 /*
3088 * If the object isn't moving to a new write domain,
3089 * let the object stay in multiple read domains
3090 */
3091 if (obj->pending_write_domain == 0)
3092 obj->pending_read_domains |= obj->read_domains;
3093 else
3094 obj_priv->dirty = 1;
3095
3096 /*
3097 * Flush the current write domain if
3098 * the new read domains don't match. Invalidate
3099 * any read domains which differ from the old
3100 * write domain
3101 */
3102 if (obj->write_domain &&
3103 (obj->write_domain != obj->pending_read_domains ||
3104 obj_priv->ring != ring)) {
3105 flush_domains |= obj->write_domain;
3106 invalidate_domains |=
3107 obj->pending_read_domains & ~obj->write_domain;
3108 }
3109 /*
3110 * Invalidate any read caches which may have
3111 * stale data. That is, any new read domains.
3112 */
3113 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3114 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3115 i915_gem_clflush_object(obj);
3116
3117 old_read_domains = obj->read_domains;
3118
3119 /* The actual obj->write_domain will be updated with
3120 * pending_write_domain after we emit the accumulated flush for all
3121 * of our domain changes in execbuffers (which clears objects'
3122 * write_domains). So if we have a current write domain that we
3123 * aren't changing, set pending_write_domain to that.
3124 */
3125 if (flush_domains == 0 && obj->pending_write_domain == 0)
3126 obj->pending_write_domain = obj->write_domain;
3127 obj->read_domains = obj->pending_read_domains;
3128
3129 dev->invalidate_domains |= invalidate_domains;
3130 dev->flush_domains |= flush_domains;
3131 if (flush_domains & I915_GEM_GPU_DOMAINS)
3132 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3133 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3134 dev_priv->mm.flush_rings |= ring->id;
3135
3136 trace_i915_gem_object_change_domain(obj,
3137 old_read_domains,
3138 obj->write_domain);
3139}
3140
3141/** 3019/**
3142 * Moves the object from a partially CPU read to a full one. 3020 * Moves the object from a partially CPU read to a full one.
3143 * 3021 *
@@ -3145,30 +3023,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3145 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 3023 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3146 */ 3024 */
3147static void 3025static void
3148i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 3026i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3149{ 3027{
3150 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3028 if (!obj->page_cpu_valid)
3151
3152 if (!obj_priv->page_cpu_valid)
3153 return; 3029 return;
3154 3030
3155 /* If we're partially in the CPU read domain, finish moving it in. 3031 /* If we're partially in the CPU read domain, finish moving it in.
3156 */ 3032 */
3157 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 3033 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3158 int i; 3034 int i;
3159 3035
3160 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 3036 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3161 if (obj_priv->page_cpu_valid[i]) 3037 if (obj->page_cpu_valid[i])
3162 continue; 3038 continue;
3163 drm_clflush_pages(obj_priv->pages + i, 1); 3039 drm_clflush_pages(obj->pages + i, 1);
3164 } 3040 }
3165 } 3041 }
3166 3042
3167 /* Free the page_cpu_valid mappings which are now stale, whether 3043 /* Free the page_cpu_valid mappings which are now stale, whether
3168 * or not we've got I915_GEM_DOMAIN_CPU. 3044 * or not we've got I915_GEM_DOMAIN_CPU.
3169 */ 3045 */
3170 kfree(obj_priv->page_cpu_valid); 3046 kfree(obj->page_cpu_valid);
3171 obj_priv->page_cpu_valid = NULL; 3047 obj->page_cpu_valid = NULL;
3172} 3048}
3173 3049
3174/** 3050/**
@@ -3184,19 +3060,16 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3184 * flushes to occur. 3060 * flushes to occur.
3185 */ 3061 */
3186static int 3062static int
3187i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 3063i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3188 uint64_t offset, uint64_t size) 3064 uint64_t offset, uint64_t size)
3189{ 3065{
3190 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3191 uint32_t old_read_domains; 3066 uint32_t old_read_domains;
3192 int i, ret; 3067 int i, ret;
3193 3068
3194 if (offset == 0 && size == obj->size) 3069 if (offset == 0 && size == obj->base.size)
3195 return i915_gem_object_set_to_cpu_domain(obj, 0); 3070 return i915_gem_object_set_to_cpu_domain(obj, 0);
3196 3071
3197 ret = i915_gem_object_flush_gpu_write_domain(obj); 3072 i915_gem_object_flush_gpu_write_domain(obj);
3198 if (ret != 0)
3199 return ret;
3200 ret = i915_gem_object_wait_rendering(obj, true); 3073 ret = i915_gem_object_wait_rendering(obj, true);
3201 if (ret) 3074 if (ret)
3202 return ret; 3075 return ret;
@@ -3204,457 +3077,45 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3204 i915_gem_object_flush_gtt_write_domain(obj); 3077 i915_gem_object_flush_gtt_write_domain(obj);
3205 3078
3206 /* If we're already fully in the CPU read domain, we're done. */ 3079 /* If we're already fully in the CPU read domain, we're done. */
3207 if (obj_priv->page_cpu_valid == NULL && 3080 if (obj->page_cpu_valid == NULL &&
3208 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 3081 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3209 return 0; 3082 return 0;
3210 3083
3211 /* Otherwise, create/clear the per-page CPU read domain flag if we're 3084 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3212 * newly adding I915_GEM_DOMAIN_CPU 3085 * newly adding I915_GEM_DOMAIN_CPU
3213 */ 3086 */
3214 if (obj_priv->page_cpu_valid == NULL) { 3087 if (obj->page_cpu_valid == NULL) {
3215 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE, 3088 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3216 GFP_KERNEL); 3089 GFP_KERNEL);
3217 if (obj_priv->page_cpu_valid == NULL) 3090 if (obj->page_cpu_valid == NULL)
3218 return -ENOMEM; 3091 return -ENOMEM;
3219 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 3092 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3220 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 3093 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3221 3094
3222 /* Flush the cache on any pages that are still invalid from the CPU's 3095 /* Flush the cache on any pages that are still invalid from the CPU's
3223 * perspective. 3096 * perspective.
3224 */ 3097 */
3225 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 3098 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3226 i++) { 3099 i++) {
3227 if (obj_priv->page_cpu_valid[i]) 3100 if (obj->page_cpu_valid[i])
3228 continue; 3101 continue;
3229 3102
3230 drm_clflush_pages(obj_priv->pages + i, 1); 3103 drm_clflush_pages(obj->pages + i, 1);
3231 3104
3232 obj_priv->page_cpu_valid[i] = 1; 3105 obj->page_cpu_valid[i] = 1;
3233 } 3106 }
3234 3107
3235 /* It should now be out of any other write domains, and we can update 3108 /* It should now be out of any other write domains, and we can update
3236 * the domain values for our changes. 3109 * the domain values for our changes.
3237 */ 3110 */
3238 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3111 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3239 3112
3240 old_read_domains = obj->read_domains; 3113 old_read_domains = obj->base.read_domains;
3241 obj->read_domains |= I915_GEM_DOMAIN_CPU; 3114 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3242 3115
3243 trace_i915_gem_object_change_domain(obj, 3116 trace_i915_gem_object_change_domain(obj,
3244 old_read_domains, 3117 old_read_domains,
3245 obj->write_domain); 3118 obj->base.write_domain);
3246
3247 return 0;
3248}
3249
3250static int
3251i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3252 struct drm_file *file_priv,
3253 struct drm_i915_gem_exec_object2 *entry,
3254 struct drm_i915_gem_relocation_entry *reloc)
3255{
3256 struct drm_device *dev = obj->base.dev;
3257 struct drm_gem_object *target_obj;
3258 uint32_t target_offset;
3259 int ret = -EINVAL;
3260
3261 target_obj = drm_gem_object_lookup(dev, file_priv,
3262 reloc->target_handle);
3263 if (target_obj == NULL)
3264 return -ENOENT;
3265
3266 target_offset = to_intel_bo(target_obj)->gtt_offset;
3267
3268#if WATCH_RELOC
3269 DRM_INFO("%s: obj %p offset %08x target %d "
3270 "read %08x write %08x gtt %08x "
3271 "presumed %08x delta %08x\n",
3272 __func__,
3273 obj,
3274 (int) reloc->offset,
3275 (int) reloc->target_handle,
3276 (int) reloc->read_domains,
3277 (int) reloc->write_domain,
3278 (int) target_offset,
3279 (int) reloc->presumed_offset,
3280 reloc->delta);
3281#endif
3282
3283 /* The target buffer should have appeared before us in the
3284 * exec_object list, so it should have a GTT space bound by now.
3285 */
3286 if (target_offset == 0) {
3287 DRM_ERROR("No GTT space found for object %d\n",
3288 reloc->target_handle);
3289 goto err;
3290 }
3291
3292 /* Validate that the target is in a valid r/w GPU domain */
3293 if (reloc->write_domain & (reloc->write_domain - 1)) {
3294 DRM_ERROR("reloc with multiple write domains: "
3295 "obj %p target %d offset %d "
3296 "read %08x write %08x",
3297 obj, reloc->target_handle,
3298 (int) reloc->offset,
3299 reloc->read_domains,
3300 reloc->write_domain);
3301 goto err;
3302 }
3303 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3304 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3305 DRM_ERROR("reloc with read/write CPU domains: "
3306 "obj %p target %d offset %d "
3307 "read %08x write %08x",
3308 obj, reloc->target_handle,
3309 (int) reloc->offset,
3310 reloc->read_domains,
3311 reloc->write_domain);
3312 goto err;
3313 }
3314 if (reloc->write_domain && target_obj->pending_write_domain &&
3315 reloc->write_domain != target_obj->pending_write_domain) {
3316 DRM_ERROR("Write domain conflict: "
3317 "obj %p target %d offset %d "
3318 "new %08x old %08x\n",
3319 obj, reloc->target_handle,
3320 (int) reloc->offset,
3321 reloc->write_domain,
3322 target_obj->pending_write_domain);
3323 goto err;
3324 }
3325
3326 target_obj->pending_read_domains |= reloc->read_domains;
3327 target_obj->pending_write_domain |= reloc->write_domain;
3328
3329 /* If the relocation already has the right value in it, no
3330 * more work needs to be done.
3331 */
3332 if (target_offset == reloc->presumed_offset)
3333 goto out;
3334
3335 /* Check that the relocation address is valid... */
3336 if (reloc->offset > obj->base.size - 4) {
3337 DRM_ERROR("Relocation beyond object bounds: "
3338 "obj %p target %d offset %d size %d.\n",
3339 obj, reloc->target_handle,
3340 (int) reloc->offset,
3341 (int) obj->base.size);
3342 goto err;
3343 }
3344 if (reloc->offset & 3) {
3345 DRM_ERROR("Relocation not 4-byte aligned: "
3346 "obj %p target %d offset %d.\n",
3347 obj, reloc->target_handle,
3348 (int) reloc->offset);
3349 goto err;
3350 }
3351
3352 /* and points to somewhere within the target object. */
3353 if (reloc->delta >= target_obj->size) {
3354 DRM_ERROR("Relocation beyond target object bounds: "
3355 "obj %p target %d delta %d size %d.\n",
3356 obj, reloc->target_handle,
3357 (int) reloc->delta,
3358 (int) target_obj->size);
3359 goto err;
3360 }
3361
3362 reloc->delta += target_offset;
3363 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3364 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3365 char *vaddr;
3366
3367 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3368 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3369 kunmap_atomic(vaddr);
3370 } else {
3371 struct drm_i915_private *dev_priv = dev->dev_private;
3372 uint32_t __iomem *reloc_entry;
3373 void __iomem *reloc_page;
3374
3375 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3376 if (ret)
3377 goto err;
3378
3379 /* Map the page containing the relocation we're going to perform. */
3380 reloc->offset += obj->gtt_offset;
3381 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3382 reloc->offset & PAGE_MASK);
3383 reloc_entry = (uint32_t __iomem *)
3384 (reloc_page + (reloc->offset & ~PAGE_MASK));
3385 iowrite32(reloc->delta, reloc_entry);
3386 io_mapping_unmap_atomic(reloc_page);
3387 }
3388
3389 /* and update the user's relocation entry */
3390 reloc->presumed_offset = target_offset;
3391
3392out:
3393 ret = 0;
3394err:
3395 drm_gem_object_unreference(target_obj);
3396 return ret;
3397}
3398
3399static int
3400i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3401 struct drm_file *file_priv,
3402 struct drm_i915_gem_exec_object2 *entry)
3403{
3404 struct drm_i915_gem_relocation_entry __user *user_relocs;
3405 int i, ret;
3406
3407 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3408 for (i = 0; i < entry->relocation_count; i++) {
3409 struct drm_i915_gem_relocation_entry reloc;
3410
3411 if (__copy_from_user_inatomic(&reloc,
3412 user_relocs+i,
3413 sizeof(reloc)))
3414 return -EFAULT;
3415
3416 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3417 if (ret)
3418 return ret;
3419
3420 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3421 &reloc.presumed_offset,
3422 sizeof(reloc.presumed_offset)))
3423 return -EFAULT;
3424 }
3425
3426 return 0;
3427}
3428
3429static int
3430i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3431 struct drm_file *file_priv,
3432 struct drm_i915_gem_exec_object2 *entry,
3433 struct drm_i915_gem_relocation_entry *relocs)
3434{
3435 int i, ret;
3436
3437 for (i = 0; i < entry->relocation_count; i++) {
3438 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3439 if (ret)
3440 return ret;
3441 }
3442
3443 return 0;
3444}
3445
3446static int
3447i915_gem_execbuffer_relocate(struct drm_device *dev,
3448 struct drm_file *file,
3449 struct drm_gem_object **object_list,
3450 struct drm_i915_gem_exec_object2 *exec_list,
3451 int count)
3452{
3453 int i, ret;
3454
3455 for (i = 0; i < count; i++) {
3456 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3457 obj->base.pending_read_domains = 0;
3458 obj->base.pending_write_domain = 0;
3459 ret = i915_gem_execbuffer_relocate_object(obj, file,
3460 &exec_list[i]);
3461 if (ret)
3462 return ret;
3463 }
3464
3465 return 0;
3466}
3467
3468static int
3469i915_gem_execbuffer_reserve(struct drm_device *dev,
3470 struct drm_file *file,
3471 struct drm_gem_object **object_list,
3472 struct drm_i915_gem_exec_object2 *exec_list,
3473 int count)
3474{
3475 struct drm_i915_private *dev_priv = dev->dev_private;
3476 int ret, i, retry;
3477
3478 /* attempt to pin all of the buffers into the GTT */
3479 for (retry = 0; retry < 2; retry++) {
3480 ret = 0;
3481 for (i = 0; i < count; i++) {
3482 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3483 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3484 bool need_fence =
3485 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3486 obj->tiling_mode != I915_TILING_NONE;
3487
3488 /* Check fence reg constraints and rebind if necessary */
3489 if (need_fence &&
3490 !i915_gem_object_fence_offset_ok(&obj->base,
3491 obj->tiling_mode)) {
3492 ret = i915_gem_object_unbind(&obj->base);
3493 if (ret)
3494 break;
3495 }
3496
3497 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3498 if (ret)
3499 break;
3500
3501 /*
3502 * Pre-965 chips need a fence register set up in order
3503 * to properly handle blits to/from tiled surfaces.
3504 */
3505 if (need_fence) {
3506 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3507 if (ret) {
3508 i915_gem_object_unpin(&obj->base);
3509 break;
3510 }
3511
3512 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3513 }
3514
3515 entry->offset = obj->gtt_offset;
3516 }
3517
3518 while (i--)
3519 i915_gem_object_unpin(object_list[i]);
3520
3521 if (ret == 0)
3522 break;
3523
3524 if (ret != -ENOSPC || retry)
3525 return ret;
3526
3527 ret = i915_gem_evict_everything(dev);
3528 if (ret)
3529 return ret;
3530 }
3531
3532 return 0;
3533}
3534
3535static int
3536i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3537 struct drm_file *file,
3538 struct drm_gem_object **object_list,
3539 struct drm_i915_gem_exec_object2 *exec_list,
3540 int count)
3541{
3542 struct drm_i915_gem_relocation_entry *reloc;
3543 int i, total, ret;
3544
3545 for (i = 0; i < count; i++) {
3546 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3547 obj->in_execbuffer = false;
3548 }
3549
3550 mutex_unlock(&dev->struct_mutex);
3551
3552 total = 0;
3553 for (i = 0; i < count; i++)
3554 total += exec_list[i].relocation_count;
3555
3556 reloc = drm_malloc_ab(total, sizeof(*reloc));
3557 if (reloc == NULL) {
3558 mutex_lock(&dev->struct_mutex);
3559 return -ENOMEM;
3560 }
3561
3562 total = 0;
3563 for (i = 0; i < count; i++) {
3564 struct drm_i915_gem_relocation_entry __user *user_relocs;
3565
3566 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3567
3568 if (copy_from_user(reloc+total, user_relocs,
3569 exec_list[i].relocation_count *
3570 sizeof(*reloc))) {
3571 ret = -EFAULT;
3572 mutex_lock(&dev->struct_mutex);
3573 goto err;
3574 }
3575
3576 total += exec_list[i].relocation_count;
3577 }
3578
3579 ret = i915_mutex_lock_interruptible(dev);
3580 if (ret) {
3581 mutex_lock(&dev->struct_mutex);
3582 goto err;
3583 }
3584
3585 ret = i915_gem_execbuffer_reserve(dev, file,
3586 object_list, exec_list,
3587 count);
3588 if (ret)
3589 goto err;
3590
3591 total = 0;
3592 for (i = 0; i < count; i++) {
3593 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3594 obj->base.pending_read_domains = 0;
3595 obj->base.pending_write_domain = 0;
3596 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3597 &exec_list[i],
3598 reloc + total);
3599 if (ret)
3600 goto err;
3601
3602 total += exec_list[i].relocation_count;
3603 }
3604
3605 /* Leave the user relocations as are, this is the painfully slow path,
3606 * and we want to avoid the complication of dropping the lock whilst
3607 * having buffers reserved in the aperture and so causing spurious
3608 * ENOSPC for random operations.
3609 */
3610
3611err:
3612 drm_free_large(reloc);
3613 return ret;
3614}
3615
3616static int
3617i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3618 struct drm_file *file,
3619 struct intel_ring_buffer *ring,
3620 struct drm_gem_object **objects,
3621 int count)
3622{
3623 struct drm_i915_private *dev_priv = dev->dev_private;
3624 int ret, i;
3625
3626 /* Zero the global flush/invalidate flags. These
3627 * will be modified as new domains are computed
3628 * for each object
3629 */
3630 dev->invalidate_domains = 0;
3631 dev->flush_domains = 0;
3632 dev_priv->mm.flush_rings = 0;
3633 for (i = 0; i < count; i++)
3634 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3635
3636 if (dev->invalidate_domains | dev->flush_domains) {
3637#if WATCH_EXEC
3638 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3639 __func__,
3640 dev->invalidate_domains,
3641 dev->flush_domains);
3642#endif
3643 i915_gem_flush(dev, file,
3644 dev->invalidate_domains,
3645 dev->flush_domains,
3646 dev_priv->mm.flush_rings);
3647 }
3648
3649 for (i = 0; i < count; i++) {
3650 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3651 /* XXX replace with semaphores */
3652 if (obj->ring && ring != obj->ring) {
3653 ret = i915_gem_object_wait_rendering(&obj->base, true);
3654 if (ret)
3655 return ret;
3656 }
3657 }
3658 3119
3659 return 0; 3120 return 0;
3660} 3121}
@@ -3694,599 +3155,129 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3694 return 0; 3155 return 0;
3695 3156
3696 ret = 0; 3157 ret = 0;
3697 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 3158 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3698 /* And wait for the seqno passing without holding any locks and 3159 /* And wait for the seqno passing without holding any locks and
3699 * causing extra latency for others. This is safe as the irq 3160 * causing extra latency for others. This is safe as the irq
3700 * generation is designed to be run atomically and so is 3161 * generation is designed to be run atomically and so is
3701 * lockless. 3162 * lockless.
3702 */ 3163 */
3703 ring->user_irq_get(dev, ring); 3164 if (ring->irq_get(ring)) {
3704 ret = wait_event_interruptible(ring->irq_queue, 3165 ret = wait_event_interruptible(ring->irq_queue,
3705 i915_seqno_passed(ring->get_seqno(dev, ring), seqno) 3166 i915_seqno_passed(ring->get_seqno(ring), seqno)
3706 || atomic_read(&dev_priv->mm.wedged)); 3167 || atomic_read(&dev_priv->mm.wedged));
3707 ring->user_irq_put(dev, ring); 3168 ring->irq_put(ring);
3708
3709 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3710 ret = -EIO;
3711 }
3712
3713 if (ret == 0)
3714 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3715
3716 return ret;
3717}
3718
3719static int
3720i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3721 uint64_t exec_offset)
3722{
3723 uint32_t exec_start, exec_len;
3724
3725 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3726 exec_len = (uint32_t) exec->batch_len;
3727
3728 if ((exec_start | exec_len) & 0x7)
3729 return -EINVAL;
3730
3731 if (!exec_start)
3732 return -EINVAL;
3733
3734 return 0;
3735}
3736
3737static int
3738validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3739 int count)
3740{
3741 int i;
3742
3743 for (i = 0; i < count; i++) {
3744 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3745 int length; /* limited by fault_in_pages_readable() */
3746
3747 /* First check for malicious input causing overflow */
3748 if (exec[i].relocation_count >
3749 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3750 return -EINVAL;
3751
3752 length = exec[i].relocation_count *
3753 sizeof(struct drm_i915_gem_relocation_entry);
3754 if (!access_ok(VERIFY_READ, ptr, length))
3755 return -EFAULT;
3756
3757 /* we may also need to update the presumed offsets */
3758 if (!access_ok(VERIFY_WRITE, ptr, length))
3759 return -EFAULT;
3760
3761 if (fault_in_pages_readable(ptr, length))
3762 return -EFAULT;
3763 }
3764
3765 return 0;
3766}
3767
3768static int
3769i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3770 struct drm_file *file,
3771 struct drm_i915_gem_execbuffer2 *args,
3772 struct drm_i915_gem_exec_object2 *exec_list)
3773{
3774 drm_i915_private_t *dev_priv = dev->dev_private;
3775 struct drm_gem_object **object_list = NULL;
3776 struct drm_gem_object *batch_obj;
3777 struct drm_i915_gem_object *obj_priv;
3778 struct drm_clip_rect *cliprects = NULL;
3779 struct drm_i915_gem_request *request = NULL;
3780 int ret, i, flips;
3781 uint64_t exec_offset;
3782
3783 struct intel_ring_buffer *ring = NULL;
3784
3785 ret = i915_gem_check_is_wedged(dev);
3786 if (ret)
3787 return ret;
3788
3789 ret = validate_exec_list(exec_list, args->buffer_count);
3790 if (ret)
3791 return ret;
3792
3793#if WATCH_EXEC
3794 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3795 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3796#endif
3797 switch (args->flags & I915_EXEC_RING_MASK) {
3798 case I915_EXEC_DEFAULT:
3799 case I915_EXEC_RENDER:
3800 ring = &dev_priv->render_ring;
3801 break;
3802 case I915_EXEC_BSD:
3803 if (!HAS_BSD(dev)) {
3804 DRM_ERROR("execbuf with invalid ring (BSD)\n");
3805 return -EINVAL;
3806 }
3807 ring = &dev_priv->bsd_ring;
3808 break;
3809 case I915_EXEC_BLT:
3810 if (!HAS_BLT(dev)) {
3811 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3812 return -EINVAL;
3813 }
3814 ring = &dev_priv->blt_ring;
3815 break;
3816 default:
3817 DRM_ERROR("execbuf with unknown ring: %d\n",
3818 (int)(args->flags & I915_EXEC_RING_MASK));
3819 return -EINVAL;
3820 }
3821
3822 if (args->buffer_count < 1) {
3823 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3824 return -EINVAL;
3825 }
3826 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3827 if (object_list == NULL) {
3828 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3829 args->buffer_count);
3830 ret = -ENOMEM;
3831 goto pre_mutex_err;
3832 }
3833
3834 if (args->num_cliprects != 0) {
3835 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3836 GFP_KERNEL);
3837 if (cliprects == NULL) {
3838 ret = -ENOMEM;
3839 goto pre_mutex_err;
3840 }
3841
3842 ret = copy_from_user(cliprects,
3843 (struct drm_clip_rect __user *)
3844 (uintptr_t) args->cliprects_ptr,
3845 sizeof(*cliprects) * args->num_cliprects);
3846 if (ret != 0) {
3847 DRM_ERROR("copy %d cliprects failed: %d\n",
3848 args->num_cliprects, ret);
3849 ret = -EFAULT;
3850 goto pre_mutex_err;
3851 }
3852 }
3853
3854 request = kzalloc(sizeof(*request), GFP_KERNEL);
3855 if (request == NULL) {
3856 ret = -ENOMEM;
3857 goto pre_mutex_err;
3858 }
3859
3860 ret = i915_mutex_lock_interruptible(dev);
3861 if (ret)
3862 goto pre_mutex_err;
3863
3864 if (dev_priv->mm.suspended) {
3865 mutex_unlock(&dev->struct_mutex);
3866 ret = -EBUSY;
3867 goto pre_mutex_err;
3868 }
3869
3870 /* Look up object handles */
3871 for (i = 0; i < args->buffer_count; i++) {
3872 object_list[i] = drm_gem_object_lookup(dev, file,
3873 exec_list[i].handle);
3874 if (object_list[i] == NULL) {
3875 DRM_ERROR("Invalid object handle %d at index %d\n",
3876 exec_list[i].handle, i);
3877 /* prevent error path from reading uninitialized data */
3878 args->buffer_count = i + 1;
3879 ret = -ENOENT;
3880 goto err;
3881 }
3882
3883 obj_priv = to_intel_bo(object_list[i]);
3884 if (obj_priv->in_execbuffer) {
3885 DRM_ERROR("Object %p appears more than once in object list\n",
3886 object_list[i]);
3887 /* prevent error path from reading uninitialized data */
3888 args->buffer_count = i + 1;
3889 ret = -EINVAL;
3890 goto err;
3891 }
3892 obj_priv->in_execbuffer = true;
3893 }
3894
3895 /* Move the objects en-masse into the GTT, evicting if necessary. */
3896 ret = i915_gem_execbuffer_reserve(dev, file,
3897 object_list, exec_list,
3898 args->buffer_count);
3899 if (ret)
3900 goto err;
3901
3902 /* The objects are in their final locations, apply the relocations. */
3903 ret = i915_gem_execbuffer_relocate(dev, file,
3904 object_list, exec_list,
3905 args->buffer_count);
3906 if (ret) {
3907 if (ret == -EFAULT) {
3908 ret = i915_gem_execbuffer_relocate_slow(dev, file,
3909 object_list,
3910 exec_list,
3911 args->buffer_count);
3912 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3913 }
3914 if (ret)
3915 goto err;
3916 }
3917
3918 /* Set the pending read domains for the batch buffer to COMMAND */
3919 batch_obj = object_list[args->buffer_count-1];
3920 if (batch_obj->pending_write_domain) {
3921 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3922 ret = -EINVAL;
3923 goto err;
3924 }
3925 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3926
3927 /* Sanity check the batch buffer */
3928 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3929 ret = i915_gem_check_execbuffer(args, exec_offset);
3930 if (ret != 0) {
3931 DRM_ERROR("execbuf with invalid offset/length\n");
3932 goto err;
3933 }
3934
3935 ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3936 object_list, args->buffer_count);
3937 if (ret)
3938 goto err;
3939
3940 for (i = 0; i < args->buffer_count; i++) {
3941 struct drm_gem_object *obj = object_list[i];
3942 uint32_t old_write_domain = obj->write_domain;
3943 obj->write_domain = obj->pending_write_domain;
3944 trace_i915_gem_object_change_domain(obj,
3945 obj->read_domains,
3946 old_write_domain);
3947 }
3948
3949#if WATCH_COHERENCY
3950 for (i = 0; i < args->buffer_count; i++) {
3951 i915_gem_object_check_coherency(object_list[i],
3952 exec_list[i].handle);
3953 }
3954#endif
3955
3956#if WATCH_EXEC
3957 i915_gem_dump_object(batch_obj,
3958 args->batch_len,
3959 __func__,
3960 ~0);
3961#endif
3962
3963 /* Check for any pending flips. As we only maintain a flip queue depth
3964 * of 1, we can simply insert a WAIT for the next display flip prior
3965 * to executing the batch and avoid stalling the CPU.
3966 */
3967 flips = 0;
3968 for (i = 0; i < args->buffer_count; i++) {
3969 if (object_list[i]->write_domain)
3970 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3971 }
3972 if (flips) {
3973 int plane, flip_mask;
3974
3975 for (plane = 0; flips >> plane; plane++) {
3976 if (((flips >> plane) & 1) == 0)
3977 continue;
3978
3979 if (plane)
3980 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3981 else
3982 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3983
3984 intel_ring_begin(dev, ring, 2);
3985 intel_ring_emit(dev, ring,
3986 MI_WAIT_FOR_EVENT | flip_mask);
3987 intel_ring_emit(dev, ring, MI_NOOP);
3988 intel_ring_advance(dev, ring);
3989 }
3990 }
3991
3992 /* Exec the batchbuffer */
3993 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3994 cliprects, exec_offset);
3995 if (ret) {
3996 DRM_ERROR("dispatch failed %d\n", ret);
3997 goto err;
3998 }
3999
4000 /*
4001 * Ensure that the commands in the batch buffer are
4002 * finished before the interrupt fires
4003 */
4004 i915_retire_commands(dev, ring);
4005
4006 for (i = 0; i < args->buffer_count; i++) {
4007 struct drm_gem_object *obj = object_list[i];
4008
4009 i915_gem_object_move_to_active(obj, ring);
4010 if (obj->write_domain)
4011 list_move_tail(&to_intel_bo(obj)->gpu_write_list,
4012 &ring->gpu_write_list);
4013 }
4014
4015 i915_add_request(dev, file, request, ring);
4016 request = NULL;
4017
4018err:
4019 for (i = 0; i < args->buffer_count; i++) {
4020 if (object_list[i]) {
4021 obj_priv = to_intel_bo(object_list[i]);
4022 obj_priv->in_execbuffer = false;
4023 }
4024 drm_gem_object_unreference(object_list[i]);
4025 }
4026
4027 mutex_unlock(&dev->struct_mutex);
4028
4029pre_mutex_err:
4030 drm_free_large(object_list);
4031 kfree(cliprects);
4032 kfree(request);
4033
4034 return ret;
4035}
4036
4037/*
4038 * Legacy execbuffer just creates an exec2 list from the original exec object
4039 * list array and passes it to the real function.
4040 */
4041int
4042i915_gem_execbuffer(struct drm_device *dev, void *data,
4043 struct drm_file *file_priv)
4044{
4045 struct drm_i915_gem_execbuffer *args = data;
4046 struct drm_i915_gem_execbuffer2 exec2;
4047 struct drm_i915_gem_exec_object *exec_list = NULL;
4048 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4049 int ret, i;
4050
4051#if WATCH_EXEC
4052 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4053 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4054#endif
4055
4056 if (args->buffer_count < 1) {
4057 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4058 return -EINVAL;
4059 }
4060
4061 /* Copy in the exec list from userland */
4062 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4063 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4064 if (exec_list == NULL || exec2_list == NULL) {
4065 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4066 args->buffer_count);
4067 drm_free_large(exec_list);
4068 drm_free_large(exec2_list);
4069 return -ENOMEM;
4070 }
4071 ret = copy_from_user(exec_list,
4072 (struct drm_i915_relocation_entry __user *)
4073 (uintptr_t) args->buffers_ptr,
4074 sizeof(*exec_list) * args->buffer_count);
4075 if (ret != 0) {
4076 DRM_ERROR("copy %d exec entries failed %d\n",
4077 args->buffer_count, ret);
4078 drm_free_large(exec_list);
4079 drm_free_large(exec2_list);
4080 return -EFAULT;
4081 }
4082
4083 for (i = 0; i < args->buffer_count; i++) {
4084 exec2_list[i].handle = exec_list[i].handle;
4085 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4086 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4087 exec2_list[i].alignment = exec_list[i].alignment;
4088 exec2_list[i].offset = exec_list[i].offset;
4089 if (INTEL_INFO(dev)->gen < 4)
4090 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4091 else
4092 exec2_list[i].flags = 0;
4093 }
4094 3169
4095 exec2.buffers_ptr = args->buffers_ptr; 3170 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
4096 exec2.buffer_count = args->buffer_count; 3171 ret = -EIO;
4097 exec2.batch_start_offset = args->batch_start_offset;
4098 exec2.batch_len = args->batch_len;
4099 exec2.DR1 = args->DR1;
4100 exec2.DR4 = args->DR4;
4101 exec2.num_cliprects = args->num_cliprects;
4102 exec2.cliprects_ptr = args->cliprects_ptr;
4103 exec2.flags = I915_EXEC_RENDER;
4104
4105 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4106 if (!ret) {
4107 /* Copy the new buffer offsets back to the user's exec list. */
4108 for (i = 0; i < args->buffer_count; i++)
4109 exec_list[i].offset = exec2_list[i].offset;
4110 /* ... and back out to userspace */
4111 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4112 (uintptr_t) args->buffers_ptr,
4113 exec_list,
4114 sizeof(*exec_list) * args->buffer_count);
4115 if (ret) {
4116 ret = -EFAULT;
4117 DRM_ERROR("failed to copy %d exec entries "
4118 "back to user (%d)\n",
4119 args->buffer_count, ret);
4120 } 3172 }
4121 } 3173 }
4122 3174
4123 drm_free_large(exec_list); 3175 if (ret == 0)
4124 drm_free_large(exec2_list); 3176 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4125 return ret;
4126}
4127
4128int
4129i915_gem_execbuffer2(struct drm_device *dev, void *data,
4130 struct drm_file *file_priv)
4131{
4132 struct drm_i915_gem_execbuffer2 *args = data;
4133 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4134 int ret;
4135
4136#if WATCH_EXEC
4137 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4138 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4139#endif
4140
4141 if (args->buffer_count < 1) {
4142 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4143 return -EINVAL;
4144 }
4145
4146 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4147 if (exec2_list == NULL) {
4148 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4149 args->buffer_count);
4150 return -ENOMEM;
4151 }
4152 ret = copy_from_user(exec2_list,
4153 (struct drm_i915_relocation_entry __user *)
4154 (uintptr_t) args->buffers_ptr,
4155 sizeof(*exec2_list) * args->buffer_count);
4156 if (ret != 0) {
4157 DRM_ERROR("copy %d exec entries failed %d\n",
4158 args->buffer_count, ret);
4159 drm_free_large(exec2_list);
4160 return -EFAULT;
4161 }
4162
4163 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4164 if (!ret) {
4165 /* Copy the new buffer offsets back to the user's exec list. */
4166 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4167 (uintptr_t) args->buffers_ptr,
4168 exec2_list,
4169 sizeof(*exec2_list) * args->buffer_count);
4170 if (ret) {
4171 ret = -EFAULT;
4172 DRM_ERROR("failed to copy %d exec entries "
4173 "back to user (%d)\n",
4174 args->buffer_count, ret);
4175 }
4176 }
4177 3177
4178 drm_free_large(exec2_list);
4179 return ret; 3178 return ret;
4180} 3179}
4181 3180
4182int 3181int
4183i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 3182i915_gem_object_pin(struct drm_i915_gem_object *obj,
3183 uint32_t alignment,
3184 bool map_and_fenceable)
4184{ 3185{
4185 struct drm_device *dev = obj->dev; 3186 struct drm_device *dev = obj->base.dev;
4186 struct drm_i915_private *dev_priv = dev->dev_private; 3187 struct drm_i915_private *dev_priv = dev->dev_private;
4187 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4188 int ret; 3188 int ret;
4189 3189
4190 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); 3190 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4191 WARN_ON(i915_verify_lists(dev)); 3191 WARN_ON(i915_verify_lists(dev));
4192 3192
4193 if (obj_priv->gtt_space != NULL) { 3193 if (obj->gtt_space != NULL) {
4194 if (alignment == 0) 3194 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4195 alignment = i915_gem_get_gtt_alignment(obj); 3195 (map_and_fenceable && !obj->map_and_fenceable)) {
4196 if (obj_priv->gtt_offset & (alignment - 1)) { 3196 WARN(obj->pin_count,
4197 WARN(obj_priv->pin_count, 3197 "bo is already pinned with incorrect alignment:"
4198 "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n", 3198 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4199 obj_priv->gtt_offset, alignment); 3199 " obj->map_and_fenceable=%d\n",
3200 obj->gtt_offset, alignment,
3201 map_and_fenceable,
3202 obj->map_and_fenceable);
4200 ret = i915_gem_object_unbind(obj); 3203 ret = i915_gem_object_unbind(obj);
4201 if (ret) 3204 if (ret)
4202 return ret; 3205 return ret;
4203 } 3206 }
4204 } 3207 }
4205 3208
4206 if (obj_priv->gtt_space == NULL) { 3209 if (obj->gtt_space == NULL) {
4207 ret = i915_gem_object_bind_to_gtt(obj, alignment); 3210 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3211 map_and_fenceable);
4208 if (ret) 3212 if (ret)
4209 return ret; 3213 return ret;
4210 } 3214 }
4211 3215
4212 obj_priv->pin_count++; 3216 if (obj->pin_count++ == 0) {
4213 3217 if (!obj->active)
4214 /* If the object is not active and not pending a flush, 3218 list_move_tail(&obj->mm_list,
4215 * remove it from the inactive list
4216 */
4217 if (obj_priv->pin_count == 1) {
4218 i915_gem_info_add_pin(dev_priv, obj->size);
4219 if (!obj_priv->active)
4220 list_move_tail(&obj_priv->mm_list,
4221 &dev_priv->mm.pinned_list); 3219 &dev_priv->mm.pinned_list);
4222 } 3220 }
3221 obj->pin_mappable |= map_and_fenceable;
4223 3222
4224 WARN_ON(i915_verify_lists(dev)); 3223 WARN_ON(i915_verify_lists(dev));
4225 return 0; 3224 return 0;
4226} 3225}
4227 3226
4228void 3227void
4229i915_gem_object_unpin(struct drm_gem_object *obj) 3228i915_gem_object_unpin(struct drm_i915_gem_object *obj)
4230{ 3229{
4231 struct drm_device *dev = obj->dev; 3230 struct drm_device *dev = obj->base.dev;
4232 drm_i915_private_t *dev_priv = dev->dev_private; 3231 drm_i915_private_t *dev_priv = dev->dev_private;
4233 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4234 3232
4235 WARN_ON(i915_verify_lists(dev)); 3233 WARN_ON(i915_verify_lists(dev));
4236 obj_priv->pin_count--; 3234 BUG_ON(obj->pin_count == 0);
4237 BUG_ON(obj_priv->pin_count < 0); 3235 BUG_ON(obj->gtt_space == NULL);
4238 BUG_ON(obj_priv->gtt_space == NULL);
4239 3236
4240 /* If the object is no longer pinned, and is 3237 if (--obj->pin_count == 0) {
4241 * neither active nor being flushed, then stick it on 3238 if (!obj->active)
4242 * the inactive list 3239 list_move_tail(&obj->mm_list,
4243 */
4244 if (obj_priv->pin_count == 0) {
4245 if (!obj_priv->active)
4246 list_move_tail(&obj_priv->mm_list,
4247 &dev_priv->mm.inactive_list); 3240 &dev_priv->mm.inactive_list);
4248 i915_gem_info_remove_pin(dev_priv, obj->size); 3241 obj->pin_mappable = false;
4249 } 3242 }
4250 WARN_ON(i915_verify_lists(dev)); 3243 WARN_ON(i915_verify_lists(dev));
4251} 3244}
4252 3245
4253int 3246int
4254i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3247i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4255 struct drm_file *file_priv) 3248 struct drm_file *file)
4256{ 3249{
4257 struct drm_i915_gem_pin *args = data; 3250 struct drm_i915_gem_pin *args = data;
4258 struct drm_gem_object *obj; 3251 struct drm_i915_gem_object *obj;
4259 struct drm_i915_gem_object *obj_priv;
4260 int ret; 3252 int ret;
4261 3253
4262 ret = i915_mutex_lock_interruptible(dev); 3254 ret = i915_mutex_lock_interruptible(dev);
4263 if (ret) 3255 if (ret)
4264 return ret; 3256 return ret;
4265 3257
4266 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3258 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4267 if (obj == NULL) { 3259 if (obj == NULL) {
4268 ret = -ENOENT; 3260 ret = -ENOENT;
4269 goto unlock; 3261 goto unlock;
4270 } 3262 }
4271 obj_priv = to_intel_bo(obj);
4272 3263
4273 if (obj_priv->madv != I915_MADV_WILLNEED) { 3264 if (obj->madv != I915_MADV_WILLNEED) {
4274 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3265 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4275 ret = -EINVAL; 3266 ret = -EINVAL;
4276 goto out; 3267 goto out;
4277 } 3268 }
4278 3269
4279 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 3270 if (obj->pin_filp != NULL && obj->pin_filp != file) {
4280 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3271 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4281 args->handle); 3272 args->handle);
4282 ret = -EINVAL; 3273 ret = -EINVAL;
4283 goto out; 3274 goto out;
4284 } 3275 }
4285 3276
4286 obj_priv->user_pin_count++; 3277 obj->user_pin_count++;
4287 obj_priv->pin_filp = file_priv; 3278 obj->pin_filp = file;
4288 if (obj_priv->user_pin_count == 1) { 3279 if (obj->user_pin_count == 1) {
4289 ret = i915_gem_object_pin(obj, args->alignment); 3280 ret = i915_gem_object_pin(obj, args->alignment, true);
4290 if (ret) 3281 if (ret)
4291 goto out; 3282 goto out;
4292 } 3283 }
@@ -4295,9 +3286,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4295 * as the X server doesn't manage domains yet 3286 * as the X server doesn't manage domains yet
4296 */ 3287 */
4297 i915_gem_object_flush_cpu_write_domain(obj); 3288 i915_gem_object_flush_cpu_write_domain(obj);
4298 args->offset = obj_priv->gtt_offset; 3289 args->offset = obj->gtt_offset;
4299out: 3290out:
4300 drm_gem_object_unreference(obj); 3291 drm_gem_object_unreference(&obj->base);
4301unlock: 3292unlock:
4302 mutex_unlock(&dev->struct_mutex); 3293 mutex_unlock(&dev->struct_mutex);
4303 return ret; 3294 return ret;
@@ -4305,38 +3296,36 @@ unlock:
4305 3296
4306int 3297int
4307i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3298i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4308 struct drm_file *file_priv) 3299 struct drm_file *file)
4309{ 3300{
4310 struct drm_i915_gem_pin *args = data; 3301 struct drm_i915_gem_pin *args = data;
4311 struct drm_gem_object *obj; 3302 struct drm_i915_gem_object *obj;
4312 struct drm_i915_gem_object *obj_priv;
4313 int ret; 3303 int ret;
4314 3304
4315 ret = i915_mutex_lock_interruptible(dev); 3305 ret = i915_mutex_lock_interruptible(dev);
4316 if (ret) 3306 if (ret)
4317 return ret; 3307 return ret;
4318 3308
4319 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3309 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4320 if (obj == NULL) { 3310 if (obj == NULL) {
4321 ret = -ENOENT; 3311 ret = -ENOENT;
4322 goto unlock; 3312 goto unlock;
4323 } 3313 }
4324 obj_priv = to_intel_bo(obj);
4325 3314
4326 if (obj_priv->pin_filp != file_priv) { 3315 if (obj->pin_filp != file) {
4327 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3316 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4328 args->handle); 3317 args->handle);
4329 ret = -EINVAL; 3318 ret = -EINVAL;
4330 goto out; 3319 goto out;
4331 } 3320 }
4332 obj_priv->user_pin_count--; 3321 obj->user_pin_count--;
4333 if (obj_priv->user_pin_count == 0) { 3322 if (obj->user_pin_count == 0) {
4334 obj_priv->pin_filp = NULL; 3323 obj->pin_filp = NULL;
4335 i915_gem_object_unpin(obj); 3324 i915_gem_object_unpin(obj);
4336 } 3325 }
4337 3326
4338out: 3327out:
4339 drm_gem_object_unreference(obj); 3328 drm_gem_object_unreference(&obj->base);
4340unlock: 3329unlock:
4341 mutex_unlock(&dev->struct_mutex); 3330 mutex_unlock(&dev->struct_mutex);
4342 return ret; 3331 return ret;
@@ -4344,48 +3333,50 @@ unlock:
4344 3333
4345int 3334int
4346i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3335i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4347 struct drm_file *file_priv) 3336 struct drm_file *file)
4348{ 3337{
4349 struct drm_i915_gem_busy *args = data; 3338 struct drm_i915_gem_busy *args = data;
4350 struct drm_gem_object *obj; 3339 struct drm_i915_gem_object *obj;
4351 struct drm_i915_gem_object *obj_priv;
4352 int ret; 3340 int ret;
4353 3341
4354 ret = i915_mutex_lock_interruptible(dev); 3342 ret = i915_mutex_lock_interruptible(dev);
4355 if (ret) 3343 if (ret)
4356 return ret; 3344 return ret;
4357 3345
4358 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3346 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4359 if (obj == NULL) { 3347 if (obj == NULL) {
4360 ret = -ENOENT; 3348 ret = -ENOENT;
4361 goto unlock; 3349 goto unlock;
4362 } 3350 }
4363 obj_priv = to_intel_bo(obj);
4364 3351
4365 /* Count all active objects as busy, even if they are currently not used 3352 /* Count all active objects as busy, even if they are currently not used
4366 * by the gpu. Users of this interface expect objects to eventually 3353 * by the gpu. Users of this interface expect objects to eventually
4367 * become non-busy without any further actions, therefore emit any 3354 * become non-busy without any further actions, therefore emit any
4368 * necessary flushes here. 3355 * necessary flushes here.
4369 */ 3356 */
4370 args->busy = obj_priv->active; 3357 args->busy = obj->active;
4371 if (args->busy) { 3358 if (args->busy) {
4372 /* Unconditionally flush objects, even when the gpu still uses this 3359 /* Unconditionally flush objects, even when the gpu still uses this
4373 * object. Userspace calling this function indicates that it wants to 3360 * object. Userspace calling this function indicates that it wants to
4374 * use this buffer rather sooner than later, so issuing the required 3361 * use this buffer rather sooner than later, so issuing the required
4375 * flush earlier is beneficial. 3362 * flush earlier is beneficial.
4376 */ 3363 */
4377 if (obj->write_domain & I915_GEM_GPU_DOMAINS) { 3364 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
4378 i915_gem_flush_ring(dev, file_priv, 3365 i915_gem_flush_ring(dev, obj->ring,
4379 obj_priv->ring, 3366 0, obj->base.write_domain);
4380 0, obj->write_domain); 3367 } else if (obj->ring->outstanding_lazy_request ==
4381 } else if (obj_priv->ring->outstanding_lazy_request) { 3368 obj->last_rendering_seqno) {
3369 struct drm_i915_gem_request *request;
3370
4382 /* This ring is not being cleared by active usage, 3371 /* This ring is not being cleared by active usage,
4383 * so emit a request to do so. 3372 * so emit a request to do so.
4384 */ 3373 */
4385 u32 seqno = i915_add_request(dev, 3374 request = kzalloc(sizeof(*request), GFP_KERNEL);
4386 NULL, NULL, 3375 if (request)
4387 obj_priv->ring); 3376 ret = i915_add_request(dev,
4388 if (seqno == 0) 3377 NULL, request,
3378 obj->ring);
3379 else
4389 ret = -ENOMEM; 3380 ret = -ENOMEM;
4390 } 3381 }
4391 3382
@@ -4394,12 +3385,12 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4394 * are actually unmasked, and our working set ends up being 3385 * are actually unmasked, and our working set ends up being
4395 * larger than required. 3386 * larger than required.
4396 */ 3387 */
4397 i915_gem_retire_requests_ring(dev, obj_priv->ring); 3388 i915_gem_retire_requests_ring(dev, obj->ring);
4398 3389
4399 args->busy = obj_priv->active; 3390 args->busy = obj->active;
4400 } 3391 }
4401 3392
4402 drm_gem_object_unreference(obj); 3393 drm_gem_object_unreference(&obj->base);
4403unlock: 3394unlock:
4404 mutex_unlock(&dev->struct_mutex); 3395 mutex_unlock(&dev->struct_mutex);
4405 return ret; 3396 return ret;
@@ -4417,8 +3408,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4417 struct drm_file *file_priv) 3408 struct drm_file *file_priv)
4418{ 3409{
4419 struct drm_i915_gem_madvise *args = data; 3410 struct drm_i915_gem_madvise *args = data;
4420 struct drm_gem_object *obj; 3411 struct drm_i915_gem_object *obj;
4421 struct drm_i915_gem_object *obj_priv;
4422 int ret; 3412 int ret;
4423 3413
4424 switch (args->madv) { 3414 switch (args->madv) {
@@ -4433,37 +3423,36 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4433 if (ret) 3423 if (ret)
4434 return ret; 3424 return ret;
4435 3425
4436 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3426 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4437 if (obj == NULL) { 3427 if (obj == NULL) {
4438 ret = -ENOENT; 3428 ret = -ENOENT;
4439 goto unlock; 3429 goto unlock;
4440 } 3430 }
4441 obj_priv = to_intel_bo(obj);
4442 3431
4443 if (obj_priv->pin_count) { 3432 if (obj->pin_count) {
4444 ret = -EINVAL; 3433 ret = -EINVAL;
4445 goto out; 3434 goto out;
4446 } 3435 }
4447 3436
4448 if (obj_priv->madv != __I915_MADV_PURGED) 3437 if (obj->madv != __I915_MADV_PURGED)
4449 obj_priv->madv = args->madv; 3438 obj->madv = args->madv;
4450 3439
4451 /* if the object is no longer bound, discard its backing storage */ 3440 /* if the object is no longer bound, discard its backing storage */
4452 if (i915_gem_object_is_purgeable(obj_priv) && 3441 if (i915_gem_object_is_purgeable(obj) &&
4453 obj_priv->gtt_space == NULL) 3442 obj->gtt_space == NULL)
4454 i915_gem_object_truncate(obj); 3443 i915_gem_object_truncate(obj);
4455 3444
4456 args->retained = obj_priv->madv != __I915_MADV_PURGED; 3445 args->retained = obj->madv != __I915_MADV_PURGED;
4457 3446
4458out: 3447out:
4459 drm_gem_object_unreference(obj); 3448 drm_gem_object_unreference(&obj->base);
4460unlock: 3449unlock:
4461 mutex_unlock(&dev->struct_mutex); 3450 mutex_unlock(&dev->struct_mutex);
4462 return ret; 3451 return ret;
4463} 3452}
4464 3453
4465struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 3454struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4466 size_t size) 3455 size_t size)
4467{ 3456{
4468 struct drm_i915_private *dev_priv = dev->dev_private; 3457 struct drm_i915_private *dev_priv = dev->dev_private;
4469 struct drm_i915_gem_object *obj; 3458 struct drm_i915_gem_object *obj;
@@ -4486,11 +3475,15 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4486 obj->base.driver_private = NULL; 3475 obj->base.driver_private = NULL;
4487 obj->fence_reg = I915_FENCE_REG_NONE; 3476 obj->fence_reg = I915_FENCE_REG_NONE;
4488 INIT_LIST_HEAD(&obj->mm_list); 3477 INIT_LIST_HEAD(&obj->mm_list);
3478 INIT_LIST_HEAD(&obj->gtt_list);
4489 INIT_LIST_HEAD(&obj->ring_list); 3479 INIT_LIST_HEAD(&obj->ring_list);
3480 INIT_LIST_HEAD(&obj->exec_list);
4490 INIT_LIST_HEAD(&obj->gpu_write_list); 3481 INIT_LIST_HEAD(&obj->gpu_write_list);
4491 obj->madv = I915_MADV_WILLNEED; 3482 obj->madv = I915_MADV_WILLNEED;
3483 /* Avoid an unnecessary call to unbind on the first bind. */
3484 obj->map_and_fenceable = true;
4492 3485
4493 return &obj->base; 3486 return obj;
4494} 3487}
4495 3488
4496int i915_gem_init_object(struct drm_gem_object *obj) 3489int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4500,42 +3493,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
4500 return 0; 3493 return 0;
4501} 3494}
4502 3495
4503static void i915_gem_free_object_tail(struct drm_gem_object *obj) 3496static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
4504{ 3497{
4505 struct drm_device *dev = obj->dev; 3498 struct drm_device *dev = obj->base.dev;
4506 drm_i915_private_t *dev_priv = dev->dev_private; 3499 drm_i915_private_t *dev_priv = dev->dev_private;
4507 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4508 int ret; 3500 int ret;
4509 3501
4510 ret = i915_gem_object_unbind(obj); 3502 ret = i915_gem_object_unbind(obj);
4511 if (ret == -ERESTARTSYS) { 3503 if (ret == -ERESTARTSYS) {
4512 list_move(&obj_priv->mm_list, 3504 list_move(&obj->mm_list,
4513 &dev_priv->mm.deferred_free_list); 3505 &dev_priv->mm.deferred_free_list);
4514 return; 3506 return;
4515 } 3507 }
4516 3508
4517 if (obj_priv->mmap_offset) 3509 if (obj->base.map_list.map)
4518 i915_gem_free_mmap_offset(obj); 3510 i915_gem_free_mmap_offset(obj);
4519 3511
4520 drm_gem_object_release(obj); 3512 drm_gem_object_release(&obj->base);
4521 i915_gem_info_remove_obj(dev_priv, obj->size); 3513 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4522 3514
4523 kfree(obj_priv->page_cpu_valid); 3515 kfree(obj->page_cpu_valid);
4524 kfree(obj_priv->bit_17); 3516 kfree(obj->bit_17);
4525 kfree(obj_priv); 3517 kfree(obj);
4526} 3518}
4527 3519
4528void i915_gem_free_object(struct drm_gem_object *obj) 3520void i915_gem_free_object(struct drm_gem_object *gem_obj)
4529{ 3521{
4530 struct drm_device *dev = obj->dev; 3522 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4531 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3523 struct drm_device *dev = obj->base.dev;
4532 3524
4533 trace_i915_gem_object_destroy(obj); 3525 trace_i915_gem_object_destroy(obj);
4534 3526
4535 while (obj_priv->pin_count > 0) 3527 while (obj->pin_count > 0)
4536 i915_gem_object_unpin(obj); 3528 i915_gem_object_unpin(obj);
4537 3529
4538 if (obj_priv->phys_obj) 3530 if (obj->phys_obj)
4539 i915_gem_detach_phys_object(dev, obj); 3531 i915_gem_detach_phys_object(dev, obj);
4540 3532
4541 i915_gem_free_object_tail(obj); 3533 i915_gem_free_object_tail(obj);
@@ -4562,13 +3554,15 @@ i915_gem_idle(struct drm_device *dev)
4562 3554
4563 /* Under UMS, be paranoid and evict. */ 3555 /* Under UMS, be paranoid and evict. */
4564 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 3556 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4565 ret = i915_gem_evict_inactive(dev); 3557 ret = i915_gem_evict_inactive(dev, false);
4566 if (ret) { 3558 if (ret) {
4567 mutex_unlock(&dev->struct_mutex); 3559 mutex_unlock(&dev->struct_mutex);
4568 return ret; 3560 return ret;
4569 } 3561 }
4570 } 3562 }
4571 3563
3564 i915_gem_reset_fences(dev);
3565
4572 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3566 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4573 * We need to replace this with a semaphore, or something. 3567 * We need to replace this with a semaphore, or something.
4574 * And not confound mm.suspended! 3568 * And not confound mm.suspended!
@@ -4587,82 +3581,15 @@ i915_gem_idle(struct drm_device *dev)
4587 return 0; 3581 return 0;
4588} 3582}
4589 3583
4590/*
4591 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4592 * over cache flushing.
4593 */
4594static int
4595i915_gem_init_pipe_control(struct drm_device *dev)
4596{
4597 drm_i915_private_t *dev_priv = dev->dev_private;
4598 struct drm_gem_object *obj;
4599 struct drm_i915_gem_object *obj_priv;
4600 int ret;
4601
4602 obj = i915_gem_alloc_object(dev, 4096);
4603 if (obj == NULL) {
4604 DRM_ERROR("Failed to allocate seqno page\n");
4605 ret = -ENOMEM;
4606 goto err;
4607 }
4608 obj_priv = to_intel_bo(obj);
4609 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4610
4611 ret = i915_gem_object_pin(obj, 4096);
4612 if (ret)
4613 goto err_unref;
4614
4615 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4616 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4617 if (dev_priv->seqno_page == NULL)
4618 goto err_unpin;
4619
4620 dev_priv->seqno_obj = obj;
4621 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4622
4623 return 0;
4624
4625err_unpin:
4626 i915_gem_object_unpin(obj);
4627err_unref:
4628 drm_gem_object_unreference(obj);
4629err:
4630 return ret;
4631}
4632
4633
4634static void
4635i915_gem_cleanup_pipe_control(struct drm_device *dev)
4636{
4637 drm_i915_private_t *dev_priv = dev->dev_private;
4638 struct drm_gem_object *obj;
4639 struct drm_i915_gem_object *obj_priv;
4640
4641 obj = dev_priv->seqno_obj;
4642 obj_priv = to_intel_bo(obj);
4643 kunmap(obj_priv->pages[0]);
4644 i915_gem_object_unpin(obj);
4645 drm_gem_object_unreference(obj);
4646 dev_priv->seqno_obj = NULL;
4647
4648 dev_priv->seqno_page = NULL;
4649}
4650
4651int 3584int
4652i915_gem_init_ringbuffer(struct drm_device *dev) 3585i915_gem_init_ringbuffer(struct drm_device *dev)
4653{ 3586{
4654 drm_i915_private_t *dev_priv = dev->dev_private; 3587 drm_i915_private_t *dev_priv = dev->dev_private;
4655 int ret; 3588 int ret;
4656 3589
4657 if (HAS_PIPE_CONTROL(dev)) {
4658 ret = i915_gem_init_pipe_control(dev);
4659 if (ret)
4660 return ret;
4661 }
4662
4663 ret = intel_init_render_ring_buffer(dev); 3590 ret = intel_init_render_ring_buffer(dev);
4664 if (ret) 3591 if (ret)
4665 goto cleanup_pipe_control; 3592 return ret;
4666 3593
4667 if (HAS_BSD(dev)) { 3594 if (HAS_BSD(dev)) {
4668 ret = intel_init_bsd_ring_buffer(dev); 3595 ret = intel_init_bsd_ring_buffer(dev);
@@ -4681,12 +3608,9 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
4681 return 0; 3608 return 0;
4682 3609
4683cleanup_bsd_ring: 3610cleanup_bsd_ring:
4684 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3611 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4685cleanup_render_ring: 3612cleanup_render_ring:
4686 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3613 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4687cleanup_pipe_control:
4688 if (HAS_PIPE_CONTROL(dev))
4689 i915_gem_cleanup_pipe_control(dev);
4690 return ret; 3614 return ret;
4691} 3615}
4692 3616
@@ -4694,12 +3618,10 @@ void
4694i915_gem_cleanup_ringbuffer(struct drm_device *dev) 3618i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4695{ 3619{
4696 drm_i915_private_t *dev_priv = dev->dev_private; 3620 drm_i915_private_t *dev_priv = dev->dev_private;
3621 int i;
4697 3622
4698 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3623 for (i = 0; i < I915_NUM_RINGS; i++)
4699 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3624 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
4700 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4701 if (HAS_PIPE_CONTROL(dev))
4702 i915_gem_cleanup_pipe_control(dev);
4703} 3625}
4704 3626
4705int 3627int
@@ -4707,7 +3629,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4707 struct drm_file *file_priv) 3629 struct drm_file *file_priv)
4708{ 3630{
4709 drm_i915_private_t *dev_priv = dev->dev_private; 3631 drm_i915_private_t *dev_priv = dev->dev_private;
4710 int ret; 3632 int ret, i;
4711 3633
4712 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3634 if (drm_core_check_feature(dev, DRIVER_MODESET))
4713 return 0; 3635 return 0;
@@ -4727,14 +3649,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4727 } 3649 }
4728 3650
4729 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 3651 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4730 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4731 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4732 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4733 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 3652 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4734 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 3653 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4735 BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); 3654 for (i = 0; i < I915_NUM_RINGS; i++) {
4736 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list)); 3655 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
4737 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list)); 3656 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3657 }
4738 mutex_unlock(&dev->struct_mutex); 3658 mutex_unlock(&dev->struct_mutex);
4739 3659
4740 ret = drm_irq_install(dev); 3660 ret = drm_irq_install(dev);
@@ -4796,17 +3716,14 @@ i915_gem_load(struct drm_device *dev)
4796 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 3716 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4797 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3717 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4798 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 3718 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4799 init_ring_lists(&dev_priv->render_ring); 3719 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
4800 init_ring_lists(&dev_priv->bsd_ring); 3720 for (i = 0; i < I915_NUM_RINGS; i++)
4801 init_ring_lists(&dev_priv->blt_ring); 3721 init_ring_lists(&dev_priv->ring[i]);
4802 for (i = 0; i < 16; i++) 3722 for (i = 0; i < 16; i++)
4803 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 3723 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4804 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 3724 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4805 i915_gem_retire_work_handler); 3725 i915_gem_retire_work_handler);
4806 init_completion(&dev_priv->error_completion); 3726 init_completion(&dev_priv->error_completion);
4807 spin_lock(&shrink_list_lock);
4808 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4809 spin_unlock(&shrink_list_lock);
4810 3727
4811 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 3728 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4812 if (IS_GEN3(dev)) { 3729 if (IS_GEN3(dev)) {
@@ -4818,6 +3735,8 @@ i915_gem_load(struct drm_device *dev)
4818 } 3735 }
4819 } 3736 }
4820 3737
3738 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3739
4821 /* Old X drivers will take 0-2 for front, back, depth buffers */ 3740 /* Old X drivers will take 0-2 for front, back, depth buffers */
4822 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 3741 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4823 dev_priv->fence_reg_start = 3; 3742 dev_priv->fence_reg_start = 3;
@@ -4849,6 +3768,10 @@ i915_gem_load(struct drm_device *dev)
4849 } 3768 }
4850 i915_gem_detect_bit_6_swizzle(dev); 3769 i915_gem_detect_bit_6_swizzle(dev);
4851 init_waitqueue_head(&dev_priv->pending_flip_queue); 3770 init_waitqueue_head(&dev_priv->pending_flip_queue);
3771
3772 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3773 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3774 register_shrinker(&dev_priv->mm.inactive_shrinker);
4852} 3775}
4853 3776
4854/* 3777/*
@@ -4918,47 +3841,47 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
4918} 3841}
4919 3842
4920void i915_gem_detach_phys_object(struct drm_device *dev, 3843void i915_gem_detach_phys_object(struct drm_device *dev,
4921 struct drm_gem_object *obj) 3844 struct drm_i915_gem_object *obj)
4922{ 3845{
4923 struct drm_i915_gem_object *obj_priv; 3846 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3847 char *vaddr;
4924 int i; 3848 int i;
4925 int ret;
4926 int page_count; 3849 int page_count;
4927 3850
4928 obj_priv = to_intel_bo(obj); 3851 if (!obj->phys_obj)
4929 if (!obj_priv->phys_obj)
4930 return; 3852 return;
3853 vaddr = obj->phys_obj->handle->vaddr;
4931 3854
4932 ret = i915_gem_object_get_pages(obj, 0); 3855 page_count = obj->base.size / PAGE_SIZE;
4933 if (ret)
4934 goto out;
4935
4936 page_count = obj->size / PAGE_SIZE;
4937
4938 for (i = 0; i < page_count; i++) { 3856 for (i = 0; i < page_count; i++) {
4939 char *dst = kmap_atomic(obj_priv->pages[i]); 3857 struct page *page = read_cache_page_gfp(mapping, i,
4940 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3858 GFP_HIGHUSER | __GFP_RECLAIMABLE);
4941 3859 if (!IS_ERR(page)) {
4942 memcpy(dst, src, PAGE_SIZE); 3860 char *dst = kmap_atomic(page);
4943 kunmap_atomic(dst); 3861 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3862 kunmap_atomic(dst);
3863
3864 drm_clflush_pages(&page, 1);
3865
3866 set_page_dirty(page);
3867 mark_page_accessed(page);
3868 page_cache_release(page);
3869 }
4944 } 3870 }
4945 drm_clflush_pages(obj_priv->pages, page_count); 3871 intel_gtt_chipset_flush();
4946 drm_agp_chipset_flush(dev);
4947 3872
4948 i915_gem_object_put_pages(obj); 3873 obj->phys_obj->cur_obj = NULL;
4949out: 3874 obj->phys_obj = NULL;
4950 obj_priv->phys_obj->cur_obj = NULL;
4951 obj_priv->phys_obj = NULL;
4952} 3875}
4953 3876
4954int 3877int
4955i915_gem_attach_phys_object(struct drm_device *dev, 3878i915_gem_attach_phys_object(struct drm_device *dev,
4956 struct drm_gem_object *obj, 3879 struct drm_i915_gem_object *obj,
4957 int id, 3880 int id,
4958 int align) 3881 int align)
4959{ 3882{
3883 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4960 drm_i915_private_t *dev_priv = dev->dev_private; 3884 drm_i915_private_t *dev_priv = dev->dev_private;
4961 struct drm_i915_gem_object *obj_priv;
4962 int ret = 0; 3885 int ret = 0;
4963 int page_count; 3886 int page_count;
4964 int i; 3887 int i;
@@ -4966,10 +3889,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4966 if (id > I915_MAX_PHYS_OBJECT) 3889 if (id > I915_MAX_PHYS_OBJECT)
4967 return -EINVAL; 3890 return -EINVAL;
4968 3891
4969 obj_priv = to_intel_bo(obj); 3892 if (obj->phys_obj) {
4970 3893 if (obj->phys_obj->id == id)
4971 if (obj_priv->phys_obj) {
4972 if (obj_priv->phys_obj->id == id)
4973 return 0; 3894 return 0;
4974 i915_gem_detach_phys_object(dev, obj); 3895 i915_gem_detach_phys_object(dev, obj);
4975 } 3896 }
@@ -4977,51 +3898,50 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4977 /* create a new object */ 3898 /* create a new object */
4978 if (!dev_priv->mm.phys_objs[id - 1]) { 3899 if (!dev_priv->mm.phys_objs[id - 1]) {
4979 ret = i915_gem_init_phys_object(dev, id, 3900 ret = i915_gem_init_phys_object(dev, id,
4980 obj->size, align); 3901 obj->base.size, align);
4981 if (ret) { 3902 if (ret) {
4982 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 3903 DRM_ERROR("failed to init phys object %d size: %zu\n",
4983 goto out; 3904 id, obj->base.size);
3905 return ret;
4984 } 3906 }
4985 } 3907 }
4986 3908
4987 /* bind to the object */ 3909 /* bind to the object */
4988 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3910 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4989 obj_priv->phys_obj->cur_obj = obj; 3911 obj->phys_obj->cur_obj = obj;
4990 3912
4991 ret = i915_gem_object_get_pages(obj, 0); 3913 page_count = obj->base.size / PAGE_SIZE;
4992 if (ret) {
4993 DRM_ERROR("failed to get page list\n");
4994 goto out;
4995 }
4996
4997 page_count = obj->size / PAGE_SIZE;
4998 3914
4999 for (i = 0; i < page_count; i++) { 3915 for (i = 0; i < page_count; i++) {
5000 char *src = kmap_atomic(obj_priv->pages[i]); 3916 struct page *page;
5001 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3917 char *dst, *src;
5002 3918
3919 page = read_cache_page_gfp(mapping, i,
3920 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3921 if (IS_ERR(page))
3922 return PTR_ERR(page);
3923
3924 src = kmap_atomic(page);
3925 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5003 memcpy(dst, src, PAGE_SIZE); 3926 memcpy(dst, src, PAGE_SIZE);
5004 kunmap_atomic(src); 3927 kunmap_atomic(src);
5005 }
5006 3928
5007 i915_gem_object_put_pages(obj); 3929 mark_page_accessed(page);
3930 page_cache_release(page);
3931 }
5008 3932
5009 return 0; 3933 return 0;
5010out:
5011 return ret;
5012} 3934}
5013 3935
5014static int 3936static int
5015i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 3937i915_gem_phys_pwrite(struct drm_device *dev,
3938 struct drm_i915_gem_object *obj,
5016 struct drm_i915_gem_pwrite *args, 3939 struct drm_i915_gem_pwrite *args,
5017 struct drm_file *file_priv) 3940 struct drm_file *file_priv)
5018{ 3941{
5019 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3942 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
5020 void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
5021 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr; 3943 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5022 3944
5023 DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
5024
5025 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 3945 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5026 unsigned long unwritten; 3946 unsigned long unwritten;
5027 3947
@@ -5036,7 +3956,7 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5036 return -EFAULT; 3956 return -EFAULT;
5037 } 3957 }
5038 3958
5039 drm_agp_chipset_flush(dev); 3959 intel_gtt_chipset_flush();
5040 return 0; 3960 return 0;
5041} 3961}
5042 3962
@@ -5074,144 +3994,68 @@ i915_gpu_is_active(struct drm_device *dev)
5074} 3994}
5075 3995
5076static int 3996static int
5077i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) 3997i915_gem_inactive_shrink(struct shrinker *shrinker,
5078{ 3998 int nr_to_scan,
5079 drm_i915_private_t *dev_priv, *next_dev; 3999 gfp_t gfp_mask)
5080 struct drm_i915_gem_object *obj_priv, *next_obj; 4000{
5081 int cnt = 0; 4001 struct drm_i915_private *dev_priv =
5082 int would_deadlock = 1; 4002 container_of(shrinker,
4003 struct drm_i915_private,
4004 mm.inactive_shrinker);
4005 struct drm_device *dev = dev_priv->dev;
4006 struct drm_i915_gem_object *obj, *next;
4007 int cnt;
4008
4009 if (!mutex_trylock(&dev->struct_mutex))
4010 return 0;
5083 4011
5084 /* "fast-path" to count number of available objects */ 4012 /* "fast-path" to count number of available objects */
5085 if (nr_to_scan == 0) { 4013 if (nr_to_scan == 0) {
5086 spin_lock(&shrink_list_lock); 4014 cnt = 0;
5087 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4015 list_for_each_entry(obj,
5088 struct drm_device *dev = dev_priv->dev; 4016 &dev_priv->mm.inactive_list,
5089 4017 mm_list)
5090 if (mutex_trylock(&dev->struct_mutex)) { 4018 cnt++;
5091 list_for_each_entry(obj_priv, 4019 mutex_unlock(&dev->struct_mutex);
5092 &dev_priv->mm.inactive_list, 4020 return cnt / 100 * sysctl_vfs_cache_pressure;
5093 mm_list)
5094 cnt++;
5095 mutex_unlock(&dev->struct_mutex);
5096 }
5097 }
5098 spin_unlock(&shrink_list_lock);
5099
5100 return (cnt / 100) * sysctl_vfs_cache_pressure;
5101 } 4021 }
5102 4022
5103 spin_lock(&shrink_list_lock);
5104
5105rescan: 4023rescan:
5106 /* first scan for clean buffers */ 4024 /* first scan for clean buffers */
5107 list_for_each_entry_safe(dev_priv, next_dev, 4025 i915_gem_retire_requests(dev);
5108 &shrink_list, mm.shrink_list) {
5109 struct drm_device *dev = dev_priv->dev;
5110
5111 if (! mutex_trylock(&dev->struct_mutex))
5112 continue;
5113
5114 spin_unlock(&shrink_list_lock);
5115 i915_gem_retire_requests(dev);
5116 4026
5117 list_for_each_entry_safe(obj_priv, next_obj, 4027 list_for_each_entry_safe(obj, next,
5118 &dev_priv->mm.inactive_list, 4028 &dev_priv->mm.inactive_list,
5119 mm_list) { 4029 mm_list) {
5120 if (i915_gem_object_is_purgeable(obj_priv)) { 4030 if (i915_gem_object_is_purgeable(obj)) {
5121 i915_gem_object_unbind(&obj_priv->base); 4031 if (i915_gem_object_unbind(obj) == 0 &&
5122 if (--nr_to_scan <= 0) 4032 --nr_to_scan == 0)
5123 break; 4033 break;
5124 }
5125 } 4034 }
5126
5127 spin_lock(&shrink_list_lock);
5128 mutex_unlock(&dev->struct_mutex);
5129
5130 would_deadlock = 0;
5131
5132 if (nr_to_scan <= 0)
5133 break;
5134 } 4035 }
5135 4036
5136 /* second pass, evict/count anything still on the inactive list */ 4037 /* second pass, evict/count anything still on the inactive list */
5137 list_for_each_entry_safe(dev_priv, next_dev, 4038 cnt = 0;
5138 &shrink_list, mm.shrink_list) { 4039 list_for_each_entry_safe(obj, next,
5139 struct drm_device *dev = dev_priv->dev; 4040 &dev_priv->mm.inactive_list,
5140 4041 mm_list) {
5141 if (! mutex_trylock(&dev->struct_mutex)) 4042 if (nr_to_scan &&
5142 continue; 4043 i915_gem_object_unbind(obj) == 0)
5143 4044 nr_to_scan--;
5144 spin_unlock(&shrink_list_lock); 4045 else
5145 4046 cnt++;
5146 list_for_each_entry_safe(obj_priv, next_obj,
5147 &dev_priv->mm.inactive_list,
5148 mm_list) {
5149 if (nr_to_scan > 0) {
5150 i915_gem_object_unbind(&obj_priv->base);
5151 nr_to_scan--;
5152 } else
5153 cnt++;
5154 }
5155
5156 spin_lock(&shrink_list_lock);
5157 mutex_unlock(&dev->struct_mutex);
5158
5159 would_deadlock = 0;
5160 } 4047 }
5161 4048
5162 if (nr_to_scan) { 4049 if (nr_to_scan && i915_gpu_is_active(dev)) {
5163 int active = 0;
5164
5165 /* 4050 /*
5166 * We are desperate for pages, so as a last resort, wait 4051 * We are desperate for pages, so as a last resort, wait
5167 * for the GPU to finish and discard whatever we can. 4052 * for the GPU to finish and discard whatever we can.
5168 * This has a dramatic impact to reduce the number of 4053 * This has a dramatic impact to reduce the number of
5169 * OOM-killer events whilst running the GPU aggressively. 4054 * OOM-killer events whilst running the GPU aggressively.
5170 */ 4055 */
5171 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4056 if (i915_gpu_idle(dev) == 0)
5172 struct drm_device *dev = dev_priv->dev;
5173
5174 if (!mutex_trylock(&dev->struct_mutex))
5175 continue;
5176
5177 spin_unlock(&shrink_list_lock);
5178
5179 if (i915_gpu_is_active(dev)) {
5180 i915_gpu_idle(dev);
5181 active++;
5182 }
5183
5184 spin_lock(&shrink_list_lock);
5185 mutex_unlock(&dev->struct_mutex);
5186 }
5187
5188 if (active)
5189 goto rescan; 4057 goto rescan;
5190 } 4058 }
5191 4059 mutex_unlock(&dev->struct_mutex);
5192 spin_unlock(&shrink_list_lock); 4060 return cnt / 100 * sysctl_vfs_cache_pressure;
5193
5194 if (would_deadlock)
5195 return -1;
5196 else if (cnt > 0)
5197 return (cnt / 100) * sysctl_vfs_cache_pressure;
5198 else
5199 return 0;
5200}
5201
5202static struct shrinker shrinker = {
5203 .shrink = i915_gem_shrink,
5204 .seeks = DEFAULT_SEEKS,
5205};
5206
5207__init void
5208i915_gem_shrinker_init(void)
5209{
5210 register_shrinker(&shrinker);
5211}
5212
5213__exit void
5214i915_gem_shrinker_exit(void)
5215{
5216 unregister_shrinker(&shrinker);
5217} 4061}
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index 48644b840a8d..29d014c48ca2 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -152,13 +152,12 @@ i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end,
152} 152}
153 153
154void 154void
155i915_gem_dump_object(struct drm_gem_object *obj, int len, 155i915_gem_dump_object(struct drm_i915_gem_object *obj, int len,
156 const char *where, uint32_t mark) 156 const char *where, uint32_t mark)
157{ 157{
158 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
159 int page; 158 int page;
160 159
161 DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset); 160 DRM_INFO("%s: object at offset %08x\n", where, obj->gtt_offset);
162 for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) { 161 for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) {
163 int page_len, chunk, chunk_len; 162 int page_len, chunk, chunk_len;
164 163
@@ -170,9 +169,9 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
170 chunk_len = page_len - chunk; 169 chunk_len = page_len - chunk;
171 if (chunk_len > 128) 170 if (chunk_len > 128)
172 chunk_len = 128; 171 chunk_len = 128;
173 i915_gem_dump_page(obj_priv->pages[page], 172 i915_gem_dump_page(obj->pages[page],
174 chunk, chunk + chunk_len, 173 chunk, chunk + chunk_len,
175 obj_priv->gtt_offset + 174 obj->gtt_offset +
176 page * PAGE_SIZE, 175 page * PAGE_SIZE,
177 mark); 176 mark);
178 } 177 }
@@ -182,21 +181,19 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len,
182 181
183#if WATCH_COHERENCY 182#if WATCH_COHERENCY
184void 183void
185i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) 184i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, int handle)
186{ 185{
187 struct drm_device *dev = obj->dev; 186 struct drm_device *dev = obj->base.dev;
188 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
189 int page; 187 int page;
190 uint32_t *gtt_mapping; 188 uint32_t *gtt_mapping;
191 uint32_t *backing_map = NULL; 189 uint32_t *backing_map = NULL;
192 int bad_count = 0; 190 int bad_count = 0;
193 191
194 DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n", 192 DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n",
195 __func__, obj, obj_priv->gtt_offset, handle, 193 __func__, obj, obj->gtt_offset, handle,
196 obj->size / 1024); 194 obj->size / 1024);
197 195
198 gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset, 196 gtt_mapping = ioremap(dev->agp->base + obj->gtt_offset, obj->base.size);
199 obj->size);
200 if (gtt_mapping == NULL) { 197 if (gtt_mapping == NULL) {
201 DRM_ERROR("failed to map GTT space\n"); 198 DRM_ERROR("failed to map GTT space\n");
202 return; 199 return;
@@ -205,7 +202,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
205 for (page = 0; page < obj->size / PAGE_SIZE; page++) { 202 for (page = 0; page < obj->size / PAGE_SIZE; page++) {
206 int i; 203 int i;
207 204
208 backing_map = kmap_atomic(obj_priv->pages[page], KM_USER0); 205 backing_map = kmap_atomic(obj->pages[page], KM_USER0);
209 206
210 if (backing_map == NULL) { 207 if (backing_map == NULL) {
211 DRM_ERROR("failed to map backing page\n"); 208 DRM_ERROR("failed to map backing page\n");
@@ -220,7 +217,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
220 if (cpuval != gttval) { 217 if (cpuval != gttval) {
221 DRM_INFO("incoherent CPU vs GPU at 0x%08x: " 218 DRM_INFO("incoherent CPU vs GPU at 0x%08x: "
222 "0x%08x vs 0x%08x\n", 219 "0x%08x vs 0x%08x\n",
223 (int)(obj_priv->gtt_offset + 220 (int)(obj->gtt_offset +
224 page * PAGE_SIZE + i * 4), 221 page * PAGE_SIZE + i * 4),
225 cpuval, gttval); 222 cpuval, gttval);
226 if (bad_count++ >= 8) { 223 if (bad_count++ >= 8) {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index d8ae7d1d0cc6..78b8cf90c922 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -32,28 +32,36 @@
32#include "i915_drm.h" 32#include "i915_drm.h"
33 33
34static bool 34static bool
35mark_free(struct drm_i915_gem_object *obj_priv, 35mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
36 struct list_head *unwind)
37{ 36{
38 list_add(&obj_priv->evict_list, unwind); 37 list_add(&obj->exec_list, unwind);
39 drm_gem_object_reference(&obj_priv->base); 38 drm_gem_object_reference(&obj->base);
40 return drm_mm_scan_add_block(obj_priv->gtt_space); 39 return drm_mm_scan_add_block(obj->gtt_space);
41} 40}
42 41
43int 42int
44i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment) 43i915_gem_evict_something(struct drm_device *dev, int min_size,
44 unsigned alignment, bool mappable)
45{ 45{
46 drm_i915_private_t *dev_priv = dev->dev_private; 46 drm_i915_private_t *dev_priv = dev->dev_private;
47 struct list_head eviction_list, unwind_list; 47 struct list_head eviction_list, unwind_list;
48 struct drm_i915_gem_object *obj_priv; 48 struct drm_i915_gem_object *obj;
49 int ret = 0; 49 int ret = 0;
50 50
51 i915_gem_retire_requests(dev); 51 i915_gem_retire_requests(dev);
52 52
53 /* Re-check for free space after retiring requests */ 53 /* Re-check for free space after retiring requests */
54 if (drm_mm_search_free(&dev_priv->mm.gtt_space, 54 if (mappable) {
55 min_size, alignment, 0)) 55 if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
56 return 0; 56 min_size, alignment, 0,
57 dev_priv->mm.gtt_mappable_end,
58 0))
59 return 0;
60 } else {
61 if (drm_mm_search_free(&dev_priv->mm.gtt_space,
62 min_size, alignment, 0))
63 return 0;
64 }
57 65
58 /* 66 /*
59 * The goal is to evict objects and amalgamate space in LRU order. 67 * The goal is to evict objects and amalgamate space in LRU order.
@@ -79,45 +87,50 @@ i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignmen
79 */ 87 */
80 88
81 INIT_LIST_HEAD(&unwind_list); 89 INIT_LIST_HEAD(&unwind_list);
82 drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment); 90 if (mappable)
91 drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size,
92 alignment, 0,
93 dev_priv->mm.gtt_mappable_end);
94 else
95 drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment);
83 96
84 /* First see if there is a large enough contiguous idle region... */ 97 /* First see if there is a large enough contiguous idle region... */
85 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { 98 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
86 if (mark_free(obj_priv, &unwind_list)) 99 if (mark_free(obj, &unwind_list))
87 goto found; 100 goto found;
88 } 101 }
89 102
90 /* Now merge in the soon-to-be-expired objects... */ 103 /* Now merge in the soon-to-be-expired objects... */
91 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 104 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
92 /* Does the object require an outstanding flush? */ 105 /* Does the object require an outstanding flush? */
93 if (obj_priv->base.write_domain || obj_priv->pin_count) 106 if (obj->base.write_domain || obj->pin_count)
94 continue; 107 continue;
95 108
96 if (mark_free(obj_priv, &unwind_list)) 109 if (mark_free(obj, &unwind_list))
97 goto found; 110 goto found;
98 } 111 }
99 112
100 /* Finally add anything with a pending flush (in order of retirement) */ 113 /* Finally add anything with a pending flush (in order of retirement) */
101 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { 114 list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
102 if (obj_priv->pin_count) 115 if (obj->pin_count)
103 continue; 116 continue;
104 117
105 if (mark_free(obj_priv, &unwind_list)) 118 if (mark_free(obj, &unwind_list))
106 goto found; 119 goto found;
107 } 120 }
108 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 121 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
109 if (! obj_priv->base.write_domain || obj_priv->pin_count) 122 if (! obj->base.write_domain || obj->pin_count)
110 continue; 123 continue;
111 124
112 if (mark_free(obj_priv, &unwind_list)) 125 if (mark_free(obj, &unwind_list))
113 goto found; 126 goto found;
114 } 127 }
115 128
116 /* Nothing found, clean up and bail out! */ 129 /* Nothing found, clean up and bail out! */
117 list_for_each_entry(obj_priv, &unwind_list, evict_list) { 130 list_for_each_entry(obj, &unwind_list, exec_list) {
118 ret = drm_mm_scan_remove_block(obj_priv->gtt_space); 131 ret = drm_mm_scan_remove_block(obj->gtt_space);
119 BUG_ON(ret); 132 BUG_ON(ret);
120 drm_gem_object_unreference(&obj_priv->base); 133 drm_gem_object_unreference(&obj->base);
121 } 134 }
122 135
123 /* We expect the caller to unpin, evict all and try again, or give up. 136 /* We expect the caller to unpin, evict all and try again, or give up.
@@ -131,33 +144,33 @@ found:
131 * temporary list. */ 144 * temporary list. */
132 INIT_LIST_HEAD(&eviction_list); 145 INIT_LIST_HEAD(&eviction_list);
133 while (!list_empty(&unwind_list)) { 146 while (!list_empty(&unwind_list)) {
134 obj_priv = list_first_entry(&unwind_list, 147 obj = list_first_entry(&unwind_list,
135 struct drm_i915_gem_object, 148 struct drm_i915_gem_object,
136 evict_list); 149 exec_list);
137 if (drm_mm_scan_remove_block(obj_priv->gtt_space)) { 150 if (drm_mm_scan_remove_block(obj->gtt_space)) {
138 list_move(&obj_priv->evict_list, &eviction_list); 151 list_move(&obj->exec_list, &eviction_list);
139 continue; 152 continue;
140 } 153 }
141 list_del(&obj_priv->evict_list); 154 list_del_init(&obj->exec_list);
142 drm_gem_object_unreference(&obj_priv->base); 155 drm_gem_object_unreference(&obj->base);
143 } 156 }
144 157
145 /* Unbinding will emit any required flushes */ 158 /* Unbinding will emit any required flushes */
146 while (!list_empty(&eviction_list)) { 159 while (!list_empty(&eviction_list)) {
147 obj_priv = list_first_entry(&eviction_list, 160 obj = list_first_entry(&eviction_list,
148 struct drm_i915_gem_object, 161 struct drm_i915_gem_object,
149 evict_list); 162 exec_list);
150 if (ret == 0) 163 if (ret == 0)
151 ret = i915_gem_object_unbind(&obj_priv->base); 164 ret = i915_gem_object_unbind(obj);
152 list_del(&obj_priv->evict_list); 165 list_del_init(&obj->exec_list);
153 drm_gem_object_unreference(&obj_priv->base); 166 drm_gem_object_unreference(&obj->base);
154 } 167 }
155 168
156 return ret; 169 return ret;
157} 170}
158 171
159int 172int
160i915_gem_evict_everything(struct drm_device *dev) 173i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
161{ 174{
162 drm_i915_private_t *dev_priv = dev->dev_private; 175 drm_i915_private_t *dev_priv = dev->dev_private;
163 int ret; 176 int ret;
@@ -176,36 +189,22 @@ i915_gem_evict_everything(struct drm_device *dev)
176 189
177 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 190 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
178 191
179 ret = i915_gem_evict_inactive(dev); 192 return i915_gem_evict_inactive(dev, purgeable_only);
180 if (ret)
181 return ret;
182
183 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
184 list_empty(&dev_priv->mm.flushing_list) &&
185 list_empty(&dev_priv->mm.active_list));
186 BUG_ON(!lists_empty);
187
188 return 0;
189} 193}
190 194
191/** Unbinds all inactive objects. */ 195/** Unbinds all inactive objects. */
192int 196int
193i915_gem_evict_inactive(struct drm_device *dev) 197i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
194{ 198{
195 drm_i915_private_t *dev_priv = dev->dev_private; 199 drm_i915_private_t *dev_priv = dev->dev_private;
196 200 struct drm_i915_gem_object *obj, *next;
197 while (!list_empty(&dev_priv->mm.inactive_list)) { 201
198 struct drm_gem_object *obj; 202 list_for_each_entry_safe(obj, next,
199 int ret; 203 &dev_priv->mm.inactive_list, mm_list) {
200 204 if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
201 obj = &list_first_entry(&dev_priv->mm.inactive_list, 205 int ret = i915_gem_object_unbind(obj);
202 struct drm_i915_gem_object, 206 if (ret)
203 mm_list)->base; 207 return ret;
204
205 ret = i915_gem_object_unbind(obj);
206 if (ret != 0) {
207 DRM_ERROR("Error unbinding object: %d\n", ret);
208 return ret;
209 } 208 }
210 } 209 }
211 210
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
new file mode 100644
index 000000000000..61129e6759eb
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -0,0 +1,1343 @@
1/*
2 * Copyright © 2008,2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 *
27 */
28
29#include "drmP.h"
30#include "drm.h"
31#include "i915_drm.h"
32#include "i915_drv.h"
33#include "i915_trace.h"
34#include "intel_drv.h"
35
36struct change_domains {
37 uint32_t invalidate_domains;
38 uint32_t flush_domains;
39 uint32_t flush_rings;
40};
41
42/*
43 * Set the next domain for the specified object. This
44 * may not actually perform the necessary flushing/invaliding though,
45 * as that may want to be batched with other set_domain operations
46 *
47 * This is (we hope) the only really tricky part of gem. The goal
48 * is fairly simple -- track which caches hold bits of the object
49 * and make sure they remain coherent. A few concrete examples may
50 * help to explain how it works. For shorthand, we use the notation
51 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
52 * a pair of read and write domain masks.
53 *
54 * Case 1: the batch buffer
55 *
56 * 1. Allocated
57 * 2. Written by CPU
58 * 3. Mapped to GTT
59 * 4. Read by GPU
60 * 5. Unmapped from GTT
61 * 6. Freed
62 *
63 * Let's take these a step at a time
64 *
65 * 1. Allocated
66 * Pages allocated from the kernel may still have
67 * cache contents, so we set them to (CPU, CPU) always.
68 * 2. Written by CPU (using pwrite)
69 * The pwrite function calls set_domain (CPU, CPU) and
70 * this function does nothing (as nothing changes)
71 * 3. Mapped by GTT
72 * This function asserts that the object is not
73 * currently in any GPU-based read or write domains
74 * 4. Read by GPU
75 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
76 * As write_domain is zero, this function adds in the
77 * current read domains (CPU+COMMAND, 0).
78 * flush_domains is set to CPU.
79 * invalidate_domains is set to COMMAND
80 * clflush is run to get data out of the CPU caches
81 * then i915_dev_set_domain calls i915_gem_flush to
82 * emit an MI_FLUSH and drm_agp_chipset_flush
83 * 5. Unmapped from GTT
84 * i915_gem_object_unbind calls set_domain (CPU, CPU)
85 * flush_domains and invalidate_domains end up both zero
86 * so no flushing/invalidating happens
87 * 6. Freed
88 * yay, done
89 *
90 * Case 2: The shared render buffer
91 *
92 * 1. Allocated
93 * 2. Mapped to GTT
94 * 3. Read/written by GPU
95 * 4. set_domain to (CPU,CPU)
96 * 5. Read/written by CPU
97 * 6. Read/written by GPU
98 *
99 * 1. Allocated
100 * Same as last example, (CPU, CPU)
101 * 2. Mapped to GTT
102 * Nothing changes (assertions find that it is not in the GPU)
103 * 3. Read/written by GPU
104 * execbuffer calls set_domain (RENDER, RENDER)
105 * flush_domains gets CPU
106 * invalidate_domains gets GPU
107 * clflush (obj)
108 * MI_FLUSH and drm_agp_chipset_flush
109 * 4. set_domain (CPU, CPU)
110 * flush_domains gets GPU
111 * invalidate_domains gets CPU
112 * wait_rendering (obj) to make sure all drawing is complete.
113 * This will include an MI_FLUSH to get the data from GPU
114 * to memory
115 * clflush (obj) to invalidate the CPU cache
116 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
117 * 5. Read/written by CPU
118 * cache lines are loaded and dirtied
119 * 6. Read written by GPU
120 * Same as last GPU access
121 *
122 * Case 3: The constant buffer
123 *
124 * 1. Allocated
125 * 2. Written by CPU
126 * 3. Read by GPU
127 * 4. Updated (written) by CPU again
128 * 5. Read by GPU
129 *
130 * 1. Allocated
131 * (CPU, CPU)
132 * 2. Written by CPU
133 * (CPU, CPU)
134 * 3. Read by GPU
135 * (CPU+RENDER, 0)
136 * flush_domains = CPU
137 * invalidate_domains = RENDER
138 * clflush (obj)
139 * MI_FLUSH
140 * drm_agp_chipset_flush
141 * 4. Updated (written) by CPU again
142 * (CPU, CPU)
143 * flush_domains = 0 (no previous write domain)
144 * invalidate_domains = 0 (no new read domains)
145 * 5. Read by GPU
146 * (CPU+RENDER, 0)
147 * flush_domains = CPU
148 * invalidate_domains = RENDER
149 * clflush (obj)
150 * MI_FLUSH
151 * drm_agp_chipset_flush
152 */
153static void
154i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
155 struct intel_ring_buffer *ring,
156 struct change_domains *cd)
157{
158 uint32_t invalidate_domains = 0, flush_domains = 0;
159
160 /*
161 * If the object isn't moving to a new write domain,
162 * let the object stay in multiple read domains
163 */
164 if (obj->base.pending_write_domain == 0)
165 obj->base.pending_read_domains |= obj->base.read_domains;
166
167 /*
168 * Flush the current write domain if
169 * the new read domains don't match. Invalidate
170 * any read domains which differ from the old
171 * write domain
172 */
173 if (obj->base.write_domain &&
174 (((obj->base.write_domain != obj->base.pending_read_domains ||
175 obj->ring != ring)) ||
176 (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
177 flush_domains |= obj->base.write_domain;
178 invalidate_domains |=
179 obj->base.pending_read_domains & ~obj->base.write_domain;
180 }
181 /*
182 * Invalidate any read caches which may have
183 * stale data. That is, any new read domains.
184 */
185 invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
186 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
187 i915_gem_clflush_object(obj);
188
189 /* blow away mappings if mapped through GTT */
190 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
191 i915_gem_release_mmap(obj);
192
193 /* The actual obj->write_domain will be updated with
194 * pending_write_domain after we emit the accumulated flush for all
195 * of our domain changes in execbuffers (which clears objects'
196 * write_domains). So if we have a current write domain that we
197 * aren't changing, set pending_write_domain to that.
198 */
199 if (flush_domains == 0 && obj->base.pending_write_domain == 0)
200 obj->base.pending_write_domain = obj->base.write_domain;
201
202 cd->invalidate_domains |= invalidate_domains;
203 cd->flush_domains |= flush_domains;
204 if (flush_domains & I915_GEM_GPU_DOMAINS)
205 cd->flush_rings |= obj->ring->id;
206 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
207 cd->flush_rings |= ring->id;
208}
209
210struct eb_objects {
211 int and;
212 struct hlist_head buckets[0];
213};
214
215static struct eb_objects *
216eb_create(int size)
217{
218 struct eb_objects *eb;
219 int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
220 while (count > size)
221 count >>= 1;
222 eb = kzalloc(count*sizeof(struct hlist_head) +
223 sizeof(struct eb_objects),
224 GFP_KERNEL);
225 if (eb == NULL)
226 return eb;
227
228 eb->and = count - 1;
229 return eb;
230}
231
232static void
233eb_reset(struct eb_objects *eb)
234{
235 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
236}
237
238static void
239eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
240{
241 hlist_add_head(&obj->exec_node,
242 &eb->buckets[obj->exec_handle & eb->and]);
243}
244
245static struct drm_i915_gem_object *
246eb_get_object(struct eb_objects *eb, unsigned long handle)
247{
248 struct hlist_head *head;
249 struct hlist_node *node;
250 struct drm_i915_gem_object *obj;
251
252 head = &eb->buckets[handle & eb->and];
253 hlist_for_each(node, head) {
254 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
255 if (obj->exec_handle == handle)
256 return obj;
257 }
258
259 return NULL;
260}
261
262static void
263eb_destroy(struct eb_objects *eb)
264{
265 kfree(eb);
266}
267
268static int
269i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
270 struct eb_objects *eb,
271 struct drm_i915_gem_exec_object2 *entry,
272 struct drm_i915_gem_relocation_entry *reloc)
273{
274 struct drm_device *dev = obj->base.dev;
275 struct drm_gem_object *target_obj;
276 uint32_t target_offset;
277 int ret = -EINVAL;
278
279 /* we've already hold a reference to all valid objects */
280 target_obj = &eb_get_object(eb, reloc->target_handle)->base;
281 if (unlikely(target_obj == NULL))
282 return -ENOENT;
283
284 target_offset = to_intel_bo(target_obj)->gtt_offset;
285
286#if WATCH_RELOC
287 DRM_INFO("%s: obj %p offset %08x target %d "
288 "read %08x write %08x gtt %08x "
289 "presumed %08x delta %08x\n",
290 __func__,
291 obj,
292 (int) reloc->offset,
293 (int) reloc->target_handle,
294 (int) reloc->read_domains,
295 (int) reloc->write_domain,
296 (int) target_offset,
297 (int) reloc->presumed_offset,
298 reloc->delta);
299#endif
300
301 /* The target buffer should have appeared before us in the
302 * exec_object list, so it should have a GTT space bound by now.
303 */
304 if (unlikely(target_offset == 0)) {
305 DRM_ERROR("No GTT space found for object %d\n",
306 reloc->target_handle);
307 return ret;
308 }
309
310 /* Validate that the target is in a valid r/w GPU domain */
311 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
312 DRM_ERROR("reloc with multiple write domains: "
313 "obj %p target %d offset %d "
314 "read %08x write %08x",
315 obj, reloc->target_handle,
316 (int) reloc->offset,
317 reloc->read_domains,
318 reloc->write_domain);
319 return ret;
320 }
321 if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
322 DRM_ERROR("reloc with read/write CPU domains: "
323 "obj %p target %d offset %d "
324 "read %08x write %08x",
325 obj, reloc->target_handle,
326 (int) reloc->offset,
327 reloc->read_domains,
328 reloc->write_domain);
329 return ret;
330 }
331 if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
332 reloc->write_domain != target_obj->pending_write_domain)) {
333 DRM_ERROR("Write domain conflict: "
334 "obj %p target %d offset %d "
335 "new %08x old %08x\n",
336 obj, reloc->target_handle,
337 (int) reloc->offset,
338 reloc->write_domain,
339 target_obj->pending_write_domain);
340 return ret;
341 }
342
343 target_obj->pending_read_domains |= reloc->read_domains;
344 target_obj->pending_write_domain |= reloc->write_domain;
345
346 /* If the relocation already has the right value in it, no
347 * more work needs to be done.
348 */
349 if (target_offset == reloc->presumed_offset)
350 return 0;
351
352 /* Check that the relocation address is valid... */
353 if (unlikely(reloc->offset > obj->base.size - 4)) {
354 DRM_ERROR("Relocation beyond object bounds: "
355 "obj %p target %d offset %d size %d.\n",
356 obj, reloc->target_handle,
357 (int) reloc->offset,
358 (int) obj->base.size);
359 return ret;
360 }
361 if (unlikely(reloc->offset & 3)) {
362 DRM_ERROR("Relocation not 4-byte aligned: "
363 "obj %p target %d offset %d.\n",
364 obj, reloc->target_handle,
365 (int) reloc->offset);
366 return ret;
367 }
368
369 /* and points to somewhere within the target object. */
370 if (unlikely(reloc->delta >= target_obj->size)) {
371 DRM_ERROR("Relocation beyond target object bounds: "
372 "obj %p target %d delta %d size %d.\n",
373 obj, reloc->target_handle,
374 (int) reloc->delta,
375 (int) target_obj->size);
376 return ret;
377 }
378
379 reloc->delta += target_offset;
380 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
381 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
382 char *vaddr;
383
384 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
385 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
386 kunmap_atomic(vaddr);
387 } else {
388 struct drm_i915_private *dev_priv = dev->dev_private;
389 uint32_t __iomem *reloc_entry;
390 void __iomem *reloc_page;
391
392 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
393 if (ret)
394 return ret;
395
396 /* Map the page containing the relocation we're going to perform. */
397 reloc->offset += obj->gtt_offset;
398 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
399 reloc->offset & PAGE_MASK);
400 reloc_entry = (uint32_t __iomem *)
401 (reloc_page + (reloc->offset & ~PAGE_MASK));
402 iowrite32(reloc->delta, reloc_entry);
403 io_mapping_unmap_atomic(reloc_page);
404 }
405
406 /* and update the user's relocation entry */
407 reloc->presumed_offset = target_offset;
408
409 return 0;
410}
411
412static int
413i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
414 struct eb_objects *eb,
415 struct drm_i915_gem_exec_object2 *entry)
416{
417 struct drm_i915_gem_relocation_entry __user *user_relocs;
418 int i, ret;
419
420 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
421 for (i = 0; i < entry->relocation_count; i++) {
422 struct drm_i915_gem_relocation_entry reloc;
423
424 if (__copy_from_user_inatomic(&reloc,
425 user_relocs+i,
426 sizeof(reloc)))
427 return -EFAULT;
428
429 ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &reloc);
430 if (ret)
431 return ret;
432
433 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
434 &reloc.presumed_offset,
435 sizeof(reloc.presumed_offset)))
436 return -EFAULT;
437 }
438
439 return 0;
440}
441
442static int
443i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
444 struct eb_objects *eb,
445 struct drm_i915_gem_exec_object2 *entry,
446 struct drm_i915_gem_relocation_entry *relocs)
447{
448 int i, ret;
449
450 for (i = 0; i < entry->relocation_count; i++) {
451 ret = i915_gem_execbuffer_relocate_entry(obj, eb, entry, &relocs[i]);
452 if (ret)
453 return ret;
454 }
455
456 return 0;
457}
458
459static int
460i915_gem_execbuffer_relocate(struct drm_device *dev,
461 struct eb_objects *eb,
462 struct list_head *objects,
463 struct drm_i915_gem_exec_object2 *exec)
464{
465 struct drm_i915_gem_object *obj;
466 int ret;
467
468 list_for_each_entry(obj, objects, exec_list) {
469 obj->base.pending_read_domains = 0;
470 obj->base.pending_write_domain = 0;
471 ret = i915_gem_execbuffer_relocate_object(obj, eb, exec++);
472 if (ret)
473 return ret;
474 }
475
476 return 0;
477}
478
479static int
480i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
481 struct drm_file *file,
482 struct list_head *objects,
483 struct drm_i915_gem_exec_object2 *exec)
484{
485 struct drm_i915_gem_object *obj;
486 struct drm_i915_gem_exec_object2 *entry;
487 int ret, retry;
488 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
489
490 /* Attempt to pin all of the buffers into the GTT.
491 * This is done in 3 phases:
492 *
493 * 1a. Unbind all objects that do not match the GTT constraints for
494 * the execbuffer (fenceable, mappable, alignment etc).
495 * 1b. Increment pin count for already bound objects.
496 * 2. Bind new objects.
497 * 3. Decrement pin count.
498 *
499 * This avoid unnecessary unbinding of later objects in order to makr
500 * room for the earlier objects *unless* we need to defragment.
501 */
502 retry = 0;
503 do {
504 ret = 0;
505
506 /* Unbind any ill-fitting objects or pin. */
507 entry = exec;
508 list_for_each_entry(obj, objects, exec_list) {
509 bool need_fence, need_mappable;
510
511 if (!obj->gtt_space) {
512 entry++;
513 continue;
514 }
515
516 need_fence =
517 has_fenced_gpu_access &&
518 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
519 obj->tiling_mode != I915_TILING_NONE;
520 need_mappable =
521 entry->relocation_count ? true : need_fence;
522
523 if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
524 (need_mappable && !obj->map_and_fenceable))
525 ret = i915_gem_object_unbind(obj);
526 else
527 ret = i915_gem_object_pin(obj,
528 entry->alignment,
529 need_mappable);
530 if (ret)
531 goto err;
532
533 entry++;
534 }
535
536 /* Bind fresh objects */
537 entry = exec;
538 list_for_each_entry(obj, objects, exec_list) {
539 bool need_fence;
540
541 need_fence =
542 has_fenced_gpu_access &&
543 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
544 obj->tiling_mode != I915_TILING_NONE;
545
546 if (!obj->gtt_space) {
547 bool need_mappable =
548 entry->relocation_count ? true : need_fence;
549
550 ret = i915_gem_object_pin(obj,
551 entry->alignment,
552 need_mappable);
553 if (ret)
554 break;
555 }
556
557 if (has_fenced_gpu_access) {
558 if (need_fence) {
559 ret = i915_gem_object_get_fence(obj, ring, 1);
560 if (ret)
561 break;
562 } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
563 obj->tiling_mode == I915_TILING_NONE) {
564 /* XXX pipelined! */
565 ret = i915_gem_object_put_fence(obj);
566 if (ret)
567 break;
568 }
569 obj->pending_fenced_gpu_access = need_fence;
570 }
571
572 entry->offset = obj->gtt_offset;
573 entry++;
574 }
575
576 /* Decrement pin count for bound objects */
577 list_for_each_entry(obj, objects, exec_list) {
578 if (obj->gtt_space)
579 i915_gem_object_unpin(obj);
580 }
581
582 if (ret != -ENOSPC || retry > 1)
583 return ret;
584
585 /* First attempt, just clear anything that is purgeable.
586 * Second attempt, clear the entire GTT.
587 */
588 ret = i915_gem_evict_everything(ring->dev, retry == 0);
589 if (ret)
590 return ret;
591
592 retry++;
593 } while (1);
594
595err:
596 obj = list_entry(obj->exec_list.prev,
597 struct drm_i915_gem_object,
598 exec_list);
599 while (objects != &obj->exec_list) {
600 if (obj->gtt_space)
601 i915_gem_object_unpin(obj);
602
603 obj = list_entry(obj->exec_list.prev,
604 struct drm_i915_gem_object,
605 exec_list);
606 }
607
608 return ret;
609}
610
611static int
612i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
613 struct drm_file *file,
614 struct intel_ring_buffer *ring,
615 struct list_head *objects,
616 struct eb_objects *eb,
617 struct drm_i915_gem_exec_object2 *exec,
618 int count)
619{
620 struct drm_i915_gem_relocation_entry *reloc;
621 struct drm_i915_gem_object *obj;
622 int i, total, ret;
623
624 /* We may process another execbuffer during the unlock... */
625 while (list_empty(objects)) {
626 obj = list_first_entry(objects,
627 struct drm_i915_gem_object,
628 exec_list);
629 list_del_init(&obj->exec_list);
630 drm_gem_object_unreference(&obj->base);
631 }
632
633 mutex_unlock(&dev->struct_mutex);
634
635 total = 0;
636 for (i = 0; i < count; i++)
637 total += exec[i].relocation_count;
638
639 reloc = drm_malloc_ab(total, sizeof(*reloc));
640 if (reloc == NULL) {
641 mutex_lock(&dev->struct_mutex);
642 return -ENOMEM;
643 }
644
645 total = 0;
646 for (i = 0; i < count; i++) {
647 struct drm_i915_gem_relocation_entry __user *user_relocs;
648
649 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
650
651 if (copy_from_user(reloc+total, user_relocs,
652 exec[i].relocation_count * sizeof(*reloc))) {
653 ret = -EFAULT;
654 mutex_lock(&dev->struct_mutex);
655 goto err;
656 }
657
658 total += exec[i].relocation_count;
659 }
660
661 ret = i915_mutex_lock_interruptible(dev);
662 if (ret) {
663 mutex_lock(&dev->struct_mutex);
664 goto err;
665 }
666
667 /* reacquire the objects */
668 INIT_LIST_HEAD(objects);
669 eb_reset(eb);
670 for (i = 0; i < count; i++) {
671 struct drm_i915_gem_object *obj;
672
673 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
674 exec[i].handle));
675 if (obj == NULL) {
676 DRM_ERROR("Invalid object handle %d at index %d\n",
677 exec[i].handle, i);
678 ret = -ENOENT;
679 goto err;
680 }
681
682 list_add_tail(&obj->exec_list, objects);
683 obj->exec_handle = exec[i].handle;
684 eb_add_object(eb, obj);
685 }
686
687 ret = i915_gem_execbuffer_reserve(ring, file, objects, exec);
688 if (ret)
689 goto err;
690
691 total = 0;
692 list_for_each_entry(obj, objects, exec_list) {
693 obj->base.pending_read_domains = 0;
694 obj->base.pending_write_domain = 0;
695 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
696 exec,
697 reloc + total);
698 if (ret)
699 goto err;
700
701 total += exec->relocation_count;
702 exec++;
703 }
704
705 /* Leave the user relocations as are, this is the painfully slow path,
706 * and we want to avoid the complication of dropping the lock whilst
707 * having buffers reserved in the aperture and so causing spurious
708 * ENOSPC for random operations.
709 */
710
711err:
712 drm_free_large(reloc);
713 return ret;
714}
715
716static void
717i915_gem_execbuffer_flush(struct drm_device *dev,
718 uint32_t invalidate_domains,
719 uint32_t flush_domains,
720 uint32_t flush_rings)
721{
722 drm_i915_private_t *dev_priv = dev->dev_private;
723 int i;
724
725 if (flush_domains & I915_GEM_DOMAIN_CPU)
726 intel_gtt_chipset_flush();
727
728 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
729 for (i = 0; i < I915_NUM_RINGS; i++)
730 if (flush_rings & (1 << i))
731 i915_gem_flush_ring(dev, &dev_priv->ring[i],
732 invalidate_domains,
733 flush_domains);
734 }
735}
736
737static int
738i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
739 struct intel_ring_buffer *to)
740{
741 struct intel_ring_buffer *from = obj->ring;
742 u32 seqno;
743 int ret, idx;
744
745 if (from == NULL || to == from)
746 return 0;
747
748 if (INTEL_INFO(obj->base.dev)->gen < 6)
749 return i915_gem_object_wait_rendering(obj, true);
750
751 idx = intel_ring_sync_index(from, to);
752
753 seqno = obj->last_rendering_seqno;
754 if (seqno <= from->sync_seqno[idx])
755 return 0;
756
757 if (seqno == from->outstanding_lazy_request) {
758 struct drm_i915_gem_request *request;
759
760 request = kzalloc(sizeof(*request), GFP_KERNEL);
761 if (request == NULL)
762 return -ENOMEM;
763
764 ret = i915_add_request(obj->base.dev, NULL, request, from);
765 if (ret) {
766 kfree(request);
767 return ret;
768 }
769
770 seqno = request->seqno;
771 }
772
773 from->sync_seqno[idx] = seqno;
774 return intel_ring_sync(to, from, seqno - 1);
775}
776
777static int
778i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
779 struct list_head *objects)
780{
781 struct drm_i915_gem_object *obj;
782 struct change_domains cd;
783 int ret;
784
785 cd.invalidate_domains = 0;
786 cd.flush_domains = 0;
787 cd.flush_rings = 0;
788 list_for_each_entry(obj, objects, exec_list)
789 i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
790
791 if (cd.invalidate_domains | cd.flush_domains) {
792#if WATCH_EXEC
793 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
794 __func__,
795 cd.invalidate_domains,
796 cd.flush_domains);
797#endif
798 i915_gem_execbuffer_flush(ring->dev,
799 cd.invalidate_domains,
800 cd.flush_domains,
801 cd.flush_rings);
802 }
803
804 list_for_each_entry(obj, objects, exec_list) {
805 ret = i915_gem_execbuffer_sync_rings(obj, ring);
806 if (ret)
807 return ret;
808 }
809
810 return 0;
811}
812
813static bool
814i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
815{
816 return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
817}
818
819static int
820validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
821 int count)
822{
823 int i;
824
825 for (i = 0; i < count; i++) {
826 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
827 int length; /* limited by fault_in_pages_readable() */
828
829 /* First check for malicious input causing overflow */
830 if (exec[i].relocation_count >
831 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
832 return -EINVAL;
833
834 length = exec[i].relocation_count *
835 sizeof(struct drm_i915_gem_relocation_entry);
836 if (!access_ok(VERIFY_READ, ptr, length))
837 return -EFAULT;
838
839 /* we may also need to update the presumed offsets */
840 if (!access_ok(VERIFY_WRITE, ptr, length))
841 return -EFAULT;
842
843 if (fault_in_pages_readable(ptr, length))
844 return -EFAULT;
845 }
846
847 return 0;
848}
849
850static int
851i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring,
852 struct list_head *objects)
853{
854 struct drm_i915_gem_object *obj;
855 int flips;
856
857 /* Check for any pending flips. As we only maintain a flip queue depth
858 * of 1, we can simply insert a WAIT for the next display flip prior
859 * to executing the batch and avoid stalling the CPU.
860 */
861 flips = 0;
862 list_for_each_entry(obj, objects, exec_list) {
863 if (obj->base.write_domain)
864 flips |= atomic_read(&obj->pending_flip);
865 }
866 if (flips) {
867 int plane, flip_mask, ret;
868
869 for (plane = 0; flips >> plane; plane++) {
870 if (((flips >> plane) & 1) == 0)
871 continue;
872
873 if (plane)
874 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
875 else
876 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
877
878 ret = intel_ring_begin(ring, 2);
879 if (ret)
880 return ret;
881
882 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
883 intel_ring_emit(ring, MI_NOOP);
884 intel_ring_advance(ring);
885 }
886 }
887
888 return 0;
889}
890
891static void
892i915_gem_execbuffer_move_to_active(struct list_head *objects,
893 struct intel_ring_buffer *ring,
894 u32 seqno)
895{
896 struct drm_i915_gem_object *obj;
897
898 list_for_each_entry(obj, objects, exec_list) {
899 obj->base.read_domains = obj->base.pending_read_domains;
900 obj->base.write_domain = obj->base.pending_write_domain;
901 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
902
903 i915_gem_object_move_to_active(obj, ring, seqno);
904 if (obj->base.write_domain) {
905 obj->dirty = 1;
906 obj->pending_gpu_write = true;
907 list_move_tail(&obj->gpu_write_list,
908 &ring->gpu_write_list);
909 intel_mark_busy(ring->dev, obj);
910 }
911
912 trace_i915_gem_object_change_domain(obj,
913 obj->base.read_domains,
914 obj->base.write_domain);
915 }
916}
917
918static void
919i915_gem_execbuffer_retire_commands(struct drm_device *dev,
920 struct drm_file *file,
921 struct intel_ring_buffer *ring)
922{
923 struct drm_i915_gem_request *request;
924 u32 flush_domains;
925
926 /*
927 * Ensure that the commands in the batch buffer are
928 * finished before the interrupt fires.
929 *
930 * The sampler always gets flushed on i965 (sigh).
931 */
932 flush_domains = 0;
933 if (INTEL_INFO(dev)->gen >= 4)
934 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
935
936 ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
937
938 /* Add a breadcrumb for the completion of the batch buffer */
939 request = kzalloc(sizeof(*request), GFP_KERNEL);
940 if (request == NULL || i915_add_request(dev, file, request, ring)) {
941 i915_gem_next_request_seqno(dev, ring);
942 kfree(request);
943 }
944}
945
946static int
947i915_gem_do_execbuffer(struct drm_device *dev, void *data,
948 struct drm_file *file,
949 struct drm_i915_gem_execbuffer2 *args,
950 struct drm_i915_gem_exec_object2 *exec)
951{
952 drm_i915_private_t *dev_priv = dev->dev_private;
953 struct list_head objects;
954 struct eb_objects *eb;
955 struct drm_i915_gem_object *batch_obj;
956 struct drm_clip_rect *cliprects = NULL;
957 struct intel_ring_buffer *ring;
958 u32 exec_start, exec_len;
959 u32 seqno;
960 int ret, mode, i;
961
962 if (!i915_gem_check_execbuffer(args)) {
963 DRM_ERROR("execbuf with invalid offset/length\n");
964 return -EINVAL;
965 }
966
967 ret = validate_exec_list(exec, args->buffer_count);
968 if (ret)
969 return ret;
970
971#if WATCH_EXEC
972 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
973 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
974#endif
975 switch (args->flags & I915_EXEC_RING_MASK) {
976 case I915_EXEC_DEFAULT:
977 case I915_EXEC_RENDER:
978 ring = &dev_priv->ring[RCS];
979 break;
980 case I915_EXEC_BSD:
981 if (!HAS_BSD(dev)) {
982 DRM_ERROR("execbuf with invalid ring (BSD)\n");
983 return -EINVAL;
984 }
985 ring = &dev_priv->ring[VCS];
986 break;
987 case I915_EXEC_BLT:
988 if (!HAS_BLT(dev)) {
989 DRM_ERROR("execbuf with invalid ring (BLT)\n");
990 return -EINVAL;
991 }
992 ring = &dev_priv->ring[BCS];
993 break;
994 default:
995 DRM_ERROR("execbuf with unknown ring: %d\n",
996 (int)(args->flags & I915_EXEC_RING_MASK));
997 return -EINVAL;
998 }
999
1000 mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1001 switch (mode) {
1002 case I915_EXEC_CONSTANTS_REL_GENERAL:
1003 case I915_EXEC_CONSTANTS_ABSOLUTE:
1004 case I915_EXEC_CONSTANTS_REL_SURFACE:
1005 if (ring == &dev_priv->ring[RCS] &&
1006 mode != dev_priv->relative_constants_mode) {
1007 if (INTEL_INFO(dev)->gen < 4)
1008 return -EINVAL;
1009
1010 if (INTEL_INFO(dev)->gen > 5 &&
1011 mode == I915_EXEC_CONSTANTS_REL_SURFACE)
1012 return -EINVAL;
1013
1014 ret = intel_ring_begin(ring, 4);
1015 if (ret)
1016 return ret;
1017
1018 intel_ring_emit(ring, MI_NOOP);
1019 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1020 intel_ring_emit(ring, INSTPM);
1021 intel_ring_emit(ring,
1022 I915_EXEC_CONSTANTS_MASK << 16 | mode);
1023 intel_ring_advance(ring);
1024
1025 dev_priv->relative_constants_mode = mode;
1026 }
1027 break;
1028 default:
1029 DRM_ERROR("execbuf with unknown constants: %d\n", mode);
1030 return -EINVAL;
1031 }
1032
1033 if (args->buffer_count < 1) {
1034 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1035 return -EINVAL;
1036 }
1037
1038 if (args->num_cliprects != 0) {
1039 if (ring != &dev_priv->ring[RCS]) {
1040 DRM_ERROR("clip rectangles are only valid with the render ring\n");
1041 return -EINVAL;
1042 }
1043
1044 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
1045 GFP_KERNEL);
1046 if (cliprects == NULL) {
1047 ret = -ENOMEM;
1048 goto pre_mutex_err;
1049 }
1050
1051 if (copy_from_user(cliprects,
1052 (struct drm_clip_rect __user *)(uintptr_t)
1053 args->cliprects_ptr,
1054 sizeof(*cliprects)*args->num_cliprects)) {
1055 ret = -EFAULT;
1056 goto pre_mutex_err;
1057 }
1058 }
1059
1060 ret = i915_mutex_lock_interruptible(dev);
1061 if (ret)
1062 goto pre_mutex_err;
1063
1064 if (dev_priv->mm.suspended) {
1065 mutex_unlock(&dev->struct_mutex);
1066 ret = -EBUSY;
1067 goto pre_mutex_err;
1068 }
1069
1070 eb = eb_create(args->buffer_count);
1071 if (eb == NULL) {
1072 mutex_unlock(&dev->struct_mutex);
1073 ret = -ENOMEM;
1074 goto pre_mutex_err;
1075 }
1076
1077 /* Look up object handles */
1078 INIT_LIST_HEAD(&objects);
1079 for (i = 0; i < args->buffer_count; i++) {
1080 struct drm_i915_gem_object *obj;
1081
1082 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
1083 exec[i].handle));
1084 if (obj == NULL) {
1085 DRM_ERROR("Invalid object handle %d at index %d\n",
1086 exec[i].handle, i);
1087 /* prevent error path from reading uninitialized data */
1088 ret = -ENOENT;
1089 goto err;
1090 }
1091
1092 if (!list_empty(&obj->exec_list)) {
1093 DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
1094 obj, exec[i].handle, i);
1095 ret = -EINVAL;
1096 goto err;
1097 }
1098
1099 list_add_tail(&obj->exec_list, &objects);
1100 obj->exec_handle = exec[i].handle;
1101 eb_add_object(eb, obj);
1102 }
1103
1104 /* Move the objects en-masse into the GTT, evicting if necessary. */
1105 ret = i915_gem_execbuffer_reserve(ring, file, &objects, exec);
1106 if (ret)
1107 goto err;
1108
1109 /* The objects are in their final locations, apply the relocations. */
1110 ret = i915_gem_execbuffer_relocate(dev, eb, &objects, exec);
1111 if (ret) {
1112 if (ret == -EFAULT) {
1113 ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
1114 &objects, eb,
1115 exec,
1116 args->buffer_count);
1117 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1118 }
1119 if (ret)
1120 goto err;
1121 }
1122
1123 /* Set the pending read domains for the batch buffer to COMMAND */
1124 batch_obj = list_entry(objects.prev,
1125 struct drm_i915_gem_object,
1126 exec_list);
1127 if (batch_obj->base.pending_write_domain) {
1128 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
1129 ret = -EINVAL;
1130 goto err;
1131 }
1132 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1133
1134 ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1135 if (ret)
1136 goto err;
1137
1138 ret = i915_gem_execbuffer_wait_for_flips(ring, &objects);
1139 if (ret)
1140 goto err;
1141
1142 seqno = i915_gem_next_request_seqno(dev, ring);
1143 for (i = 0; i < I915_NUM_RINGS-1; i++) {
1144 if (seqno < ring->sync_seqno[i]) {
1145 /* The GPU can not handle its semaphore value wrapping,
1146 * so every billion or so execbuffers, we need to stall
1147 * the GPU in order to reset the counters.
1148 */
1149 ret = i915_gpu_idle(dev);
1150 if (ret)
1151 goto err;
1152
1153 BUG_ON(ring->sync_seqno[i]);
1154 }
1155 }
1156
1157 exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1158 exec_len = args->batch_len;
1159 if (cliprects) {
1160 for (i = 0; i < args->num_cliprects; i++) {
1161 ret = i915_emit_box(dev, &cliprects[i],
1162 args->DR1, args->DR4);
1163 if (ret)
1164 goto err;
1165
1166 ret = ring->dispatch_execbuffer(ring,
1167 exec_start, exec_len);
1168 if (ret)
1169 goto err;
1170 }
1171 } else {
1172 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1173 if (ret)
1174 goto err;
1175 }
1176
1177 i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1178 i915_gem_execbuffer_retire_commands(dev, file, ring);
1179
1180err:
1181 eb_destroy(eb);
1182 while (!list_empty(&objects)) {
1183 struct drm_i915_gem_object *obj;
1184
1185 obj = list_first_entry(&objects,
1186 struct drm_i915_gem_object,
1187 exec_list);
1188 list_del_init(&obj->exec_list);
1189 drm_gem_object_unreference(&obj->base);
1190 }
1191
1192 mutex_unlock(&dev->struct_mutex);
1193
1194pre_mutex_err:
1195 kfree(cliprects);
1196 return ret;
1197}
1198
1199/*
1200 * Legacy execbuffer just creates an exec2 list from the original exec object
1201 * list array and passes it to the real function.
1202 */
1203int
1204i915_gem_execbuffer(struct drm_device *dev, void *data,
1205 struct drm_file *file)
1206{
1207 struct drm_i915_gem_execbuffer *args = data;
1208 struct drm_i915_gem_execbuffer2 exec2;
1209 struct drm_i915_gem_exec_object *exec_list = NULL;
1210 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1211 int ret, i;
1212
1213#if WATCH_EXEC
1214 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1215 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1216#endif
1217
1218 if (args->buffer_count < 1) {
1219 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1220 return -EINVAL;
1221 }
1222
1223 /* Copy in the exec list from userland */
1224 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1225 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1226 if (exec_list == NULL || exec2_list == NULL) {
1227 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1228 args->buffer_count);
1229 drm_free_large(exec_list);
1230 drm_free_large(exec2_list);
1231 return -ENOMEM;
1232 }
1233 ret = copy_from_user(exec_list,
1234 (struct drm_i915_relocation_entry __user *)
1235 (uintptr_t) args->buffers_ptr,
1236 sizeof(*exec_list) * args->buffer_count);
1237 if (ret != 0) {
1238 DRM_ERROR("copy %d exec entries failed %d\n",
1239 args->buffer_count, ret);
1240 drm_free_large(exec_list);
1241 drm_free_large(exec2_list);
1242 return -EFAULT;
1243 }
1244
1245 for (i = 0; i < args->buffer_count; i++) {
1246 exec2_list[i].handle = exec_list[i].handle;
1247 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1248 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1249 exec2_list[i].alignment = exec_list[i].alignment;
1250 exec2_list[i].offset = exec_list[i].offset;
1251 if (INTEL_INFO(dev)->gen < 4)
1252 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1253 else
1254 exec2_list[i].flags = 0;
1255 }
1256
1257 exec2.buffers_ptr = args->buffers_ptr;
1258 exec2.buffer_count = args->buffer_count;
1259 exec2.batch_start_offset = args->batch_start_offset;
1260 exec2.batch_len = args->batch_len;
1261 exec2.DR1 = args->DR1;
1262 exec2.DR4 = args->DR4;
1263 exec2.num_cliprects = args->num_cliprects;
1264 exec2.cliprects_ptr = args->cliprects_ptr;
1265 exec2.flags = I915_EXEC_RENDER;
1266
1267 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1268 if (!ret) {
1269 /* Copy the new buffer offsets back to the user's exec list. */
1270 for (i = 0; i < args->buffer_count; i++)
1271 exec_list[i].offset = exec2_list[i].offset;
1272 /* ... and back out to userspace */
1273 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1274 (uintptr_t) args->buffers_ptr,
1275 exec_list,
1276 sizeof(*exec_list) * args->buffer_count);
1277 if (ret) {
1278 ret = -EFAULT;
1279 DRM_ERROR("failed to copy %d exec entries "
1280 "back to user (%d)\n",
1281 args->buffer_count, ret);
1282 }
1283 }
1284
1285 drm_free_large(exec_list);
1286 drm_free_large(exec2_list);
1287 return ret;
1288}
1289
1290int
1291i915_gem_execbuffer2(struct drm_device *dev, void *data,
1292 struct drm_file *file)
1293{
1294 struct drm_i915_gem_execbuffer2 *args = data;
1295 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1296 int ret;
1297
1298#if WATCH_EXEC
1299 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1300 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1301#endif
1302
1303 if (args->buffer_count < 1) {
1304 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
1305 return -EINVAL;
1306 }
1307
1308 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1309 if (exec2_list == NULL) {
1310 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1311 args->buffer_count);
1312 return -ENOMEM;
1313 }
1314 ret = copy_from_user(exec2_list,
1315 (struct drm_i915_relocation_entry __user *)
1316 (uintptr_t) args->buffers_ptr,
1317 sizeof(*exec2_list) * args->buffer_count);
1318 if (ret != 0) {
1319 DRM_ERROR("copy %d exec entries failed %d\n",
1320 args->buffer_count, ret);
1321 drm_free_large(exec2_list);
1322 return -EFAULT;
1323 }
1324
1325 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1326 if (!ret) {
1327 /* Copy the new buffer offsets back to the user's exec list. */
1328 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1329 (uintptr_t) args->buffers_ptr,
1330 exec2_list,
1331 sizeof(*exec2_list) * args->buffer_count);
1332 if (ret) {
1333 ret = -EFAULT;
1334 DRM_ERROR("failed to copy %d exec entries "
1335 "back to user (%d)\n",
1336 args->buffer_count, ret);
1337 }
1338 }
1339
1340 drm_free_large(exec2_list);
1341 return ret;
1342}
1343
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
new file mode 100644
index 000000000000..86673e77d7cb
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -0,0 +1,99 @@
1/*
2 * Copyright © 2010 Daniel Vetter
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "drmP.h"
26#include "drm.h"
27#include "i915_drm.h"
28#include "i915_drv.h"
29#include "i915_trace.h"
30#include "intel_drv.h"
31
32void i915_gem_restore_gtt_mappings(struct drm_device *dev)
33{
34 struct drm_i915_private *dev_priv = dev->dev_private;
35 struct drm_i915_gem_object *obj;
36
37 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
38 i915_gem_clflush_object(obj);
39
40 if (dev_priv->mm.gtt->needs_dmar) {
41 BUG_ON(!obj->sg_list);
42
43 intel_gtt_insert_sg_entries(obj->sg_list,
44 obj->num_sg,
45 obj->gtt_space->start
46 >> PAGE_SHIFT,
47 obj->agp_type);
48 } else
49 intel_gtt_insert_pages(obj->gtt_space->start
50 >> PAGE_SHIFT,
51 obj->base.size >> PAGE_SHIFT,
52 obj->pages,
53 obj->agp_type);
54 }
55
56 intel_gtt_chipset_flush();
57}
58
59int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
60{
61 struct drm_device *dev = obj->base.dev;
62 struct drm_i915_private *dev_priv = dev->dev_private;
63 int ret;
64
65 if (dev_priv->mm.gtt->needs_dmar) {
66 ret = intel_gtt_map_memory(obj->pages,
67 obj->base.size >> PAGE_SHIFT,
68 &obj->sg_list,
69 &obj->num_sg);
70 if (ret != 0)
71 return ret;
72
73 intel_gtt_insert_sg_entries(obj->sg_list,
74 obj->num_sg,
75 obj->gtt_space->start >> PAGE_SHIFT,
76 obj->agp_type);
77 } else
78 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
79 obj->base.size >> PAGE_SHIFT,
80 obj->pages,
81 obj->agp_type);
82
83 return 0;
84}
85
86void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
87{
88 struct drm_device *dev = obj->base.dev;
89 struct drm_i915_private *dev_priv = dev->dev_private;
90
91 if (dev_priv->mm.gtt->needs_dmar) {
92 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
93 obj->sg_list = NULL;
94 obj->num_sg = 0;
95 }
96
97 intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
98 obj->base.size >> PAGE_SHIFT);
99}
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index af352de70be1..22a32b9932c5 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -181,7 +181,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
181} 181}
182 182
183/* Check pitch constriants for all chips & tiling formats */ 183/* Check pitch constriants for all chips & tiling formats */
184bool 184static bool
185i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) 185i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
186{ 186{
187 int tile_width; 187 int tile_width;
@@ -232,32 +232,44 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
232 return true; 232 return true;
233} 233}
234 234
235bool 235/* Is the current GTT allocation valid for the change in tiling? */
236i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode) 236static bool
237i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
237{ 238{
238 struct drm_device *dev = obj->dev; 239 u32 size;
239 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
240
241 if (obj_priv->gtt_space == NULL)
242 return true;
243 240
244 if (tiling_mode == I915_TILING_NONE) 241 if (tiling_mode == I915_TILING_NONE)
245 return true; 242 return true;
246 243
247 if (INTEL_INFO(dev)->gen >= 4) 244 if (INTEL_INFO(obj->base.dev)->gen >= 4)
248 return true; 245 return true;
249 246
250 if (obj_priv->gtt_offset & (obj->size - 1)) 247 if (INTEL_INFO(obj->base.dev)->gen == 3) {
251 return false; 248 if (obj->gtt_offset & ~I915_FENCE_START_MASK)
252
253 if (IS_GEN3(dev)) {
254 if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK)
255 return false; 249 return false;
256 } else { 250 } else {
257 if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK) 251 if (obj->gtt_offset & ~I830_FENCE_START_MASK)
258 return false; 252 return false;
259 } 253 }
260 254
255 /*
256 * Previous chips need to be aligned to the size of the smallest
257 * fence register that can contain the object.
258 */
259 if (INTEL_INFO(obj->base.dev)->gen == 3)
260 size = 1024*1024;
261 else
262 size = 512*1024;
263
264 while (size < obj->base.size)
265 size <<= 1;
266
267 if (obj->gtt_space->size != size)
268 return false;
269
270 if (obj->gtt_offset & (size - 1))
271 return false;
272
261 return true; 273 return true;
262} 274}
263 275
@@ -267,30 +279,29 @@ i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode)
267 */ 279 */
268int 280int
269i915_gem_set_tiling(struct drm_device *dev, void *data, 281i915_gem_set_tiling(struct drm_device *dev, void *data,
270 struct drm_file *file_priv) 282 struct drm_file *file)
271{ 283{
272 struct drm_i915_gem_set_tiling *args = data; 284 struct drm_i915_gem_set_tiling *args = data;
273 drm_i915_private_t *dev_priv = dev->dev_private; 285 drm_i915_private_t *dev_priv = dev->dev_private;
274 struct drm_gem_object *obj; 286 struct drm_i915_gem_object *obj;
275 struct drm_i915_gem_object *obj_priv;
276 int ret; 287 int ret;
277 288
278 ret = i915_gem_check_is_wedged(dev); 289 ret = i915_gem_check_is_wedged(dev);
279 if (ret) 290 if (ret)
280 return ret; 291 return ret;
281 292
282 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 293 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
283 if (obj == NULL) 294 if (obj == NULL)
284 return -ENOENT; 295 return -ENOENT;
285 obj_priv = to_intel_bo(obj);
286 296
287 if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) { 297 if (!i915_tiling_ok(dev,
288 drm_gem_object_unreference_unlocked(obj); 298 args->stride, obj->base.size, args->tiling_mode)) {
299 drm_gem_object_unreference_unlocked(&obj->base);
289 return -EINVAL; 300 return -EINVAL;
290 } 301 }
291 302
292 if (obj_priv->pin_count) { 303 if (obj->pin_count) {
293 drm_gem_object_unreference_unlocked(obj); 304 drm_gem_object_unreference_unlocked(&obj->base);
294 return -EBUSY; 305 return -EBUSY;
295 } 306 }
296 307
@@ -324,34 +335,28 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
324 } 335 }
325 336
326 mutex_lock(&dev->struct_mutex); 337 mutex_lock(&dev->struct_mutex);
327 if (args->tiling_mode != obj_priv->tiling_mode || 338 if (args->tiling_mode != obj->tiling_mode ||
328 args->stride != obj_priv->stride) { 339 args->stride != obj->stride) {
329 /* We need to rebind the object if its current allocation 340 /* We need to rebind the object if its current allocation
330 * no longer meets the alignment restrictions for its new 341 * no longer meets the alignment restrictions for its new
331 * tiling mode. Otherwise we can just leave it alone, but 342 * tiling mode. Otherwise we can just leave it alone, but
332 * need to ensure that any fence register is cleared. 343 * need to ensure that any fence register is cleared.
333 */ 344 */
334 if (!i915_gem_object_fence_offset_ok(obj, args->tiling_mode)) 345 i915_gem_release_mmap(obj);
335 ret = i915_gem_object_unbind(obj);
336 else if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
337 ret = i915_gem_object_put_fence_reg(obj, true);
338 else
339 i915_gem_release_mmap(obj);
340 346
341 if (ret != 0) { 347 obj->map_and_fenceable =
342 args->tiling_mode = obj_priv->tiling_mode; 348 obj->gtt_space == NULL ||
343 args->stride = obj_priv->stride; 349 (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end &&
344 goto err; 350 i915_gem_object_fence_ok(obj, args->tiling_mode));
345 }
346 351
347 obj_priv->tiling_mode = args->tiling_mode; 352 obj->tiling_changed = true;
348 obj_priv->stride = args->stride; 353 obj->tiling_mode = args->tiling_mode;
354 obj->stride = args->stride;
349 } 355 }
350err: 356 drm_gem_object_unreference(&obj->base);
351 drm_gem_object_unreference(obj);
352 mutex_unlock(&dev->struct_mutex); 357 mutex_unlock(&dev->struct_mutex);
353 358
354 return ret; 359 return 0;
355} 360}
356 361
357/** 362/**
@@ -359,22 +364,20 @@ err:
359 */ 364 */
360int 365int
361i915_gem_get_tiling(struct drm_device *dev, void *data, 366i915_gem_get_tiling(struct drm_device *dev, void *data,
362 struct drm_file *file_priv) 367 struct drm_file *file)
363{ 368{
364 struct drm_i915_gem_get_tiling *args = data; 369 struct drm_i915_gem_get_tiling *args = data;
365 drm_i915_private_t *dev_priv = dev->dev_private; 370 drm_i915_private_t *dev_priv = dev->dev_private;
366 struct drm_gem_object *obj; 371 struct drm_i915_gem_object *obj;
367 struct drm_i915_gem_object *obj_priv;
368 372
369 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 373 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
370 if (obj == NULL) 374 if (obj == NULL)
371 return -ENOENT; 375 return -ENOENT;
372 obj_priv = to_intel_bo(obj);
373 376
374 mutex_lock(&dev->struct_mutex); 377 mutex_lock(&dev->struct_mutex);
375 378
376 args->tiling_mode = obj_priv->tiling_mode; 379 args->tiling_mode = obj->tiling_mode;
377 switch (obj_priv->tiling_mode) { 380 switch (obj->tiling_mode) {
378 case I915_TILING_X: 381 case I915_TILING_X:
379 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; 382 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
380 break; 383 break;
@@ -394,7 +397,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
394 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 397 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
395 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 398 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
396 399
397 drm_gem_object_unreference(obj); 400 drm_gem_object_unreference(&obj->base);
398 mutex_unlock(&dev->struct_mutex); 401 mutex_unlock(&dev->struct_mutex);
399 402
400 return 0; 403 return 0;
@@ -424,46 +427,44 @@ i915_gem_swizzle_page(struct page *page)
424} 427}
425 428
426void 429void
427i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj) 430i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
428{ 431{
429 struct drm_device *dev = obj->dev; 432 struct drm_device *dev = obj->base.dev;
430 drm_i915_private_t *dev_priv = dev->dev_private; 433 drm_i915_private_t *dev_priv = dev->dev_private;
431 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 434 int page_count = obj->base.size >> PAGE_SHIFT;
432 int page_count = obj->size >> PAGE_SHIFT;
433 int i; 435 int i;
434 436
435 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) 437 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
436 return; 438 return;
437 439
438 if (obj_priv->bit_17 == NULL) 440 if (obj->bit_17 == NULL)
439 return; 441 return;
440 442
441 for (i = 0; i < page_count; i++) { 443 for (i = 0; i < page_count; i++) {
442 char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17; 444 char new_bit_17 = page_to_phys(obj->pages[i]) >> 17;
443 if ((new_bit_17 & 0x1) != 445 if ((new_bit_17 & 0x1) !=
444 (test_bit(i, obj_priv->bit_17) != 0)) { 446 (test_bit(i, obj->bit_17) != 0)) {
445 i915_gem_swizzle_page(obj_priv->pages[i]); 447 i915_gem_swizzle_page(obj->pages[i]);
446 set_page_dirty(obj_priv->pages[i]); 448 set_page_dirty(obj->pages[i]);
447 } 449 }
448 } 450 }
449} 451}
450 452
451void 453void
452i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj) 454i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
453{ 455{
454 struct drm_device *dev = obj->dev; 456 struct drm_device *dev = obj->base.dev;
455 drm_i915_private_t *dev_priv = dev->dev_private; 457 drm_i915_private_t *dev_priv = dev->dev_private;
456 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 458 int page_count = obj->base.size >> PAGE_SHIFT;
457 int page_count = obj->size >> PAGE_SHIFT;
458 int i; 459 int i;
459 460
460 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) 461 if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
461 return; 462 return;
462 463
463 if (obj_priv->bit_17 == NULL) { 464 if (obj->bit_17 == NULL) {
464 obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) * 465 obj->bit_17 = kmalloc(BITS_TO_LONGS(page_count) *
465 sizeof(long), GFP_KERNEL); 466 sizeof(long), GFP_KERNEL);
466 if (obj_priv->bit_17 == NULL) { 467 if (obj->bit_17 == NULL) {
467 DRM_ERROR("Failed to allocate memory for bit 17 " 468 DRM_ERROR("Failed to allocate memory for bit 17 "
468 "record\n"); 469 "record\n");
469 return; 470 return;
@@ -471,9 +472,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj)
471 } 472 }
472 473
473 for (i = 0; i < page_count; i++) { 474 for (i = 0; i < page_count; i++) {
474 if (page_to_phys(obj_priv->pages[i]) & (1 << 17)) 475 if (page_to_phys(obj->pages[i]) & (1 << 17))
475 __set_bit(i, obj_priv->bit_17); 476 __set_bit(i, obj->bit_17);
476 else 477 else
477 __clear_bit(i, obj_priv->bit_17); 478 __clear_bit(i, obj->bit_17);
478 } 479 }
479} 480}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 729fd0c91d7b..0dadc025b77b 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -67,20 +67,20 @@
67void 67void
68ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) 68ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
69{ 69{
70 if ((dev_priv->gt_irq_mask_reg & mask) != 0) { 70 if ((dev_priv->gt_irq_mask & mask) != 0) {
71 dev_priv->gt_irq_mask_reg &= ~mask; 71 dev_priv->gt_irq_mask &= ~mask;
72 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 72 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
73 (void) I915_READ(GTIMR); 73 POSTING_READ(GTIMR);
74 } 74 }
75} 75}
76 76
77void 77void
78ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) 78ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
79{ 79{
80 if ((dev_priv->gt_irq_mask_reg & mask) != mask) { 80 if ((dev_priv->gt_irq_mask & mask) != mask) {
81 dev_priv->gt_irq_mask_reg |= mask; 81 dev_priv->gt_irq_mask |= mask;
82 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 82 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
83 (void) I915_READ(GTIMR); 83 POSTING_READ(GTIMR);
84 } 84 }
85} 85}
86 86
@@ -88,40 +88,40 @@ ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask)
88static void 88static void
89ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 89ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
90{ 90{
91 if ((dev_priv->irq_mask_reg & mask) != 0) { 91 if ((dev_priv->irq_mask & mask) != 0) {
92 dev_priv->irq_mask_reg &= ~mask; 92 dev_priv->irq_mask &= ~mask;
93 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 93 I915_WRITE(DEIMR, dev_priv->irq_mask);
94 (void) I915_READ(DEIMR); 94 POSTING_READ(DEIMR);
95 } 95 }
96} 96}
97 97
98static inline void 98static inline void
99ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) 99ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
100{ 100{
101 if ((dev_priv->irq_mask_reg & mask) != mask) { 101 if ((dev_priv->irq_mask & mask) != mask) {
102 dev_priv->irq_mask_reg |= mask; 102 dev_priv->irq_mask |= mask;
103 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 103 I915_WRITE(DEIMR, dev_priv->irq_mask);
104 (void) I915_READ(DEIMR); 104 POSTING_READ(DEIMR);
105 } 105 }
106} 106}
107 107
108void 108void
109i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) 109i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
110{ 110{
111 if ((dev_priv->irq_mask_reg & mask) != 0) { 111 if ((dev_priv->irq_mask & mask) != 0) {
112 dev_priv->irq_mask_reg &= ~mask; 112 dev_priv->irq_mask &= ~mask;
113 I915_WRITE(IMR, dev_priv->irq_mask_reg); 113 I915_WRITE(IMR, dev_priv->irq_mask);
114 (void) I915_READ(IMR); 114 POSTING_READ(IMR);
115 } 115 }
116} 116}
117 117
118void 118void
119i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask) 119i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
120{ 120{
121 if ((dev_priv->irq_mask_reg & mask) != mask) { 121 if ((dev_priv->irq_mask & mask) != mask) {
122 dev_priv->irq_mask_reg |= mask; 122 dev_priv->irq_mask |= mask;
123 I915_WRITE(IMR, dev_priv->irq_mask_reg); 123 I915_WRITE(IMR, dev_priv->irq_mask);
124 (void) I915_READ(IMR); 124 POSTING_READ(IMR);
125 } 125 }
126} 126}
127 127
@@ -144,7 +144,7 @@ i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
144 dev_priv->pipestat[pipe] |= mask; 144 dev_priv->pipestat[pipe] |= mask;
145 /* Enable the interrupt, clear any pending status */ 145 /* Enable the interrupt, clear any pending status */
146 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); 146 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
147 (void) I915_READ(reg); 147 POSTING_READ(reg);
148 } 148 }
149} 149}
150 150
@@ -156,16 +156,19 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
156 156
157 dev_priv->pipestat[pipe] &= ~mask; 157 dev_priv->pipestat[pipe] &= ~mask;
158 I915_WRITE(reg, dev_priv->pipestat[pipe]); 158 I915_WRITE(reg, dev_priv->pipestat[pipe]);
159 (void) I915_READ(reg); 159 POSTING_READ(reg);
160 } 160 }
161} 161}
162 162
163/** 163/**
164 * intel_enable_asle - enable ASLE interrupt for OpRegion 164 * intel_enable_asle - enable ASLE interrupt for OpRegion
165 */ 165 */
166void intel_enable_asle (struct drm_device *dev) 166void intel_enable_asle(struct drm_device *dev)
167{ 167{
168 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 168 drm_i915_private_t *dev_priv = dev->dev_private;
169 unsigned long irqflags;
170
171 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
169 172
170 if (HAS_PCH_SPLIT(dev)) 173 if (HAS_PCH_SPLIT(dev))
171 ironlake_enable_display_irq(dev_priv, DE_GSE); 174 ironlake_enable_display_irq(dev_priv, DE_GSE);
@@ -176,6 +179,8 @@ void intel_enable_asle (struct drm_device *dev)
176 i915_enable_pipestat(dev_priv, 0, 179 i915_enable_pipestat(dev_priv, 0,
177 PIPE_LEGACY_BLC_EVENT_ENABLE); 180 PIPE_LEGACY_BLC_EVENT_ENABLE);
178 } 181 }
182
183 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
179} 184}
180 185
181/** 186/**
@@ -243,6 +248,92 @@ u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
243 return I915_READ(reg); 248 return I915_READ(reg);
244} 249}
245 250
251int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
252 int *vpos, int *hpos)
253{
254 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
255 u32 vbl = 0, position = 0;
256 int vbl_start, vbl_end, htotal, vtotal;
257 bool in_vbl = true;
258 int ret = 0;
259
260 if (!i915_pipe_enabled(dev, pipe)) {
261 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
262 "pipe %d\n", pipe);
263 return 0;
264 }
265
266 /* Get vtotal. */
267 vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff);
268
269 if (INTEL_INFO(dev)->gen >= 4) {
270 /* No obvious pixelcount register. Only query vertical
271 * scanout position from Display scan line register.
272 */
273 position = I915_READ(PIPEDSL(pipe));
274
275 /* Decode into vertical scanout position. Don't have
276 * horizontal scanout position.
277 */
278 *vpos = position & 0x1fff;
279 *hpos = 0;
280 } else {
281 /* Have access to pixelcount since start of frame.
282 * We can split this into vertical and horizontal
283 * scanout position.
284 */
285 position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
286
287 htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff);
288 *vpos = position / htotal;
289 *hpos = position - (*vpos * htotal);
290 }
291
292 /* Query vblank area. */
293 vbl = I915_READ(VBLANK(pipe));
294
295 /* Test position against vblank region. */
296 vbl_start = vbl & 0x1fff;
297 vbl_end = (vbl >> 16) & 0x1fff;
298
299 if ((*vpos < vbl_start) || (*vpos > vbl_end))
300 in_vbl = false;
301
302 /* Inside "upper part" of vblank area? Apply corrective offset: */
303 if (in_vbl && (*vpos >= vbl_start))
304 *vpos = *vpos - vtotal;
305
306 /* Readouts valid? */
307 if (vbl > 0)
308 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
309
310 /* In vblank? */
311 if (in_vbl)
312 ret |= DRM_SCANOUTPOS_INVBL;
313
314 return ret;
315}
316
317int i915_get_vblank_timestamp(struct drm_device *dev, int crtc,
318 int *max_error,
319 struct timeval *vblank_time,
320 unsigned flags)
321{
322 struct drm_crtc *drmcrtc;
323
324 if (crtc < 0 || crtc >= dev->num_crtcs) {
325 DRM_ERROR("Invalid crtc %d\n", crtc);
326 return -EINVAL;
327 }
328
329 /* Get drm_crtc to timestamp: */
330 drmcrtc = intel_get_crtc_for_pipe(dev, crtc);
331
332 /* Helper routine in DRM core does all the work: */
333 return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
334 vblank_time, flags, drmcrtc);
335}
336
246/* 337/*
247 * Handle hotplug events outside the interrupt handler proper. 338 * Handle hotplug events outside the interrupt handler proper.
248 */ 339 */
@@ -297,8 +388,8 @@ static void notify_ring(struct drm_device *dev,
297 struct intel_ring_buffer *ring) 388 struct intel_ring_buffer *ring)
298{ 389{
299 struct drm_i915_private *dev_priv = dev->dev_private; 390 struct drm_i915_private *dev_priv = dev->dev_private;
300 u32 seqno = ring->get_seqno(dev, ring); 391 u32 seqno = ring->get_seqno(ring);
301 ring->irq_gem_seqno = seqno; 392 ring->irq_seqno = seqno;
302 trace_i915_gem_request_complete(dev, seqno); 393 trace_i915_gem_request_complete(dev, seqno);
303 wake_up_all(&ring->irq_queue); 394 wake_up_all(&ring->irq_queue);
304 dev_priv->hangcheck_count = 0; 395 dev_priv->hangcheck_count = 0;
@@ -306,11 +397,49 @@ static void notify_ring(struct drm_device *dev,
306 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); 397 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
307} 398}
308 399
400static void gen6_pm_irq_handler(struct drm_device *dev)
401{
402 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
403 u8 new_delay = dev_priv->cur_delay;
404 u32 pm_iir;
405
406 pm_iir = I915_READ(GEN6_PMIIR);
407 if (!pm_iir)
408 return;
409
410 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
411 if (dev_priv->cur_delay != dev_priv->max_delay)
412 new_delay = dev_priv->cur_delay + 1;
413 if (new_delay > dev_priv->max_delay)
414 new_delay = dev_priv->max_delay;
415 } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) {
416 if (dev_priv->cur_delay != dev_priv->min_delay)
417 new_delay = dev_priv->cur_delay - 1;
418 if (new_delay < dev_priv->min_delay) {
419 new_delay = dev_priv->min_delay;
420 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
421 I915_READ(GEN6_RP_INTERRUPT_LIMITS) |
422 ((new_delay << 16) & 0x3f0000));
423 } else {
424 /* Make sure we continue to get down interrupts
425 * until we hit the minimum frequency */
426 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
427 I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000);
428 }
429
430 }
431
432 gen6_set_rps(dev, new_delay);
433 dev_priv->cur_delay = new_delay;
434
435 I915_WRITE(GEN6_PMIIR, pm_iir);
436}
437
309static irqreturn_t ironlake_irq_handler(struct drm_device *dev) 438static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
310{ 439{
311 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 440 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
312 int ret = IRQ_NONE; 441 int ret = IRQ_NONE;
313 u32 de_iir, gt_iir, de_ier, pch_iir; 442 u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
314 u32 hotplug_mask; 443 u32 hotplug_mask;
315 struct drm_i915_master_private *master_priv; 444 struct drm_i915_master_private *master_priv;
316 u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT; 445 u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;
@@ -321,13 +450,15 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
321 /* disable master interrupt before clearing iir */ 450 /* disable master interrupt before clearing iir */
322 de_ier = I915_READ(DEIER); 451 de_ier = I915_READ(DEIER);
323 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); 452 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
324 (void)I915_READ(DEIER); 453 POSTING_READ(DEIER);
325 454
326 de_iir = I915_READ(DEIIR); 455 de_iir = I915_READ(DEIIR);
327 gt_iir = I915_READ(GTIIR); 456 gt_iir = I915_READ(GTIIR);
328 pch_iir = I915_READ(SDEIIR); 457 pch_iir = I915_READ(SDEIIR);
458 pm_iir = I915_READ(GEN6_PMIIR);
329 459
330 if (de_iir == 0 && gt_iir == 0 && pch_iir == 0) 460 if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
461 (!IS_GEN6(dev) || pm_iir == 0))
331 goto done; 462 goto done;
332 463
333 if (HAS_PCH_CPT(dev)) 464 if (HAS_PCH_CPT(dev))
@@ -344,12 +475,12 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
344 READ_BREADCRUMB(dev_priv); 475 READ_BREADCRUMB(dev_priv);
345 } 476 }
346 477
347 if (gt_iir & GT_PIPE_NOTIFY) 478 if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
348 notify_ring(dev, &dev_priv->render_ring); 479 notify_ring(dev, &dev_priv->ring[RCS]);
349 if (gt_iir & bsd_usr_interrupt) 480 if (gt_iir & bsd_usr_interrupt)
350 notify_ring(dev, &dev_priv->bsd_ring); 481 notify_ring(dev, &dev_priv->ring[VCS]);
351 if (HAS_BLT(dev) && gt_iir & GT_BLT_USER_INTERRUPT) 482 if (gt_iir & GT_BLT_USER_INTERRUPT)
352 notify_ring(dev, &dev_priv->blt_ring); 483 notify_ring(dev, &dev_priv->ring[BCS]);
353 484
354 if (de_iir & DE_GSE) 485 if (de_iir & DE_GSE)
355 intel_opregion_gse_intr(dev); 486 intel_opregion_gse_intr(dev);
@@ -379,6 +510,9 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
379 i915_handle_rps_change(dev); 510 i915_handle_rps_change(dev);
380 } 511 }
381 512
513 if (IS_GEN6(dev))
514 gen6_pm_irq_handler(dev);
515
382 /* should clear PCH hotplug event before clear CPU irq */ 516 /* should clear PCH hotplug event before clear CPU irq */
383 I915_WRITE(SDEIIR, pch_iir); 517 I915_WRITE(SDEIIR, pch_iir);
384 I915_WRITE(GTIIR, gt_iir); 518 I915_WRITE(GTIIR, gt_iir);
@@ -386,7 +520,7 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev)
386 520
387done: 521done:
388 I915_WRITE(DEIER, de_ier); 522 I915_WRITE(DEIER, de_ier);
389 (void)I915_READ(DEIER); 523 POSTING_READ(DEIER);
390 524
391 return ret; 525 return ret;
392} 526}
@@ -423,28 +557,23 @@ static void i915_error_work_func(struct work_struct *work)
423#ifdef CONFIG_DEBUG_FS 557#ifdef CONFIG_DEBUG_FS
424static struct drm_i915_error_object * 558static struct drm_i915_error_object *
425i915_error_object_create(struct drm_device *dev, 559i915_error_object_create(struct drm_device *dev,
426 struct drm_gem_object *src) 560 struct drm_i915_gem_object *src)
427{ 561{
428 drm_i915_private_t *dev_priv = dev->dev_private; 562 drm_i915_private_t *dev_priv = dev->dev_private;
429 struct drm_i915_error_object *dst; 563 struct drm_i915_error_object *dst;
430 struct drm_i915_gem_object *src_priv;
431 int page, page_count; 564 int page, page_count;
432 u32 reloc_offset; 565 u32 reloc_offset;
433 566
434 if (src == NULL) 567 if (src == NULL || src->pages == NULL)
435 return NULL;
436
437 src_priv = to_intel_bo(src);
438 if (src_priv->pages == NULL)
439 return NULL; 568 return NULL;
440 569
441 page_count = src->size / PAGE_SIZE; 570 page_count = src->base.size / PAGE_SIZE;
442 571
443 dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC); 572 dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
444 if (dst == NULL) 573 if (dst == NULL)
445 return NULL; 574 return NULL;
446 575
447 reloc_offset = src_priv->gtt_offset; 576 reloc_offset = src->gtt_offset;
448 for (page = 0; page < page_count; page++) { 577 for (page = 0; page < page_count; page++) {
449 unsigned long flags; 578 unsigned long flags;
450 void __iomem *s; 579 void __iomem *s;
@@ -466,7 +595,7 @@ i915_error_object_create(struct drm_device *dev,
466 reloc_offset += PAGE_SIZE; 595 reloc_offset += PAGE_SIZE;
467 } 596 }
468 dst->page_count = page_count; 597 dst->page_count = page_count;
469 dst->gtt_offset = src_priv->gtt_offset; 598 dst->gtt_offset = src->gtt_offset;
470 599
471 return dst; 600 return dst;
472 601
@@ -520,36 +649,96 @@ i915_get_bbaddr(struct drm_device *dev, u32 *ring)
520} 649}
521 650
522static u32 651static u32
523i915_ringbuffer_last_batch(struct drm_device *dev) 652i915_ringbuffer_last_batch(struct drm_device *dev,
653 struct intel_ring_buffer *ring)
524{ 654{
525 struct drm_i915_private *dev_priv = dev->dev_private; 655 struct drm_i915_private *dev_priv = dev->dev_private;
526 u32 head, bbaddr; 656 u32 head, bbaddr;
527 u32 *ring; 657 u32 *val;
528 658
529 /* Locate the current position in the ringbuffer and walk back 659 /* Locate the current position in the ringbuffer and walk back
530 * to find the most recently dispatched batch buffer. 660 * to find the most recently dispatched batch buffer.
531 */ 661 */
532 bbaddr = 0; 662 head = I915_READ_HEAD(ring) & HEAD_ADDR;
533 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
534 ring = (u32 *)(dev_priv->render_ring.virtual_start + head);
535 663
536 while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { 664 val = (u32 *)(ring->virtual_start + head);
537 bbaddr = i915_get_bbaddr(dev, ring); 665 while (--val >= (u32 *)ring->virtual_start) {
666 bbaddr = i915_get_bbaddr(dev, val);
538 if (bbaddr) 667 if (bbaddr)
539 break; 668 return bbaddr;
540 } 669 }
541 670
542 if (bbaddr == 0) { 671 val = (u32 *)(ring->virtual_start + ring->size);
543 ring = (u32 *)(dev_priv->render_ring.virtual_start 672 while (--val >= (u32 *)ring->virtual_start) {
544 + dev_priv->render_ring.size); 673 bbaddr = i915_get_bbaddr(dev, val);
545 while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { 674 if (bbaddr)
546 bbaddr = i915_get_bbaddr(dev, ring); 675 return bbaddr;
547 if (bbaddr)
548 break;
549 }
550 } 676 }
551 677
552 return bbaddr; 678 return 0;
679}
680
681static u32 capture_bo_list(struct drm_i915_error_buffer *err,
682 int count,
683 struct list_head *head)
684{
685 struct drm_i915_gem_object *obj;
686 int i = 0;
687
688 list_for_each_entry(obj, head, mm_list) {
689 err->size = obj->base.size;
690 err->name = obj->base.name;
691 err->seqno = obj->last_rendering_seqno;
692 err->gtt_offset = obj->gtt_offset;
693 err->read_domains = obj->base.read_domains;
694 err->write_domain = obj->base.write_domain;
695 err->fence_reg = obj->fence_reg;
696 err->pinned = 0;
697 if (obj->pin_count > 0)
698 err->pinned = 1;
699 if (obj->user_pin_count > 0)
700 err->pinned = -1;
701 err->tiling = obj->tiling_mode;
702 err->dirty = obj->dirty;
703 err->purgeable = obj->madv != I915_MADV_WILLNEED;
704 err->ring = obj->ring ? obj->ring->id : 0;
705
706 if (++i == count)
707 break;
708
709 err++;
710 }
711
712 return i;
713}
714
715static void i915_gem_record_fences(struct drm_device *dev,
716 struct drm_i915_error_state *error)
717{
718 struct drm_i915_private *dev_priv = dev->dev_private;
719 int i;
720
721 /* Fences */
722 switch (INTEL_INFO(dev)->gen) {
723 case 6:
724 for (i = 0; i < 16; i++)
725 error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
726 break;
727 case 5:
728 case 4:
729 for (i = 0; i < 16; i++)
730 error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
731 break;
732 case 3:
733 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
734 for (i = 0; i < 8; i++)
735 error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
736 case 2:
737 for (i = 0; i < 8; i++)
738 error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
739 break;
740
741 }
553} 742}
554 743
555/** 744/**
@@ -564,9 +753,9 @@ i915_ringbuffer_last_batch(struct drm_device *dev)
564static void i915_capture_error_state(struct drm_device *dev) 753static void i915_capture_error_state(struct drm_device *dev)
565{ 754{
566 struct drm_i915_private *dev_priv = dev->dev_private; 755 struct drm_i915_private *dev_priv = dev->dev_private;
567 struct drm_i915_gem_object *obj_priv; 756 struct drm_i915_gem_object *obj;
568 struct drm_i915_error_state *error; 757 struct drm_i915_error_state *error;
569 struct drm_gem_object *batchbuffer[2]; 758 struct drm_i915_gem_object *batchbuffer[2];
570 unsigned long flags; 759 unsigned long flags;
571 u32 bbaddr; 760 u32 bbaddr;
572 int count; 761 int count;
@@ -585,20 +774,33 @@ static void i915_capture_error_state(struct drm_device *dev)
585 774
586 DRM_DEBUG_DRIVER("generating error event\n"); 775 DRM_DEBUG_DRIVER("generating error event\n");
587 776
588 error->seqno = 777 error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
589 dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring);
590 error->eir = I915_READ(EIR); 778 error->eir = I915_READ(EIR);
591 error->pgtbl_er = I915_READ(PGTBL_ER); 779 error->pgtbl_er = I915_READ(PGTBL_ER);
592 error->pipeastat = I915_READ(PIPEASTAT); 780 error->pipeastat = I915_READ(PIPEASTAT);
593 error->pipebstat = I915_READ(PIPEBSTAT); 781 error->pipebstat = I915_READ(PIPEBSTAT);
594 error->instpm = I915_READ(INSTPM); 782 error->instpm = I915_READ(INSTPM);
595 if (INTEL_INFO(dev)->gen < 4) { 783 error->error = 0;
596 error->ipeir = I915_READ(IPEIR); 784 if (INTEL_INFO(dev)->gen >= 6) {
597 error->ipehr = I915_READ(IPEHR); 785 error->error = I915_READ(ERROR_GEN6);
598 error->instdone = I915_READ(INSTDONE); 786
599 error->acthd = I915_READ(ACTHD); 787 error->bcs_acthd = I915_READ(BCS_ACTHD);
600 error->bbaddr = 0; 788 error->bcs_ipehr = I915_READ(BCS_IPEHR);
601 } else { 789 error->bcs_ipeir = I915_READ(BCS_IPEIR);
790 error->bcs_instdone = I915_READ(BCS_INSTDONE);
791 error->bcs_seqno = 0;
792 if (dev_priv->ring[BCS].get_seqno)
793 error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
794
795 error->vcs_acthd = I915_READ(VCS_ACTHD);
796 error->vcs_ipehr = I915_READ(VCS_IPEHR);
797 error->vcs_ipeir = I915_READ(VCS_IPEIR);
798 error->vcs_instdone = I915_READ(VCS_INSTDONE);
799 error->vcs_seqno = 0;
800 if (dev_priv->ring[VCS].get_seqno)
801 error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
802 }
803 if (INTEL_INFO(dev)->gen >= 4) {
602 error->ipeir = I915_READ(IPEIR_I965); 804 error->ipeir = I915_READ(IPEIR_I965);
603 error->ipehr = I915_READ(IPEHR_I965); 805 error->ipehr = I915_READ(IPEHR_I965);
604 error->instdone = I915_READ(INSTDONE_I965); 806 error->instdone = I915_READ(INSTDONE_I965);
@@ -606,42 +808,45 @@ static void i915_capture_error_state(struct drm_device *dev)
606 error->instdone1 = I915_READ(INSTDONE1); 808 error->instdone1 = I915_READ(INSTDONE1);
607 error->acthd = I915_READ(ACTHD_I965); 809 error->acthd = I915_READ(ACTHD_I965);
608 error->bbaddr = I915_READ64(BB_ADDR); 810 error->bbaddr = I915_READ64(BB_ADDR);
811 } else {
812 error->ipeir = I915_READ(IPEIR);
813 error->ipehr = I915_READ(IPEHR);
814 error->instdone = I915_READ(INSTDONE);
815 error->acthd = I915_READ(ACTHD);
816 error->bbaddr = 0;
609 } 817 }
818 i915_gem_record_fences(dev, error);
610 819
611 bbaddr = i915_ringbuffer_last_batch(dev); 820 bbaddr = i915_ringbuffer_last_batch(dev, &dev_priv->ring[RCS]);
612 821
613 /* Grab the current batchbuffer, most likely to have crashed. */ 822 /* Grab the current batchbuffer, most likely to have crashed. */
614 batchbuffer[0] = NULL; 823 batchbuffer[0] = NULL;
615 batchbuffer[1] = NULL; 824 batchbuffer[1] = NULL;
616 count = 0; 825 count = 0;
617 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { 826 list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
618 struct drm_gem_object *obj = &obj_priv->base;
619
620 if (batchbuffer[0] == NULL && 827 if (batchbuffer[0] == NULL &&
621 bbaddr >= obj_priv->gtt_offset && 828 bbaddr >= obj->gtt_offset &&
622 bbaddr < obj_priv->gtt_offset + obj->size) 829 bbaddr < obj->gtt_offset + obj->base.size)
623 batchbuffer[0] = obj; 830 batchbuffer[0] = obj;
624 831
625 if (batchbuffer[1] == NULL && 832 if (batchbuffer[1] == NULL &&
626 error->acthd >= obj_priv->gtt_offset && 833 error->acthd >= obj->gtt_offset &&
627 error->acthd < obj_priv->gtt_offset + obj->size) 834 error->acthd < obj->gtt_offset + obj->base.size)
628 batchbuffer[1] = obj; 835 batchbuffer[1] = obj;
629 836
630 count++; 837 count++;
631 } 838 }
632 /* Scan the other lists for completeness for those bizarre errors. */ 839 /* Scan the other lists for completeness for those bizarre errors. */
633 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { 840 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
634 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { 841 list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
635 struct drm_gem_object *obj = &obj_priv->base;
636
637 if (batchbuffer[0] == NULL && 842 if (batchbuffer[0] == NULL &&
638 bbaddr >= obj_priv->gtt_offset && 843 bbaddr >= obj->gtt_offset &&
639 bbaddr < obj_priv->gtt_offset + obj->size) 844 bbaddr < obj->gtt_offset + obj->base.size)
640 batchbuffer[0] = obj; 845 batchbuffer[0] = obj;
641 846
642 if (batchbuffer[1] == NULL && 847 if (batchbuffer[1] == NULL &&
643 error->acthd >= obj_priv->gtt_offset && 848 error->acthd >= obj->gtt_offset &&
644 error->acthd < obj_priv->gtt_offset + obj->size) 849 error->acthd < obj->gtt_offset + obj->base.size)
645 batchbuffer[1] = obj; 850 batchbuffer[1] = obj;
646 851
647 if (batchbuffer[0] && batchbuffer[1]) 852 if (batchbuffer[0] && batchbuffer[1])
@@ -649,17 +854,15 @@ static void i915_capture_error_state(struct drm_device *dev)
649 } 854 }
650 } 855 }
651 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { 856 if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) {
652 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { 857 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
653 struct drm_gem_object *obj = &obj_priv->base;
654
655 if (batchbuffer[0] == NULL && 858 if (batchbuffer[0] == NULL &&
656 bbaddr >= obj_priv->gtt_offset && 859 bbaddr >= obj->gtt_offset &&
657 bbaddr < obj_priv->gtt_offset + obj->size) 860 bbaddr < obj->gtt_offset + obj->base.size)
658 batchbuffer[0] = obj; 861 batchbuffer[0] = obj;
659 862
660 if (batchbuffer[1] == NULL && 863 if (batchbuffer[1] == NULL &&
661 error->acthd >= obj_priv->gtt_offset && 864 error->acthd >= obj->gtt_offset &&
662 error->acthd < obj_priv->gtt_offset + obj->size) 865 error->acthd < obj->gtt_offset + obj->base.size)
663 batchbuffer[1] = obj; 866 batchbuffer[1] = obj;
664 867
665 if (batchbuffer[0] && batchbuffer[1]) 868 if (batchbuffer[0] && batchbuffer[1])
@@ -678,46 +881,41 @@ static void i915_capture_error_state(struct drm_device *dev)
678 881
679 /* Record the ringbuffer */ 882 /* Record the ringbuffer */
680 error->ringbuffer = i915_error_object_create(dev, 883 error->ringbuffer = i915_error_object_create(dev,
681 dev_priv->render_ring.gem_object); 884 dev_priv->ring[RCS].obj);
682 885
683 /* Record buffers on the active list. */ 886 /* Record buffers on the active and pinned lists. */
684 error->active_bo = NULL; 887 error->active_bo = NULL;
685 error->active_bo_count = 0; 888 error->pinned_bo = NULL;
686 889
687 if (count) 890 error->active_bo_count = count;
891 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
892 count++;
893 error->pinned_bo_count = count - error->active_bo_count;
894
895 if (count) {
688 error->active_bo = kmalloc(sizeof(*error->active_bo)*count, 896 error->active_bo = kmalloc(sizeof(*error->active_bo)*count,
689 GFP_ATOMIC); 897 GFP_ATOMIC);
690 898 if (error->active_bo)
691 if (error->active_bo) { 899 error->pinned_bo =
692 int i = 0; 900 error->active_bo + error->active_bo_count;
693 list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) {
694 struct drm_gem_object *obj = &obj_priv->base;
695
696 error->active_bo[i].size = obj->size;
697 error->active_bo[i].name = obj->name;
698 error->active_bo[i].seqno = obj_priv->last_rendering_seqno;
699 error->active_bo[i].gtt_offset = obj_priv->gtt_offset;
700 error->active_bo[i].read_domains = obj->read_domains;
701 error->active_bo[i].write_domain = obj->write_domain;
702 error->active_bo[i].fence_reg = obj_priv->fence_reg;
703 error->active_bo[i].pinned = 0;
704 if (obj_priv->pin_count > 0)
705 error->active_bo[i].pinned = 1;
706 if (obj_priv->user_pin_count > 0)
707 error->active_bo[i].pinned = -1;
708 error->active_bo[i].tiling = obj_priv->tiling_mode;
709 error->active_bo[i].dirty = obj_priv->dirty;
710 error->active_bo[i].purgeable = obj_priv->madv != I915_MADV_WILLNEED;
711
712 if (++i == count)
713 break;
714 }
715 error->active_bo_count = i;
716 } 901 }
717 902
903 if (error->active_bo)
904 error->active_bo_count =
905 capture_bo_list(error->active_bo,
906 error->active_bo_count,
907 &dev_priv->mm.active_list);
908
909 if (error->pinned_bo)
910 error->pinned_bo_count =
911 capture_bo_list(error->pinned_bo,
912 error->pinned_bo_count,
913 &dev_priv->mm.pinned_list);
914
718 do_gettimeofday(&error->time); 915 do_gettimeofday(&error->time);
719 916
720 error->overlay = intel_overlay_capture_error_state(dev); 917 error->overlay = intel_overlay_capture_error_state(dev);
918 error->display = intel_display_capture_error_state(dev);
721 919
722 spin_lock_irqsave(&dev_priv->error_lock, flags); 920 spin_lock_irqsave(&dev_priv->error_lock, flags);
723 if (dev_priv->first_error == NULL) { 921 if (dev_priv->first_error == NULL) {
@@ -775,7 +973,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
775 printk(KERN_ERR " ACTHD: 0x%08x\n", 973 printk(KERN_ERR " ACTHD: 0x%08x\n",
776 I915_READ(ACTHD_I965)); 974 I915_READ(ACTHD_I965));
777 I915_WRITE(IPEIR_I965, ipeir); 975 I915_WRITE(IPEIR_I965, ipeir);
778 (void)I915_READ(IPEIR_I965); 976 POSTING_READ(IPEIR_I965);
779 } 977 }
780 if (eir & GM45_ERROR_PAGE_TABLE) { 978 if (eir & GM45_ERROR_PAGE_TABLE) {
781 u32 pgtbl_err = I915_READ(PGTBL_ER); 979 u32 pgtbl_err = I915_READ(PGTBL_ER);
@@ -783,7 +981,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
783 printk(KERN_ERR " PGTBL_ER: 0x%08x\n", 981 printk(KERN_ERR " PGTBL_ER: 0x%08x\n",
784 pgtbl_err); 982 pgtbl_err);
785 I915_WRITE(PGTBL_ER, pgtbl_err); 983 I915_WRITE(PGTBL_ER, pgtbl_err);
786 (void)I915_READ(PGTBL_ER); 984 POSTING_READ(PGTBL_ER);
787 } 985 }
788 } 986 }
789 987
@@ -794,7 +992,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
794 printk(KERN_ERR " PGTBL_ER: 0x%08x\n", 992 printk(KERN_ERR " PGTBL_ER: 0x%08x\n",
795 pgtbl_err); 993 pgtbl_err);
796 I915_WRITE(PGTBL_ER, pgtbl_err); 994 I915_WRITE(PGTBL_ER, pgtbl_err);
797 (void)I915_READ(PGTBL_ER); 995 POSTING_READ(PGTBL_ER);
798 } 996 }
799 } 997 }
800 998
@@ -825,7 +1023,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
825 printk(KERN_ERR " ACTHD: 0x%08x\n", 1023 printk(KERN_ERR " ACTHD: 0x%08x\n",
826 I915_READ(ACTHD)); 1024 I915_READ(ACTHD));
827 I915_WRITE(IPEIR, ipeir); 1025 I915_WRITE(IPEIR, ipeir);
828 (void)I915_READ(IPEIR); 1026 POSTING_READ(IPEIR);
829 } else { 1027 } else {
830 u32 ipeir = I915_READ(IPEIR_I965); 1028 u32 ipeir = I915_READ(IPEIR_I965);
831 1029
@@ -842,12 +1040,12 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
842 printk(KERN_ERR " ACTHD: 0x%08x\n", 1040 printk(KERN_ERR " ACTHD: 0x%08x\n",
843 I915_READ(ACTHD_I965)); 1041 I915_READ(ACTHD_I965));
844 I915_WRITE(IPEIR_I965, ipeir); 1042 I915_WRITE(IPEIR_I965, ipeir);
845 (void)I915_READ(IPEIR_I965); 1043 POSTING_READ(IPEIR_I965);
846 } 1044 }
847 } 1045 }
848 1046
849 I915_WRITE(EIR, eir); 1047 I915_WRITE(EIR, eir);
850 (void)I915_READ(EIR); 1048 POSTING_READ(EIR);
851 eir = I915_READ(EIR); 1049 eir = I915_READ(EIR);
852 if (eir) { 1050 if (eir) {
853 /* 1051 /*
@@ -870,7 +1068,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
870 * so userspace knows something bad happened (should trigger collection 1068 * so userspace knows something bad happened (should trigger collection
871 * of a ring dump etc.). 1069 * of a ring dump etc.).
872 */ 1070 */
873static void i915_handle_error(struct drm_device *dev, bool wedged) 1071void i915_handle_error(struct drm_device *dev, bool wedged)
874{ 1072{
875 struct drm_i915_private *dev_priv = dev->dev_private; 1073 struct drm_i915_private *dev_priv = dev->dev_private;
876 1074
@@ -884,11 +1082,11 @@ static void i915_handle_error(struct drm_device *dev, bool wedged)
884 /* 1082 /*
885 * Wakeup waiting processes so they don't hang 1083 * Wakeup waiting processes so they don't hang
886 */ 1084 */
887 wake_up_all(&dev_priv->render_ring.irq_queue); 1085 wake_up_all(&dev_priv->ring[RCS].irq_queue);
888 if (HAS_BSD(dev)) 1086 if (HAS_BSD(dev))
889 wake_up_all(&dev_priv->bsd_ring.irq_queue); 1087 wake_up_all(&dev_priv->ring[VCS].irq_queue);
890 if (HAS_BLT(dev)) 1088 if (HAS_BLT(dev))
891 wake_up_all(&dev_priv->blt_ring.irq_queue); 1089 wake_up_all(&dev_priv->ring[BCS].irq_queue);
892 } 1090 }
893 1091
894 queue_work(dev_priv->wq, &dev_priv->error_work); 1092 queue_work(dev_priv->wq, &dev_priv->error_work);
@@ -899,7 +1097,7 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
899 drm_i915_private_t *dev_priv = dev->dev_private; 1097 drm_i915_private_t *dev_priv = dev->dev_private;
900 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; 1098 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
901 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1099 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
902 struct drm_i915_gem_object *obj_priv; 1100 struct drm_i915_gem_object *obj;
903 struct intel_unpin_work *work; 1101 struct intel_unpin_work *work;
904 unsigned long flags; 1102 unsigned long flags;
905 bool stall_detected; 1103 bool stall_detected;
@@ -918,13 +1116,13 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
918 } 1116 }
919 1117
920 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ 1118 /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
921 obj_priv = to_intel_bo(work->pending_flip_obj); 1119 obj = work->pending_flip_obj;
922 if (INTEL_INFO(dev)->gen >= 4) { 1120 if (INTEL_INFO(dev)->gen >= 4) {
923 int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF; 1121 int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF;
924 stall_detected = I915_READ(dspsurf) == obj_priv->gtt_offset; 1122 stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
925 } else { 1123 } else {
926 int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR; 1124 int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR;
927 stall_detected = I915_READ(dspaddr) == (obj_priv->gtt_offset + 1125 stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
928 crtc->y * crtc->fb->pitch + 1126 crtc->y * crtc->fb->pitch +
929 crtc->x * crtc->fb->bits_per_pixel/8); 1127 crtc->x * crtc->fb->bits_per_pixel/8);
930 } 1128 }
@@ -970,7 +1168,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
970 * It doesn't set the bit in iir again, but it still produces 1168 * It doesn't set the bit in iir again, but it still produces
971 * interrupts (for non-MSI). 1169 * interrupts (for non-MSI).
972 */ 1170 */
973 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1171 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
974 pipea_stats = I915_READ(PIPEASTAT); 1172 pipea_stats = I915_READ(PIPEASTAT);
975 pipeb_stats = I915_READ(PIPEBSTAT); 1173 pipeb_stats = I915_READ(PIPEBSTAT);
976 1174
@@ -993,7 +1191,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
993 I915_WRITE(PIPEBSTAT, pipeb_stats); 1191 I915_WRITE(PIPEBSTAT, pipeb_stats);
994 irq_received = 1; 1192 irq_received = 1;
995 } 1193 }
996 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1194 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
997 1195
998 if (!irq_received) 1196 if (!irq_received)
999 break; 1197 break;
@@ -1026,9 +1224,9 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
1026 } 1224 }
1027 1225
1028 if (iir & I915_USER_INTERRUPT) 1226 if (iir & I915_USER_INTERRUPT)
1029 notify_ring(dev, &dev_priv->render_ring); 1227 notify_ring(dev, &dev_priv->ring[RCS]);
1030 if (HAS_BSD(dev) && (iir & I915_BSD_USER_INTERRUPT)) 1228 if (iir & I915_BSD_USER_INTERRUPT)
1031 notify_ring(dev, &dev_priv->bsd_ring); 1229 notify_ring(dev, &dev_priv->ring[VCS]);
1032 1230
1033 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) { 1231 if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
1034 intel_prepare_page_flip(dev, 0); 1232 intel_prepare_page_flip(dev, 0);
@@ -1101,12 +1299,13 @@ static int i915_emit_irq(struct drm_device * dev)
1101 if (master_priv->sarea_priv) 1299 if (master_priv->sarea_priv)
1102 master_priv->sarea_priv->last_enqueue = dev_priv->counter; 1300 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
1103 1301
1104 BEGIN_LP_RING(4); 1302 if (BEGIN_LP_RING(4) == 0) {
1105 OUT_RING(MI_STORE_DWORD_INDEX); 1303 OUT_RING(MI_STORE_DWORD_INDEX);
1106 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 1304 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1107 OUT_RING(dev_priv->counter); 1305 OUT_RING(dev_priv->counter);
1108 OUT_RING(MI_USER_INTERRUPT); 1306 OUT_RING(MI_USER_INTERRUPT);
1109 ADVANCE_LP_RING(); 1307 ADVANCE_LP_RING();
1308 }
1110 1309
1111 return dev_priv->counter; 1310 return dev_priv->counter;
1112} 1311}
@@ -1114,12 +1313,11 @@ static int i915_emit_irq(struct drm_device * dev)
1114void i915_trace_irq_get(struct drm_device *dev, u32 seqno) 1313void i915_trace_irq_get(struct drm_device *dev, u32 seqno)
1115{ 1314{
1116 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1315 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1117 struct intel_ring_buffer *render_ring = &dev_priv->render_ring; 1316 struct intel_ring_buffer *ring = LP_RING(dev_priv);
1118 1317
1119 if (dev_priv->trace_irq_seqno == 0) 1318 if (dev_priv->trace_irq_seqno == 0 &&
1120 render_ring->user_irq_get(dev, render_ring); 1319 ring->irq_get(ring))
1121 1320 dev_priv->trace_irq_seqno = seqno;
1122 dev_priv->trace_irq_seqno = seqno;
1123} 1321}
1124 1322
1125static int i915_wait_irq(struct drm_device * dev, int irq_nr) 1323static int i915_wait_irq(struct drm_device * dev, int irq_nr)
@@ -1127,7 +1325,7 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
1127 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1325 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1128 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; 1326 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1129 int ret = 0; 1327 int ret = 0;
1130 struct intel_ring_buffer *render_ring = &dev_priv->render_ring; 1328 struct intel_ring_buffer *ring = LP_RING(dev_priv);
1131 1329
1132 DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr, 1330 DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
1133 READ_BREADCRUMB(dev_priv)); 1331 READ_BREADCRUMB(dev_priv));
@@ -1141,10 +1339,12 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
1141 if (master_priv->sarea_priv) 1339 if (master_priv->sarea_priv)
1142 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 1340 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1143 1341
1144 render_ring->user_irq_get(dev, render_ring); 1342 ret = -ENODEV;
1145 DRM_WAIT_ON(ret, dev_priv->render_ring.irq_queue, 3 * DRM_HZ, 1343 if (ring->irq_get(ring)) {
1146 READ_BREADCRUMB(dev_priv) >= irq_nr); 1344 DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
1147 render_ring->user_irq_put(dev, render_ring); 1345 READ_BREADCRUMB(dev_priv) >= irq_nr);
1346 ring->irq_put(ring);
1347 }
1148 1348
1149 if (ret == -EBUSY) { 1349 if (ret == -EBUSY) {
1150 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", 1350 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
@@ -1163,7 +1363,7 @@ int i915_irq_emit(struct drm_device *dev, void *data,
1163 drm_i915_irq_emit_t *emit = data; 1363 drm_i915_irq_emit_t *emit = data;
1164 int result; 1364 int result;
1165 1365
1166 if (!dev_priv || !dev_priv->render_ring.virtual_start) { 1366 if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
1167 DRM_ERROR("called with no initialization\n"); 1367 DRM_ERROR("called with no initialization\n");
1168 return -EINVAL; 1368 return -EINVAL;
1169 } 1369 }
@@ -1209,9 +1409,9 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
1209 if (!i915_pipe_enabled(dev, pipe)) 1409 if (!i915_pipe_enabled(dev, pipe))
1210 return -EINVAL; 1410 return -EINVAL;
1211 1411
1212 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1412 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1213 if (HAS_PCH_SPLIT(dev)) 1413 if (HAS_PCH_SPLIT(dev))
1214 ironlake_enable_display_irq(dev_priv, (pipe == 0) ? 1414 ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
1215 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); 1415 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1216 else if (INTEL_INFO(dev)->gen >= 4) 1416 else if (INTEL_INFO(dev)->gen >= 4)
1217 i915_enable_pipestat(dev_priv, pipe, 1417 i915_enable_pipestat(dev_priv, pipe,
@@ -1219,7 +1419,7 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
1219 else 1419 else
1220 i915_enable_pipestat(dev_priv, pipe, 1420 i915_enable_pipestat(dev_priv, pipe,
1221 PIPE_VBLANK_INTERRUPT_ENABLE); 1421 PIPE_VBLANK_INTERRUPT_ENABLE);
1222 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1422 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1223 return 0; 1423 return 0;
1224} 1424}
1225 1425
@@ -1231,15 +1431,15 @@ void i915_disable_vblank(struct drm_device *dev, int pipe)
1231 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 1431 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1232 unsigned long irqflags; 1432 unsigned long irqflags;
1233 1433
1234 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1434 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1235 if (HAS_PCH_SPLIT(dev)) 1435 if (HAS_PCH_SPLIT(dev))
1236 ironlake_disable_display_irq(dev_priv, (pipe == 0) ? 1436 ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
1237 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); 1437 DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1238 else 1438 else
1239 i915_disable_pipestat(dev_priv, pipe, 1439 i915_disable_pipestat(dev_priv, pipe,
1240 PIPE_VBLANK_INTERRUPT_ENABLE | 1440 PIPE_VBLANK_INTERRUPT_ENABLE |
1241 PIPE_START_VBLANK_INTERRUPT_ENABLE); 1441 PIPE_START_VBLANK_INTERRUPT_ENABLE);
1242 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1442 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1243} 1443}
1244 1444
1245void i915_enable_interrupt (struct drm_device *dev) 1445void i915_enable_interrupt (struct drm_device *dev)
@@ -1306,12 +1506,50 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
1306 return -EINVAL; 1506 return -EINVAL;
1307} 1507}
1308 1508
1309static struct drm_i915_gem_request * 1509static u32
1310i915_get_tail_request(struct drm_device *dev) 1510ring_last_seqno(struct intel_ring_buffer *ring)
1311{ 1511{
1312 drm_i915_private_t *dev_priv = dev->dev_private; 1512 return list_entry(ring->request_list.prev,
1313 return list_entry(dev_priv->render_ring.request_list.prev, 1513 struct drm_i915_gem_request, list)->seqno;
1314 struct drm_i915_gem_request, list); 1514}
1515
1516static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
1517{
1518 if (list_empty(&ring->request_list) ||
1519 i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
1520 /* Issue a wake-up to catch stuck h/w. */
1521 if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
1522 DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
1523 ring->name,
1524 ring->waiting_seqno,
1525 ring->get_seqno(ring));
1526 wake_up_all(&ring->irq_queue);
1527 *err = true;
1528 }
1529 return true;
1530 }
1531 return false;
1532}
1533
1534static bool kick_ring(struct intel_ring_buffer *ring)
1535{
1536 struct drm_device *dev = ring->dev;
1537 struct drm_i915_private *dev_priv = dev->dev_private;
1538 u32 tmp = I915_READ_CTL(ring);
1539 if (tmp & RING_WAIT) {
1540 DRM_ERROR("Kicking stuck wait on %s\n",
1541 ring->name);
1542 I915_WRITE_CTL(ring, tmp);
1543 return true;
1544 }
1545 if (IS_GEN6(dev) &&
1546 (tmp & RING_WAIT_SEMAPHORE)) {
1547 DRM_ERROR("Kicking stuck semaphore on %s\n",
1548 ring->name);
1549 I915_WRITE_CTL(ring, tmp);
1550 return true;
1551 }
1552 return false;
1315} 1553}
1316 1554
1317/** 1555/**
@@ -1325,6 +1563,17 @@ void i915_hangcheck_elapsed(unsigned long data)
1325 struct drm_device *dev = (struct drm_device *)data; 1563 struct drm_device *dev = (struct drm_device *)data;
1326 drm_i915_private_t *dev_priv = dev->dev_private; 1564 drm_i915_private_t *dev_priv = dev->dev_private;
1327 uint32_t acthd, instdone, instdone1; 1565 uint32_t acthd, instdone, instdone1;
1566 bool err = false;
1567
1568 /* If all work is done then ACTHD clearly hasn't advanced. */
1569 if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
1570 i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
1571 i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
1572 dev_priv->hangcheck_count = 0;
1573 if (err)
1574 goto repeat;
1575 return;
1576 }
1328 1577
1329 if (INTEL_INFO(dev)->gen < 4) { 1578 if (INTEL_INFO(dev)->gen < 4) {
1330 acthd = I915_READ(ACTHD); 1579 acthd = I915_READ(ACTHD);
@@ -1336,38 +1585,6 @@ void i915_hangcheck_elapsed(unsigned long data)
1336 instdone1 = I915_READ(INSTDONE1); 1585 instdone1 = I915_READ(INSTDONE1);
1337 } 1586 }
1338 1587
1339 /* If all work is done then ACTHD clearly hasn't advanced. */
1340 if (list_empty(&dev_priv->render_ring.request_list) ||
1341 i915_seqno_passed(dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring),
1342 i915_get_tail_request(dev)->seqno)) {
1343 bool missed_wakeup = false;
1344
1345 dev_priv->hangcheck_count = 0;
1346
1347 /* Issue a wake-up to catch stuck h/w. */
1348 if (dev_priv->render_ring.waiting_gem_seqno &&
1349 waitqueue_active(&dev_priv->render_ring.irq_queue)) {
1350 wake_up_all(&dev_priv->render_ring.irq_queue);
1351 missed_wakeup = true;
1352 }
1353
1354 if (dev_priv->bsd_ring.waiting_gem_seqno &&
1355 waitqueue_active(&dev_priv->bsd_ring.irq_queue)) {
1356 wake_up_all(&dev_priv->bsd_ring.irq_queue);
1357 missed_wakeup = true;
1358 }
1359
1360 if (dev_priv->blt_ring.waiting_gem_seqno &&
1361 waitqueue_active(&dev_priv->blt_ring.irq_queue)) {
1362 wake_up_all(&dev_priv->blt_ring.irq_queue);
1363 missed_wakeup = true;
1364 }
1365
1366 if (missed_wakeup)
1367 DRM_ERROR("Hangcheck timer elapsed... GPU idle, missed IRQ.\n");
1368 return;
1369 }
1370
1371 if (dev_priv->last_acthd == acthd && 1588 if (dev_priv->last_acthd == acthd &&
1372 dev_priv->last_instdone == instdone && 1589 dev_priv->last_instdone == instdone &&
1373 dev_priv->last_instdone1 == instdone1) { 1590 dev_priv->last_instdone1 == instdone1) {
@@ -1380,12 +1597,17 @@ void i915_hangcheck_elapsed(unsigned long data)
1380 * and break the hang. This should work on 1597 * and break the hang. This should work on
1381 * all but the second generation chipsets. 1598 * all but the second generation chipsets.
1382 */ 1599 */
1383 u32 tmp = I915_READ(PRB0_CTL); 1600
1384 if (tmp & RING_WAIT) { 1601 if (kick_ring(&dev_priv->ring[RCS]))
1385 I915_WRITE(PRB0_CTL, tmp); 1602 goto repeat;
1386 POSTING_READ(PRB0_CTL); 1603
1387 goto out; 1604 if (HAS_BSD(dev) &&
1388 } 1605 kick_ring(&dev_priv->ring[VCS]))
1606 goto repeat;
1607
1608 if (HAS_BLT(dev) &&
1609 kick_ring(&dev_priv->ring[BCS]))
1610 goto repeat;
1389 } 1611 }
1390 1612
1391 i915_handle_error(dev, true); 1613 i915_handle_error(dev, true);
@@ -1399,7 +1621,7 @@ void i915_hangcheck_elapsed(unsigned long data)
1399 dev_priv->last_instdone1 = instdone1; 1621 dev_priv->last_instdone1 = instdone1;
1400 } 1622 }
1401 1623
1402out: 1624repeat:
1403 /* Reset timer case chip hangs without another request being added */ 1625 /* Reset timer case chip hangs without another request being added */
1404 mod_timer(&dev_priv->hangcheck_timer, 1626 mod_timer(&dev_priv->hangcheck_timer,
1405 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); 1627 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
@@ -1417,17 +1639,17 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
1417 1639
1418 I915_WRITE(DEIMR, 0xffffffff); 1640 I915_WRITE(DEIMR, 0xffffffff);
1419 I915_WRITE(DEIER, 0x0); 1641 I915_WRITE(DEIER, 0x0);
1420 (void) I915_READ(DEIER); 1642 POSTING_READ(DEIER);
1421 1643
1422 /* and GT */ 1644 /* and GT */
1423 I915_WRITE(GTIMR, 0xffffffff); 1645 I915_WRITE(GTIMR, 0xffffffff);
1424 I915_WRITE(GTIER, 0x0); 1646 I915_WRITE(GTIER, 0x0);
1425 (void) I915_READ(GTIER); 1647 POSTING_READ(GTIER);
1426 1648
1427 /* south display irq */ 1649 /* south display irq */
1428 I915_WRITE(SDEIMR, 0xffffffff); 1650 I915_WRITE(SDEIMR, 0xffffffff);
1429 I915_WRITE(SDEIER, 0x0); 1651 I915_WRITE(SDEIER, 0x0);
1430 (void) I915_READ(SDEIER); 1652 POSTING_READ(SDEIER);
1431} 1653}
1432 1654
1433static int ironlake_irq_postinstall(struct drm_device *dev) 1655static int ironlake_irq_postinstall(struct drm_device *dev)
@@ -1436,38 +1658,39 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
1436 /* enable kind of interrupts always enabled */ 1658 /* enable kind of interrupts always enabled */
1437 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | 1659 u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
1438 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE; 1660 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
1439 u32 render_mask = GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT; 1661 u32 render_irqs;
1440 u32 hotplug_mask; 1662 u32 hotplug_mask;
1441 1663
1442 dev_priv->irq_mask_reg = ~display_mask; 1664 dev_priv->irq_mask = ~display_mask;
1443 dev_priv->de_irq_enable_reg = display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK;
1444 1665
1445 /* should always can generate irq */ 1666 /* should always can generate irq */
1446 I915_WRITE(DEIIR, I915_READ(DEIIR)); 1667 I915_WRITE(DEIIR, I915_READ(DEIIR));
1447 I915_WRITE(DEIMR, dev_priv->irq_mask_reg); 1668 I915_WRITE(DEIMR, dev_priv->irq_mask);
1448 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg); 1669 I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
1449 (void) I915_READ(DEIER); 1670 POSTING_READ(DEIER);
1450 1671
1451 if (IS_GEN6(dev)) { 1672 dev_priv->gt_irq_mask = ~0;
1452 render_mask =
1453 GT_PIPE_NOTIFY |
1454 GT_GEN6_BSD_USER_INTERRUPT |
1455 GT_BLT_USER_INTERRUPT;
1456 }
1457
1458 dev_priv->gt_irq_mask_reg = ~render_mask;
1459 dev_priv->gt_irq_enable_reg = render_mask;
1460 1673
1461 I915_WRITE(GTIIR, I915_READ(GTIIR)); 1674 I915_WRITE(GTIIR, I915_READ(GTIIR));
1462 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); 1675 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1463 if (IS_GEN6(dev)) { 1676 if (IS_GEN6(dev)) {
1464 I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT); 1677 I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_USER_INTERRUPT);
1465 I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_IMR_USER_INTERRUPT); 1678 I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_USER_INTERRUPT);
1466 I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT); 1679 I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT);
1467 } 1680 }
1468 1681
1469 I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg); 1682 if (IS_GEN6(dev))
1470 (void) I915_READ(GTIER); 1683 render_irqs =
1684 GT_USER_INTERRUPT |
1685 GT_GEN6_BSD_USER_INTERRUPT |
1686 GT_BLT_USER_INTERRUPT;
1687 else
1688 render_irqs =
1689 GT_USER_INTERRUPT |
1690 GT_PIPE_NOTIFY |
1691 GT_BSD_USER_INTERRUPT;
1692 I915_WRITE(GTIER, render_irqs);
1693 POSTING_READ(GTIER);
1471 1694
1472 if (HAS_PCH_CPT(dev)) { 1695 if (HAS_PCH_CPT(dev)) {
1473 hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT | 1696 hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT |
@@ -1477,13 +1700,12 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
1477 SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG; 1700 SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG;
1478 } 1701 }
1479 1702
1480 dev_priv->pch_irq_mask_reg = ~hotplug_mask; 1703 dev_priv->pch_irq_mask = ~hotplug_mask;
1481 dev_priv->pch_irq_enable_reg = hotplug_mask;
1482 1704
1483 I915_WRITE(SDEIIR, I915_READ(SDEIIR)); 1705 I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1484 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask_reg); 1706 I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
1485 I915_WRITE(SDEIER, dev_priv->pch_irq_enable_reg); 1707 I915_WRITE(SDEIER, hotplug_mask);
1486 (void) I915_READ(SDEIER); 1708 POSTING_READ(SDEIER);
1487 1709
1488 if (IS_IRONLAKE_M(dev)) { 1710 if (IS_IRONLAKE_M(dev)) {
1489 /* Clear & enable PCU event interrupts */ 1711 /* Clear & enable PCU event interrupts */
@@ -1519,7 +1741,7 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
1519 I915_WRITE(PIPEBSTAT, 0); 1741 I915_WRITE(PIPEBSTAT, 0);
1520 I915_WRITE(IMR, 0xffffffff); 1742 I915_WRITE(IMR, 0xffffffff);
1521 I915_WRITE(IER, 0x0); 1743 I915_WRITE(IER, 0x0);
1522 (void) I915_READ(IER); 1744 POSTING_READ(IER);
1523} 1745}
1524 1746
1525/* 1747/*
@@ -1532,11 +1754,11 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1532 u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR; 1754 u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
1533 u32 error_mask; 1755 u32 error_mask;
1534 1756
1535 DRM_INIT_WAITQUEUE(&dev_priv->render_ring.irq_queue); 1757 DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
1536 if (HAS_BSD(dev)) 1758 if (HAS_BSD(dev))
1537 DRM_INIT_WAITQUEUE(&dev_priv->bsd_ring.irq_queue); 1759 DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
1538 if (HAS_BLT(dev)) 1760 if (HAS_BLT(dev))
1539 DRM_INIT_WAITQUEUE(&dev_priv->blt_ring.irq_queue); 1761 DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);
1540 1762
1541 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; 1763 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1542 1764
@@ -1544,7 +1766,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1544 return ironlake_irq_postinstall(dev); 1766 return ironlake_irq_postinstall(dev);
1545 1767
1546 /* Unmask the interrupts that we always want on. */ 1768 /* Unmask the interrupts that we always want on. */
1547 dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; 1769 dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
1548 1770
1549 dev_priv->pipestat[0] = 0; 1771 dev_priv->pipestat[0] = 0;
1550 dev_priv->pipestat[1] = 0; 1772 dev_priv->pipestat[1] = 0;
@@ -1553,7 +1775,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1553 /* Enable in IER... */ 1775 /* Enable in IER... */
1554 enable_mask |= I915_DISPLAY_PORT_INTERRUPT; 1776 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
1555 /* and unmask in IMR */ 1777 /* and unmask in IMR */
1556 dev_priv->irq_mask_reg &= ~I915_DISPLAY_PORT_INTERRUPT; 1778 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1557 } 1779 }
1558 1780
1559 /* 1781 /*
@@ -1571,9 +1793,9 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
1571 } 1793 }
1572 I915_WRITE(EMR, error_mask); 1794 I915_WRITE(EMR, error_mask);
1573 1795
1574 I915_WRITE(IMR, dev_priv->irq_mask_reg); 1796 I915_WRITE(IMR, dev_priv->irq_mask);
1575 I915_WRITE(IER, enable_mask); 1797 I915_WRITE(IER, enable_mask);
1576 (void) I915_READ(IER); 1798 POSTING_READ(IER);
1577 1799
1578 if (I915_HAS_HOTPLUG(dev)) { 1800 if (I915_HAS_HOTPLUG(dev)) {
1579 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); 1801 u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index cb8f43429279..8f948a6fbc1c 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -78,6 +78,12 @@
78#define GRDOM_RENDER (1<<2) 78#define GRDOM_RENDER (1<<2)
79#define GRDOM_MEDIA (3<<2) 79#define GRDOM_MEDIA (3<<2)
80 80
81#define GEN6_GDRST 0x941c
82#define GEN6_GRDOM_FULL (1 << 0)
83#define GEN6_GRDOM_RENDER (1 << 1)
84#define GEN6_GRDOM_MEDIA (1 << 2)
85#define GEN6_GRDOM_BLT (1 << 3)
86
81/* VGA stuff */ 87/* VGA stuff */
82 88
83#define VGA_ST01_MDA 0x3ba 89#define VGA_ST01_MDA 0x3ba
@@ -158,12 +164,23 @@
158#define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */ 164#define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */
159#define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) 165#define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1)
160#define MI_STORE_DWORD_INDEX_SHIFT 2 166#define MI_STORE_DWORD_INDEX_SHIFT 2
161#define MI_LOAD_REGISTER_IMM MI_INSTR(0x22, 1) 167/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
168 * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
169 * simply ignores the register load under certain conditions.
170 * - One can actually load arbitrary many arbitrary registers: Simply issue x
171 * address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
172 */
173#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1)
162#define MI_FLUSH_DW MI_INSTR(0x26, 2) /* for GEN6 */ 174#define MI_FLUSH_DW MI_INSTR(0x26, 2) /* for GEN6 */
163#define MI_BATCH_BUFFER MI_INSTR(0x30, 1) 175#define MI_BATCH_BUFFER MI_INSTR(0x30, 1)
164#define MI_BATCH_NON_SECURE (1) 176#define MI_BATCH_NON_SECURE (1)
165#define MI_BATCH_NON_SECURE_I965 (1<<8) 177#define MI_BATCH_NON_SECURE_I965 (1<<8)
166#define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0) 178#define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0)
179#define MI_SEMAPHORE_MBOX MI_INSTR(0x16, 1) /* gen6+ */
180#define MI_SEMAPHORE_GLOBAL_GTT (1<<22)
181#define MI_SEMAPHORE_UPDATE (1<<21)
182#define MI_SEMAPHORE_COMPARE (1<<20)
183#define MI_SEMAPHORE_REGISTER (1<<18)
167/* 184/*
168 * 3D instructions used by the kernel 185 * 3D instructions used by the kernel
169 */ 186 */
@@ -256,10 +273,6 @@
256 * Instruction and interrupt control regs 273 * Instruction and interrupt control regs
257 */ 274 */
258#define PGTBL_ER 0x02024 275#define PGTBL_ER 0x02024
259#define PRB0_TAIL 0x02030
260#define PRB0_HEAD 0x02034
261#define PRB0_START 0x02038
262#define PRB0_CTL 0x0203c
263#define RENDER_RING_BASE 0x02000 276#define RENDER_RING_BASE 0x02000
264#define BSD_RING_BASE 0x04000 277#define BSD_RING_BASE 0x04000
265#define GEN6_BSD_RING_BASE 0x12000 278#define GEN6_BSD_RING_BASE 0x12000
@@ -268,9 +281,13 @@
268#define RING_HEAD(base) ((base)+0x34) 281#define RING_HEAD(base) ((base)+0x34)
269#define RING_START(base) ((base)+0x38) 282#define RING_START(base) ((base)+0x38)
270#define RING_CTL(base) ((base)+0x3c) 283#define RING_CTL(base) ((base)+0x3c)
284#define RING_SYNC_0(base) ((base)+0x40)
285#define RING_SYNC_1(base) ((base)+0x44)
286#define RING_MAX_IDLE(base) ((base)+0x54)
271#define RING_HWS_PGA(base) ((base)+0x80) 287#define RING_HWS_PGA(base) ((base)+0x80)
272#define RING_HWS_PGA_GEN6(base) ((base)+0x2080) 288#define RING_HWS_PGA_GEN6(base) ((base)+0x2080)
273#define RING_ACTHD(base) ((base)+0x74) 289#define RING_ACTHD(base) ((base)+0x74)
290#define RING_NOPID(base) ((base)+0x94)
274#define TAIL_ADDR 0x001FFFF8 291#define TAIL_ADDR 0x001FFFF8
275#define HEAD_WRAP_COUNT 0xFFE00000 292#define HEAD_WRAP_COUNT 0xFFE00000
276#define HEAD_WRAP_ONE 0x00200000 293#define HEAD_WRAP_ONE 0x00200000
@@ -285,10 +302,17 @@
285#define RING_INVALID 0x00000000 302#define RING_INVALID 0x00000000
286#define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */ 303#define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */
287#define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ 304#define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */
305#define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */
306#if 0
307#define PRB0_TAIL 0x02030
308#define PRB0_HEAD 0x02034
309#define PRB0_START 0x02038
310#define PRB0_CTL 0x0203c
288#define PRB1_TAIL 0x02040 /* 915+ only */ 311#define PRB1_TAIL 0x02040 /* 915+ only */
289#define PRB1_HEAD 0x02044 /* 915+ only */ 312#define PRB1_HEAD 0x02044 /* 915+ only */
290#define PRB1_START 0x02048 /* 915+ only */ 313#define PRB1_START 0x02048 /* 915+ only */
291#define PRB1_CTL 0x0204c /* 915+ only */ 314#define PRB1_CTL 0x0204c /* 915+ only */
315#endif
292#define IPEIR_I965 0x02064 316#define IPEIR_I965 0x02064
293#define IPEHR_I965 0x02068 317#define IPEHR_I965 0x02068
294#define INSTDONE_I965 0x0206c 318#define INSTDONE_I965 0x0206c
@@ -305,11 +329,42 @@
305#define INSTDONE 0x02090 329#define INSTDONE 0x02090
306#define NOPID 0x02094 330#define NOPID 0x02094
307#define HWSTAM 0x02098 331#define HWSTAM 0x02098
332#define VCS_INSTDONE 0x1206C
333#define VCS_IPEIR 0x12064
334#define VCS_IPEHR 0x12068
335#define VCS_ACTHD 0x12074
336#define BCS_INSTDONE 0x2206C
337#define BCS_IPEIR 0x22064
338#define BCS_IPEHR 0x22068
339#define BCS_ACTHD 0x22074
340
341#define ERROR_GEN6 0x040a0
342
343/* GM45+ chicken bits -- debug workaround bits that may be required
344 * for various sorts of correct behavior. The top 16 bits of each are
345 * the enables for writing to the corresponding low bit.
346 */
347#define _3D_CHICKEN 0x02084
348#define _3D_CHICKEN2 0x0208c
349/* Disables pipelining of read flushes past the SF-WIZ interface.
350 * Required on all Ironlake steppings according to the B-Spec, but the
351 * particular danger of not doing so is not specified.
352 */
353# define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14)
354#define _3D_CHICKEN3 0x02090
308 355
309#define MI_MODE 0x0209c 356#define MI_MODE 0x0209c
310# define VS_TIMER_DISPATCH (1 << 6) 357# define VS_TIMER_DISPATCH (1 << 6)
311# define MI_FLUSH_ENABLE (1 << 11) 358# define MI_FLUSH_ENABLE (1 << 11)
312 359
360#define GFX_MODE 0x02520
361#define GFX_RUN_LIST_ENABLE (1<<15)
362#define GFX_TLB_INVALIDATE_ALWAYS (1<<13)
363#define GFX_SURFACE_FAULT_ENABLE (1<<12)
364#define GFX_REPLAY_MODE (1<<11)
365#define GFX_PSMI_GRANULARITY (1<<10)
366#define GFX_PPGTT_ENABLE (1<<9)
367
313#define SCPD0 0x0209c /* 915+ only */ 368#define SCPD0 0x0209c /* 915+ only */
314#define IER 0x020a0 369#define IER 0x020a0
315#define IIR 0x020a4 370#define IIR 0x020a4
@@ -461,7 +516,7 @@
461#define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3) 516#define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3)
462 517
463#define GEN6_BSD_IMR 0x120a8 518#define GEN6_BSD_IMR 0x120a8
464#define GEN6_BSD_IMR_USER_INTERRUPT (1 << 12) 519#define GEN6_BSD_USER_INTERRUPT (1 << 12)
465 520
466#define GEN6_BSD_RNCID 0x12198 521#define GEN6_BSD_RNCID 0x12198
467 522
@@ -541,6 +596,18 @@
541 596
542#define ILK_DISPLAY_CHICKEN1 0x42000 597#define ILK_DISPLAY_CHICKEN1 0x42000
543#define ILK_FBCQ_DIS (1<<22) 598#define ILK_FBCQ_DIS (1<<22)
599#define ILK_PABSTRETCH_DIS (1<<21)
600
601
602/*
603 * Framebuffer compression for Sandybridge
604 *
605 * The following two registers are of type GTTMMADR
606 */
607#define SNB_DPFC_CTL_SA 0x100100
608#define SNB_CPU_FENCE_ENABLE (1<<29)
609#define DPFC_CPU_FENCE_OFFSET 0x100104
610
544 611
545/* 612/*
546 * GPIO regs 613 * GPIO regs
@@ -900,6 +967,8 @@
900 */ 967 */
901#define MCHBAR_MIRROR_BASE 0x10000 968#define MCHBAR_MIRROR_BASE 0x10000
902 969
970#define MCHBAR_MIRROR_BASE_SNB 0x140000
971
903/** 915-945 and GM965 MCH register controlling DRAM channel access */ 972/** 915-945 and GM965 MCH register controlling DRAM channel access */
904#define DCC 0x10200 973#define DCC 0x10200
905#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) 974#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0)
@@ -1119,6 +1188,10 @@
1119#define DDRMPLL1 0X12c20 1188#define DDRMPLL1 0X12c20
1120#define PEG_BAND_GAP_DATA 0x14d68 1189#define PEG_BAND_GAP_DATA 0x14d68
1121 1190
1191#define GEN6_GT_PERF_STATUS 0x145948
1192#define GEN6_RP_STATE_LIMITS 0x145994
1193#define GEN6_RP_STATE_CAP 0x145998
1194
1122/* 1195/*
1123 * Logical Context regs 1196 * Logical Context regs
1124 */ 1197 */
@@ -1168,7 +1241,6 @@
1168#define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B) 1241#define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B)
1169#define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B) 1242#define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B)
1170#define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B) 1243#define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B)
1171#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
1172#define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B) 1244#define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B)
1173 1245
1174/* VGA port control */ 1246/* VGA port control */
@@ -2182,8 +2254,10 @@
2182#define PIPE_6BPC (2 << 5) 2254#define PIPE_6BPC (2 << 5)
2183#define PIPE_12BPC (3 << 5) 2255#define PIPE_12BPC (3 << 5)
2184 2256
2257#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC)
2185#define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF) 2258#define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF)
2186#define PIPEDSL(pipe) _PIPE(pipe, PIPEADSL, PIPEBDSL) 2259#define PIPEDSL(pipe) _PIPE(pipe, PIPEADSL, PIPEBDSL)
2260#define PIPEFRAMEPIXEL(pipe) _PIPE(pipe, PIPEAFRAMEPIXEL, PIPEBFRAMEPIXEL)
2187 2261
2188#define DSPARB 0x70030 2262#define DSPARB 0x70030
2189#define DSPARB_CSTART_MASK (0x7f << 7) 2263#define DSPARB_CSTART_MASK (0x7f << 7)
@@ -2291,6 +2365,40 @@
2291 2365
2292#define ILK_FIFO_LINE_SIZE 64 2366#define ILK_FIFO_LINE_SIZE 64
2293 2367
2368/* define the WM info on Sandybridge */
2369#define SNB_DISPLAY_FIFO 128
2370#define SNB_DISPLAY_MAXWM 0x7f /* bit 16:22 */
2371#define SNB_DISPLAY_DFTWM 8
2372#define SNB_CURSOR_FIFO 32
2373#define SNB_CURSOR_MAXWM 0x1f /* bit 4:0 */
2374#define SNB_CURSOR_DFTWM 8
2375
2376#define SNB_DISPLAY_SR_FIFO 512
2377#define SNB_DISPLAY_MAX_SRWM 0x1ff /* bit 16:8 */
2378#define SNB_DISPLAY_DFT_SRWM 0x3f
2379#define SNB_CURSOR_SR_FIFO 64
2380#define SNB_CURSOR_MAX_SRWM 0x3f /* bit 5:0 */
2381#define SNB_CURSOR_DFT_SRWM 8
2382
2383#define SNB_FBC_MAX_SRWM 0xf /* bit 23:20 */
2384
2385#define SNB_FIFO_LINE_SIZE 64
2386
2387
2388/* the address where we get all kinds of latency value */
2389#define SSKPD 0x5d10
2390#define SSKPD_WM_MASK 0x3f
2391#define SSKPD_WM0_SHIFT 0
2392#define SSKPD_WM1_SHIFT 8
2393#define SSKPD_WM2_SHIFT 16
2394#define SSKPD_WM3_SHIFT 24
2395
2396#define SNB_LATENCY(shift) (I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD) >> (shift) & SSKPD_WM_MASK)
2397#define SNB_READ_WM0_LATENCY() SNB_LATENCY(SSKPD_WM0_SHIFT)
2398#define SNB_READ_WM1_LATENCY() SNB_LATENCY(SSKPD_WM1_SHIFT)
2399#define SNB_READ_WM2_LATENCY() SNB_LATENCY(SSKPD_WM2_SHIFT)
2400#define SNB_READ_WM3_LATENCY() SNB_LATENCY(SSKPD_WM3_SHIFT)
2401
2294/* 2402/*
2295 * The two pipe frame counter registers are not synchronized, so 2403 * The two pipe frame counter registers are not synchronized, so
2296 * reading a stable value is somewhat tricky. The following code 2404 * reading a stable value is somewhat tricky. The following code
@@ -2351,6 +2459,10 @@
2351#define CURBBASE 0x700c4 2459#define CURBBASE 0x700c4
2352#define CURBPOS 0x700c8 2460#define CURBPOS 0x700c8
2353 2461
2462#define CURCNTR(pipe) _PIPE(pipe, CURACNTR, CURBCNTR)
2463#define CURBASE(pipe) _PIPE(pipe, CURABASE, CURBBASE)
2464#define CURPOS(pipe) _PIPE(pipe, CURAPOS, CURBPOS)
2465
2354/* Display A control */ 2466/* Display A control */
2355#define DSPACNTR 0x70180 2467#define DSPACNTR 0x70180
2356#define DISPLAY_PLANE_ENABLE (1<<31) 2468#define DISPLAY_PLANE_ENABLE (1<<31)
@@ -2589,6 +2701,8 @@
2589#define GTIER 0x4401c 2701#define GTIER 0x4401c
2590 2702
2591#define ILK_DISPLAY_CHICKEN2 0x42004 2703#define ILK_DISPLAY_CHICKEN2 0x42004
2704/* Required on all Ironlake and Sandybridge according to the B-Spec. */
2705#define ILK_ELPIN_409_SELECT (1 << 25)
2592#define ILK_DPARB_GATE (1<<22) 2706#define ILK_DPARB_GATE (1<<22)
2593#define ILK_VSDPFD_FULL (1<<21) 2707#define ILK_VSDPFD_FULL (1<<21)
2594#define ILK_DISPLAY_CHICKEN_FUSES 0x42014 2708#define ILK_DISPLAY_CHICKEN_FUSES 0x42014
@@ -2600,6 +2714,8 @@
2600#define ILK_DESKTOP (1<<23) 2714#define ILK_DESKTOP (1<<23)
2601#define ILK_DSPCLK_GATE 0x42020 2715#define ILK_DSPCLK_GATE 0x42020
2602#define ILK_DPARB_CLK_GATE (1<<5) 2716#define ILK_DPARB_CLK_GATE (1<<5)
2717#define ILK_DPFD_CLK_GATE (1<<7)
2718
2603/* According to spec this bit 7/8/9 of 0x42020 should be set to enable FBC */ 2719/* According to spec this bit 7/8/9 of 0x42020 should be set to enable FBC */
2604#define ILK_CLK_FBC (1<<7) 2720#define ILK_CLK_FBC (1<<7)
2605#define ILK_DPFC_DIS1 (1<<8) 2721#define ILK_DPFC_DIS1 (1<<8)
@@ -2679,6 +2795,7 @@
2679#define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B) 2795#define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B)
2680 2796
2681#define PCH_FPA0 0xc6040 2797#define PCH_FPA0 0xc6040
2798#define FP_CB_TUNE (0x3<<22)
2682#define PCH_FPA1 0xc6044 2799#define PCH_FPA1 0xc6044
2683#define PCH_FPB0 0xc6048 2800#define PCH_FPB0 0xc6048
2684#define PCH_FPB1 0xc604c 2801#define PCH_FPB1 0xc604c
@@ -3063,4 +3180,66 @@
3063#define EDP_LINK_TRAIN_800MV_0DB_SNB_B (0x38<<22) 3180#define EDP_LINK_TRAIN_800MV_0DB_SNB_B (0x38<<22)
3064#define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f<<22) 3181#define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f<<22)
3065 3182
3183#define FORCEWAKE 0xA18C
3184#define FORCEWAKE_ACK 0x130090
3185
3186#define GEN6_RPNSWREQ 0xA008
3187#define GEN6_TURBO_DISABLE (1<<31)
3188#define GEN6_FREQUENCY(x) ((x)<<25)
3189#define GEN6_OFFSET(x) ((x)<<19)
3190#define GEN6_AGGRESSIVE_TURBO (0<<15)
3191#define GEN6_RC_VIDEO_FREQ 0xA00C
3192#define GEN6_RC_CONTROL 0xA090
3193#define GEN6_RC_CTL_RC6pp_ENABLE (1<<16)
3194#define GEN6_RC_CTL_RC6p_ENABLE (1<<17)
3195#define GEN6_RC_CTL_RC6_ENABLE (1<<18)
3196#define GEN6_RC_CTL_RC1e_ENABLE (1<<20)
3197#define GEN6_RC_CTL_RC7_ENABLE (1<<22)
3198#define GEN6_RC_CTL_EI_MODE(x) ((x)<<27)
3199#define GEN6_RC_CTL_HW_ENABLE (1<<31)
3200#define GEN6_RP_DOWN_TIMEOUT 0xA010
3201#define GEN6_RP_INTERRUPT_LIMITS 0xA014
3202#define GEN6_RPSTAT1 0xA01C
3203#define GEN6_RP_CONTROL 0xA024
3204#define GEN6_RP_MEDIA_TURBO (1<<11)
3205#define GEN6_RP_USE_NORMAL_FREQ (1<<9)
3206#define GEN6_RP_MEDIA_IS_GFX (1<<8)
3207#define GEN6_RP_ENABLE (1<<7)
3208#define GEN6_RP_UP_BUSY_MAX (0x2<<3)
3209#define GEN6_RP_DOWN_BUSY_MIN (0x2<<0)
3210#define GEN6_RP_UP_THRESHOLD 0xA02C
3211#define GEN6_RP_DOWN_THRESHOLD 0xA030
3212#define GEN6_RP_UP_EI 0xA068
3213#define GEN6_RP_DOWN_EI 0xA06C
3214#define GEN6_RP_IDLE_HYSTERSIS 0xA070
3215#define GEN6_RC_STATE 0xA094
3216#define GEN6_RC1_WAKE_RATE_LIMIT 0xA098
3217#define GEN6_RC6_WAKE_RATE_LIMIT 0xA09C
3218#define GEN6_RC6pp_WAKE_RATE_LIMIT 0xA0A0
3219#define GEN6_RC_EVALUATION_INTERVAL 0xA0A8
3220#define GEN6_RC_IDLE_HYSTERSIS 0xA0AC
3221#define GEN6_RC_SLEEP 0xA0B0
3222#define GEN6_RC1e_THRESHOLD 0xA0B4
3223#define GEN6_RC6_THRESHOLD 0xA0B8
3224#define GEN6_RC6p_THRESHOLD 0xA0BC
3225#define GEN6_RC6pp_THRESHOLD 0xA0C0
3226#define GEN6_PMINTRMSK 0xA168
3227
3228#define GEN6_PMISR 0x44020
3229#define GEN6_PMIMR 0x44024
3230#define GEN6_PMIIR 0x44028
3231#define GEN6_PMIER 0x4402C
3232#define GEN6_PM_MBOX_EVENT (1<<25)
3233#define GEN6_PM_THERMAL_EVENT (1<<24)
3234#define GEN6_PM_RP_DOWN_TIMEOUT (1<<6)
3235#define GEN6_PM_RP_UP_THRESHOLD (1<<5)
3236#define GEN6_PM_RP_DOWN_THRESHOLD (1<<4)
3237#define GEN6_PM_RP_UP_EI_EXPIRED (1<<2)
3238#define GEN6_PM_RP_DOWN_EI_EXPIRED (1<<1)
3239
3240#define GEN6_PCODE_MAILBOX 0x138124
3241#define GEN6_PCODE_READY (1<<31)
3242#define GEN6_PCODE_WRITE_MIN_FREQ_TABLE 0x9
3243#define GEN6_PCODE_DATA 0x138128
3244
3066#endif /* _I915_REG_H_ */ 3245#endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 42729d25da58..410772466fa7 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -235,6 +235,7 @@ static void i915_restore_vga(struct drm_device *dev)
235static void i915_save_modeset_reg(struct drm_device *dev) 235static void i915_save_modeset_reg(struct drm_device *dev)
236{ 236{
237 struct drm_i915_private *dev_priv = dev->dev_private; 237 struct drm_i915_private *dev_priv = dev->dev_private;
238 int i;
238 239
239 if (drm_core_check_feature(dev, DRIVER_MODESET)) 240 if (drm_core_check_feature(dev, DRIVER_MODESET))
240 return; 241 return;
@@ -367,6 +368,28 @@ static void i915_save_modeset_reg(struct drm_device *dev)
367 } 368 }
368 i915_save_palette(dev, PIPE_B); 369 i915_save_palette(dev, PIPE_B);
369 dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT); 370 dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
371
372 /* Fences */
373 switch (INTEL_INFO(dev)->gen) {
374 case 6:
375 for (i = 0; i < 16; i++)
376 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
377 break;
378 case 5:
379 case 4:
380 for (i = 0; i < 16; i++)
381 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
382 break;
383 case 3:
384 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
385 for (i = 0; i < 8; i++)
386 dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
387 case 2:
388 for (i = 0; i < 8; i++)
389 dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
390 break;
391 }
392
370 return; 393 return;
371} 394}
372 395
@@ -375,10 +398,33 @@ static void i915_restore_modeset_reg(struct drm_device *dev)
375 struct drm_i915_private *dev_priv = dev->dev_private; 398 struct drm_i915_private *dev_priv = dev->dev_private;
376 int dpll_a_reg, fpa0_reg, fpa1_reg; 399 int dpll_a_reg, fpa0_reg, fpa1_reg;
377 int dpll_b_reg, fpb0_reg, fpb1_reg; 400 int dpll_b_reg, fpb0_reg, fpb1_reg;
401 int i;
378 402
379 if (drm_core_check_feature(dev, DRIVER_MODESET)) 403 if (drm_core_check_feature(dev, DRIVER_MODESET))
380 return; 404 return;
381 405
406 /* Fences */
407 switch (INTEL_INFO(dev)->gen) {
408 case 6:
409 for (i = 0; i < 16; i++)
410 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
411 break;
412 case 5:
413 case 4:
414 for (i = 0; i < 16; i++)
415 I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
416 break;
417 case 3:
418 case 2:
419 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
420 for (i = 0; i < 8; i++)
421 I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
422 for (i = 0; i < 8; i++)
423 I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
424 break;
425 }
426
427
382 if (HAS_PCH_SPLIT(dev)) { 428 if (HAS_PCH_SPLIT(dev)) {
383 dpll_a_reg = PCH_DPLL_A; 429 dpll_a_reg = PCH_DPLL_A;
384 dpll_b_reg = PCH_DPLL_B; 430 dpll_b_reg = PCH_DPLL_B;
@@ -771,8 +817,14 @@ int i915_save_state(struct drm_device *dev)
771 dev_priv->saveIMR = I915_READ(IMR); 817 dev_priv->saveIMR = I915_READ(IMR);
772 } 818 }
773 819
774 if (HAS_PCH_SPLIT(dev)) 820 if (IS_IRONLAKE_M(dev))
775 ironlake_disable_drps(dev); 821 ironlake_disable_drps(dev);
822 if (IS_GEN6(dev))
823 gen6_disable_rps(dev);
824
825 /* XXX disabling the clock gating breaks suspend on gm45
826 intel_disable_clock_gating(dev);
827 */
776 828
777 /* Cache mode state */ 829 /* Cache mode state */
778 dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); 830 dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
@@ -788,28 +840,6 @@ int i915_save_state(struct drm_device *dev)
788 for (i = 0; i < 3; i++) 840 for (i = 0; i < 3; i++)
789 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2)); 841 dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));
790 842
791 /* Fences */
792 switch (INTEL_INFO(dev)->gen) {
793 case 6:
794 for (i = 0; i < 16; i++)
795 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
796 break;
797 case 5:
798 case 4:
799 for (i = 0; i < 16; i++)
800 dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
801 break;
802 case 3:
803 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
804 for (i = 0; i < 8; i++)
805 dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
806 case 2:
807 for (i = 0; i < 8; i++)
808 dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
809 break;
810
811 }
812
813 return 0; 843 return 0;
814} 844}
815 845
@@ -823,27 +853,6 @@ int i915_restore_state(struct drm_device *dev)
823 /* Hardware status page */ 853 /* Hardware status page */
824 I915_WRITE(HWS_PGA, dev_priv->saveHWS); 854 I915_WRITE(HWS_PGA, dev_priv->saveHWS);
825 855
826 /* Fences */
827 switch (INTEL_INFO(dev)->gen) {
828 case 6:
829 for (i = 0; i < 16; i++)
830 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]);
831 break;
832 case 5:
833 case 4:
834 for (i = 0; i < 16; i++)
835 I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
836 break;
837 case 3:
838 case 2:
839 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
840 for (i = 0; i < 8; i++)
841 I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
842 for (i = 0; i < 8; i++)
843 I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
844 break;
845 }
846
847 i915_restore_display(dev); 856 i915_restore_display(dev);
848 857
849 /* Interrupt state */ 858 /* Interrupt state */
@@ -860,13 +869,16 @@ int i915_restore_state(struct drm_device *dev)
860 } 869 }
861 870
862 /* Clock gating state */ 871 /* Clock gating state */
863 intel_init_clock_gating(dev); 872 intel_enable_clock_gating(dev);
864 873
865 if (HAS_PCH_SPLIT(dev)) { 874 if (IS_IRONLAKE_M(dev)) {
866 ironlake_enable_drps(dev); 875 ironlake_enable_drps(dev);
867 intel_init_emon(dev); 876 intel_init_emon(dev);
868 } 877 }
869 878
879 if (IS_GEN6(dev))
880 gen6_enable_rps(dev_priv);
881
870 /* Cache mode state */ 882 /* Cache mode state */
871 I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000); 883 I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);
872 884
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index fea97a21cc14..7f0fc3ed61aa 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -6,6 +6,7 @@
6#include <linux/tracepoint.h> 6#include <linux/tracepoint.h>
7 7
8#include <drm/drmP.h> 8#include <drm/drmP.h>
9#include "i915_drv.h"
9 10
10#undef TRACE_SYSTEM 11#undef TRACE_SYSTEM
11#define TRACE_SYSTEM i915 12#define TRACE_SYSTEM i915
@@ -16,18 +17,18 @@
16 17
17TRACE_EVENT(i915_gem_object_create, 18TRACE_EVENT(i915_gem_object_create,
18 19
19 TP_PROTO(struct drm_gem_object *obj), 20 TP_PROTO(struct drm_i915_gem_object *obj),
20 21
21 TP_ARGS(obj), 22 TP_ARGS(obj),
22 23
23 TP_STRUCT__entry( 24 TP_STRUCT__entry(
24 __field(struct drm_gem_object *, obj) 25 __field(struct drm_i915_gem_object *, obj)
25 __field(u32, size) 26 __field(u32, size)
26 ), 27 ),
27 28
28 TP_fast_assign( 29 TP_fast_assign(
29 __entry->obj = obj; 30 __entry->obj = obj;
30 __entry->size = obj->size; 31 __entry->size = obj->base.size;
31 ), 32 ),
32 33
33 TP_printk("obj=%p, size=%u", __entry->obj, __entry->size) 34 TP_printk("obj=%p, size=%u", __entry->obj, __entry->size)
@@ -35,40 +36,43 @@ TRACE_EVENT(i915_gem_object_create,
35 36
36TRACE_EVENT(i915_gem_object_bind, 37TRACE_EVENT(i915_gem_object_bind,
37 38
38 TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset), 39 TP_PROTO(struct drm_i915_gem_object *obj, u32 gtt_offset, bool mappable),
39 40
40 TP_ARGS(obj, gtt_offset), 41 TP_ARGS(obj, gtt_offset, mappable),
41 42
42 TP_STRUCT__entry( 43 TP_STRUCT__entry(
43 __field(struct drm_gem_object *, obj) 44 __field(struct drm_i915_gem_object *, obj)
44 __field(u32, gtt_offset) 45 __field(u32, gtt_offset)
46 __field(bool, mappable)
45 ), 47 ),
46 48
47 TP_fast_assign( 49 TP_fast_assign(
48 __entry->obj = obj; 50 __entry->obj = obj;
49 __entry->gtt_offset = gtt_offset; 51 __entry->gtt_offset = gtt_offset;
52 __entry->mappable = mappable;
50 ), 53 ),
51 54
52 TP_printk("obj=%p, gtt_offset=%08x", 55 TP_printk("obj=%p, gtt_offset=%08x%s",
53 __entry->obj, __entry->gtt_offset) 56 __entry->obj, __entry->gtt_offset,
57 __entry->mappable ? ", mappable" : "")
54); 58);
55 59
56TRACE_EVENT(i915_gem_object_change_domain, 60TRACE_EVENT(i915_gem_object_change_domain,
57 61
58 TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain), 62 TP_PROTO(struct drm_i915_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain),
59 63
60 TP_ARGS(obj, old_read_domains, old_write_domain), 64 TP_ARGS(obj, old_read_domains, old_write_domain),
61 65
62 TP_STRUCT__entry( 66 TP_STRUCT__entry(
63 __field(struct drm_gem_object *, obj) 67 __field(struct drm_i915_gem_object *, obj)
64 __field(u32, read_domains) 68 __field(u32, read_domains)
65 __field(u32, write_domain) 69 __field(u32, write_domain)
66 ), 70 ),
67 71
68 TP_fast_assign( 72 TP_fast_assign(
69 __entry->obj = obj; 73 __entry->obj = obj;
70 __entry->read_domains = obj->read_domains | (old_read_domains << 16); 74 __entry->read_domains = obj->base.read_domains | (old_read_domains << 16);
71 __entry->write_domain = obj->write_domain | (old_write_domain << 16); 75 __entry->write_domain = obj->base.write_domain | (old_write_domain << 16);
72 ), 76 ),
73 77
74 TP_printk("obj=%p, read=%04x, write=%04x", 78 TP_printk("obj=%p, read=%04x, write=%04x",
@@ -76,36 +80,14 @@ TRACE_EVENT(i915_gem_object_change_domain,
76 __entry->read_domains, __entry->write_domain) 80 __entry->read_domains, __entry->write_domain)
77); 81);
78 82
79TRACE_EVENT(i915_gem_object_get_fence,
80
81 TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode),
82
83 TP_ARGS(obj, fence, tiling_mode),
84
85 TP_STRUCT__entry(
86 __field(struct drm_gem_object *, obj)
87 __field(int, fence)
88 __field(int, tiling_mode)
89 ),
90
91 TP_fast_assign(
92 __entry->obj = obj;
93 __entry->fence = fence;
94 __entry->tiling_mode = tiling_mode;
95 ),
96
97 TP_printk("obj=%p, fence=%d, tiling=%d",
98 __entry->obj, __entry->fence, __entry->tiling_mode)
99);
100
101DECLARE_EVENT_CLASS(i915_gem_object, 83DECLARE_EVENT_CLASS(i915_gem_object,
102 84
103 TP_PROTO(struct drm_gem_object *obj), 85 TP_PROTO(struct drm_i915_gem_object *obj),
104 86
105 TP_ARGS(obj), 87 TP_ARGS(obj),
106 88
107 TP_STRUCT__entry( 89 TP_STRUCT__entry(
108 __field(struct drm_gem_object *, obj) 90 __field(struct drm_i915_gem_object *, obj)
109 ), 91 ),
110 92
111 TP_fast_assign( 93 TP_fast_assign(
@@ -117,21 +99,21 @@ DECLARE_EVENT_CLASS(i915_gem_object,
117 99
118DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush, 100DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush,
119 101
120 TP_PROTO(struct drm_gem_object *obj), 102 TP_PROTO(struct drm_i915_gem_object *obj),
121 103
122 TP_ARGS(obj) 104 TP_ARGS(obj)
123); 105);
124 106
125DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind, 107DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind,
126 108
127 TP_PROTO(struct drm_gem_object *obj), 109 TP_PROTO(struct drm_i915_gem_object *obj),
128 110
129 TP_ARGS(obj) 111 TP_ARGS(obj)
130); 112);
131 113
132DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy, 114DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
133 115
134 TP_PROTO(struct drm_gem_object *obj), 116 TP_PROTO(struct drm_i915_gem_object *obj),
135 117
136 TP_ARGS(obj) 118 TP_ARGS(obj)
137); 119);
@@ -263,13 +245,13 @@ DEFINE_EVENT(i915_ring, i915_ring_wait_end,
263); 245);
264 246
265TRACE_EVENT(i915_flip_request, 247TRACE_EVENT(i915_flip_request,
266 TP_PROTO(int plane, struct drm_gem_object *obj), 248 TP_PROTO(int plane, struct drm_i915_gem_object *obj),
267 249
268 TP_ARGS(plane, obj), 250 TP_ARGS(plane, obj),
269 251
270 TP_STRUCT__entry( 252 TP_STRUCT__entry(
271 __field(int, plane) 253 __field(int, plane)
272 __field(struct drm_gem_object *, obj) 254 __field(struct drm_i915_gem_object *, obj)
273 ), 255 ),
274 256
275 TP_fast_assign( 257 TP_fast_assign(
@@ -281,13 +263,13 @@ TRACE_EVENT(i915_flip_request,
281); 263);
282 264
283TRACE_EVENT(i915_flip_complete, 265TRACE_EVENT(i915_flip_complete,
284 TP_PROTO(int plane, struct drm_gem_object *obj), 266 TP_PROTO(int plane, struct drm_i915_gem_object *obj),
285 267
286 TP_ARGS(plane, obj), 268 TP_ARGS(plane, obj),
287 269
288 TP_STRUCT__entry( 270 TP_STRUCT__entry(
289 __field(int, plane) 271 __field(int, plane)
290 __field(struct drm_gem_object *, obj) 272 __field(struct drm_i915_gem_object *, obj)
291 ), 273 ),
292 274
293 TP_fast_assign( 275 TP_fast_assign(
@@ -298,6 +280,29 @@ TRACE_EVENT(i915_flip_complete,
298 TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj) 280 TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj)
299); 281);
300 282
283TRACE_EVENT(i915_reg_rw,
284 TP_PROTO(int cmd, uint32_t reg, uint64_t val, int len),
285
286 TP_ARGS(cmd, reg, val, len),
287
288 TP_STRUCT__entry(
289 __field(int, cmd)
290 __field(uint32_t, reg)
291 __field(uint64_t, val)
292 __field(int, len)
293 ),
294
295 TP_fast_assign(
296 __entry->cmd = cmd;
297 __entry->reg = reg;
298 __entry->val = (uint64_t)val;
299 __entry->len = len;
300 ),
301
302 TP_printk("cmd=%c, reg=0x%x, val=0x%llx, len=%d",
303 __entry->cmd, __entry->reg, __entry->val, __entry->len)
304);
305
301#endif /* _I915_TRACE_H_ */ 306#endif /* _I915_TRACE_H_ */
302 307
303/* This part must be outside protection */ 308/* This part must be outside protection */
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index fca523288aca..0abe79fb6385 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -642,26 +642,23 @@ static const intel_limit_t intel_limits_ironlake_display_port = {
642 .find_pll = intel_find_pll_ironlake_dp, 642 .find_pll = intel_find_pll_ironlake_dp,
643}; 643};
644 644
645static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc) 645static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
646 int refclk)
646{ 647{
647 struct drm_device *dev = crtc->dev; 648 struct drm_device *dev = crtc->dev;
648 struct drm_i915_private *dev_priv = dev->dev_private; 649 struct drm_i915_private *dev_priv = dev->dev_private;
649 const intel_limit_t *limit; 650 const intel_limit_t *limit;
650 int refclk = 120;
651 651
652 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { 652 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
653 if (dev_priv->lvds_use_ssc && dev_priv->lvds_ssc_freq == 100)
654 refclk = 100;
655
656 if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == 653 if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
657 LVDS_CLKB_POWER_UP) { 654 LVDS_CLKB_POWER_UP) {
658 /* LVDS dual channel */ 655 /* LVDS dual channel */
659 if (refclk == 100) 656 if (refclk == 100000)
660 limit = &intel_limits_ironlake_dual_lvds_100m; 657 limit = &intel_limits_ironlake_dual_lvds_100m;
661 else 658 else
662 limit = &intel_limits_ironlake_dual_lvds; 659 limit = &intel_limits_ironlake_dual_lvds;
663 } else { 660 } else {
664 if (refclk == 100) 661 if (refclk == 100000)
665 limit = &intel_limits_ironlake_single_lvds_100m; 662 limit = &intel_limits_ironlake_single_lvds_100m;
666 else 663 else
667 limit = &intel_limits_ironlake_single_lvds; 664 limit = &intel_limits_ironlake_single_lvds;
@@ -702,13 +699,13 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
702 return limit; 699 return limit;
703} 700}
704 701
705static const intel_limit_t *intel_limit(struct drm_crtc *crtc) 702static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk)
706{ 703{
707 struct drm_device *dev = crtc->dev; 704 struct drm_device *dev = crtc->dev;
708 const intel_limit_t *limit; 705 const intel_limit_t *limit;
709 706
710 if (HAS_PCH_SPLIT(dev)) 707 if (HAS_PCH_SPLIT(dev))
711 limit = intel_ironlake_limit(crtc); 708 limit = intel_ironlake_limit(crtc, refclk);
712 else if (IS_G4X(dev)) { 709 else if (IS_G4X(dev)) {
713 limit = intel_g4x_limit(crtc); 710 limit = intel_g4x_limit(crtc);
714 } else if (IS_PINEVIEW(dev)) { 711 } else if (IS_PINEVIEW(dev)) {
@@ -773,11 +770,10 @@ bool intel_pipe_has_type(struct drm_crtc *crtc, int type)
773 * the given connectors. 770 * the given connectors.
774 */ 771 */
775 772
776static bool intel_PLL_is_valid(struct drm_crtc *crtc, intel_clock_t *clock) 773static bool intel_PLL_is_valid(struct drm_device *dev,
774 const intel_limit_t *limit,
775 const intel_clock_t *clock)
777{ 776{
778 const intel_limit_t *limit = intel_limit (crtc);
779 struct drm_device *dev = crtc->dev;
780
781 if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) 777 if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1)
782 INTELPllInvalid ("p1 out of range\n"); 778 INTELPllInvalid ("p1 out of range\n");
783 if (clock->p < limit->p.min || limit->p.max < clock->p) 779 if (clock->p < limit->p.min || limit->p.max < clock->p)
@@ -849,8 +845,8 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
849 int this_err; 845 int this_err;
850 846
851 intel_clock(dev, refclk, &clock); 847 intel_clock(dev, refclk, &clock);
852 848 if (!intel_PLL_is_valid(dev, limit,
853 if (!intel_PLL_is_valid(crtc, &clock)) 849 &clock))
854 continue; 850 continue;
855 851
856 this_err = abs(clock.dot - target); 852 this_err = abs(clock.dot - target);
@@ -912,9 +908,11 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
912 int this_err; 908 int this_err;
913 909
914 intel_clock(dev, refclk, &clock); 910 intel_clock(dev, refclk, &clock);
915 if (!intel_PLL_is_valid(crtc, &clock)) 911 if (!intel_PLL_is_valid(dev, limit,
912 &clock))
916 continue; 913 continue;
917 this_err = abs(clock.dot - target) ; 914
915 this_err = abs(clock.dot - target);
918 if (this_err < err_most) { 916 if (this_err < err_most) {
919 *best_clock = clock; 917 *best_clock = clock;
920 err_most = this_err; 918 err_most = this_err;
@@ -1066,13 +1064,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1066 struct drm_i915_private *dev_priv = dev->dev_private; 1064 struct drm_i915_private *dev_priv = dev->dev_private;
1067 struct drm_framebuffer *fb = crtc->fb; 1065 struct drm_framebuffer *fb = crtc->fb;
1068 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1066 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1069 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1067 struct drm_i915_gem_object *obj = intel_fb->obj;
1070 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1068 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1071 int plane, i; 1069 int plane, i;
1072 u32 fbc_ctl, fbc_ctl2; 1070 u32 fbc_ctl, fbc_ctl2;
1073 1071
1074 if (fb->pitch == dev_priv->cfb_pitch && 1072 if (fb->pitch == dev_priv->cfb_pitch &&
1075 obj_priv->fence_reg == dev_priv->cfb_fence && 1073 obj->fence_reg == dev_priv->cfb_fence &&
1076 intel_crtc->plane == dev_priv->cfb_plane && 1074 intel_crtc->plane == dev_priv->cfb_plane &&
1077 I915_READ(FBC_CONTROL) & FBC_CTL_EN) 1075 I915_READ(FBC_CONTROL) & FBC_CTL_EN)
1078 return; 1076 return;
@@ -1086,7 +1084,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1086 1084
1087 /* FBC_CTL wants 64B units */ 1085 /* FBC_CTL wants 64B units */
1088 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1086 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1089 dev_priv->cfb_fence = obj_priv->fence_reg; 1087 dev_priv->cfb_fence = obj->fence_reg;
1090 dev_priv->cfb_plane = intel_crtc->plane; 1088 dev_priv->cfb_plane = intel_crtc->plane;
1091 plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; 1089 plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
1092 1090
@@ -1096,7 +1094,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1096 1094
1097 /* Set it up... */ 1095 /* Set it up... */
1098 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane; 1096 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane;
1099 if (obj_priv->tiling_mode != I915_TILING_NONE) 1097 if (obj->tiling_mode != I915_TILING_NONE)
1100 fbc_ctl2 |= FBC_CTL_CPU_FENCE; 1098 fbc_ctl2 |= FBC_CTL_CPU_FENCE;
1101 I915_WRITE(FBC_CONTROL2, fbc_ctl2); 1099 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
1102 I915_WRITE(FBC_FENCE_OFF, crtc->y); 1100 I915_WRITE(FBC_FENCE_OFF, crtc->y);
@@ -1107,7 +1105,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1107 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ 1105 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
1108 fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; 1106 fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
1109 fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT; 1107 fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
1110 if (obj_priv->tiling_mode != I915_TILING_NONE) 1108 if (obj->tiling_mode != I915_TILING_NONE)
1111 fbc_ctl |= dev_priv->cfb_fence; 1109 fbc_ctl |= dev_priv->cfb_fence;
1112 I915_WRITE(FBC_CONTROL, fbc_ctl); 1110 I915_WRITE(FBC_CONTROL, fbc_ctl);
1113 1111
@@ -1150,7 +1148,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1150 struct drm_i915_private *dev_priv = dev->dev_private; 1148 struct drm_i915_private *dev_priv = dev->dev_private;
1151 struct drm_framebuffer *fb = crtc->fb; 1149 struct drm_framebuffer *fb = crtc->fb;
1152 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1150 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1153 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1151 struct drm_i915_gem_object *obj = intel_fb->obj;
1154 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1152 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1155 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; 1153 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1156 unsigned long stall_watermark = 200; 1154 unsigned long stall_watermark = 200;
@@ -1159,7 +1157,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1159 dpfc_ctl = I915_READ(DPFC_CONTROL); 1157 dpfc_ctl = I915_READ(DPFC_CONTROL);
1160 if (dpfc_ctl & DPFC_CTL_EN) { 1158 if (dpfc_ctl & DPFC_CTL_EN) {
1161 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && 1159 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
1162 dev_priv->cfb_fence == obj_priv->fence_reg && 1160 dev_priv->cfb_fence == obj->fence_reg &&
1163 dev_priv->cfb_plane == intel_crtc->plane && 1161 dev_priv->cfb_plane == intel_crtc->plane &&
1164 dev_priv->cfb_y == crtc->y) 1162 dev_priv->cfb_y == crtc->y)
1165 return; 1163 return;
@@ -1170,12 +1168,12 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1170 } 1168 }
1171 1169
1172 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1170 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1173 dev_priv->cfb_fence = obj_priv->fence_reg; 1171 dev_priv->cfb_fence = obj->fence_reg;
1174 dev_priv->cfb_plane = intel_crtc->plane; 1172 dev_priv->cfb_plane = intel_crtc->plane;
1175 dev_priv->cfb_y = crtc->y; 1173 dev_priv->cfb_y = crtc->y;
1176 1174
1177 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; 1175 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
1178 if (obj_priv->tiling_mode != I915_TILING_NONE) { 1176 if (obj->tiling_mode != I915_TILING_NONE) {
1179 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence; 1177 dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence;
1180 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); 1178 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
1181 } else { 1179 } else {
@@ -1221,7 +1219,7 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1221 struct drm_i915_private *dev_priv = dev->dev_private; 1219 struct drm_i915_private *dev_priv = dev->dev_private;
1222 struct drm_framebuffer *fb = crtc->fb; 1220 struct drm_framebuffer *fb = crtc->fb;
1223 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 1221 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
1224 struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); 1222 struct drm_i915_gem_object *obj = intel_fb->obj;
1225 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1223 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1226 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; 1224 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
1227 unsigned long stall_watermark = 200; 1225 unsigned long stall_watermark = 200;
@@ -1230,9 +1228,9 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1230 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); 1228 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
1231 if (dpfc_ctl & DPFC_CTL_EN) { 1229 if (dpfc_ctl & DPFC_CTL_EN) {
1232 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && 1230 if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 &&
1233 dev_priv->cfb_fence == obj_priv->fence_reg && 1231 dev_priv->cfb_fence == obj->fence_reg &&
1234 dev_priv->cfb_plane == intel_crtc->plane && 1232 dev_priv->cfb_plane == intel_crtc->plane &&
1235 dev_priv->cfb_offset == obj_priv->gtt_offset && 1233 dev_priv->cfb_offset == obj->gtt_offset &&
1236 dev_priv->cfb_y == crtc->y) 1234 dev_priv->cfb_y == crtc->y)
1237 return; 1235 return;
1238 1236
@@ -1242,14 +1240,14 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1242 } 1240 }
1243 1241
1244 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; 1242 dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1;
1245 dev_priv->cfb_fence = obj_priv->fence_reg; 1243 dev_priv->cfb_fence = obj->fence_reg;
1246 dev_priv->cfb_plane = intel_crtc->plane; 1244 dev_priv->cfb_plane = intel_crtc->plane;
1247 dev_priv->cfb_offset = obj_priv->gtt_offset; 1245 dev_priv->cfb_offset = obj->gtt_offset;
1248 dev_priv->cfb_y = crtc->y; 1246 dev_priv->cfb_y = crtc->y;
1249 1247
1250 dpfc_ctl &= DPFC_RESERVED; 1248 dpfc_ctl &= DPFC_RESERVED;
1251 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); 1249 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
1252 if (obj_priv->tiling_mode != I915_TILING_NONE) { 1250 if (obj->tiling_mode != I915_TILING_NONE) {
1253 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence); 1251 dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence);
1254 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); 1252 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
1255 } else { 1253 } else {
@@ -1260,10 +1258,16 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
1260 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) | 1258 (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
1261 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT)); 1259 (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
1262 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); 1260 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
1263 I915_WRITE(ILK_FBC_RT_BASE, obj_priv->gtt_offset | ILK_FBC_RT_VALID); 1261 I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
1264 /* enable it... */ 1262 /* enable it... */
1265 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); 1263 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
1266 1264
1265 if (IS_GEN6(dev)) {
1266 I915_WRITE(SNB_DPFC_CTL_SA,
1267 SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence);
1268 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
1269 }
1270
1267 DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane); 1271 DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
1268} 1272}
1269 1273
@@ -1345,7 +1349,7 @@ static void intel_update_fbc(struct drm_device *dev)
1345 struct intel_crtc *intel_crtc; 1349 struct intel_crtc *intel_crtc;
1346 struct drm_framebuffer *fb; 1350 struct drm_framebuffer *fb;
1347 struct intel_framebuffer *intel_fb; 1351 struct intel_framebuffer *intel_fb;
1348 struct drm_i915_gem_object *obj_priv; 1352 struct drm_i915_gem_object *obj;
1349 1353
1350 DRM_DEBUG_KMS("\n"); 1354 DRM_DEBUG_KMS("\n");
1351 1355
@@ -1384,9 +1388,9 @@ static void intel_update_fbc(struct drm_device *dev)
1384 intel_crtc = to_intel_crtc(crtc); 1388 intel_crtc = to_intel_crtc(crtc);
1385 fb = crtc->fb; 1389 fb = crtc->fb;
1386 intel_fb = to_intel_framebuffer(fb); 1390 intel_fb = to_intel_framebuffer(fb);
1387 obj_priv = to_intel_bo(intel_fb->obj); 1391 obj = intel_fb->obj;
1388 1392
1389 if (intel_fb->obj->size > dev_priv->cfb_size) { 1393 if (intel_fb->obj->base.size > dev_priv->cfb_size) {
1390 DRM_DEBUG_KMS("framebuffer too large, disabling " 1394 DRM_DEBUG_KMS("framebuffer too large, disabling "
1391 "compression\n"); 1395 "compression\n");
1392 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; 1396 dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
@@ -1410,7 +1414,7 @@ static void intel_update_fbc(struct drm_device *dev)
1410 dev_priv->no_fbc_reason = FBC_BAD_PLANE; 1414 dev_priv->no_fbc_reason = FBC_BAD_PLANE;
1411 goto out_disable; 1415 goto out_disable;
1412 } 1416 }
1413 if (obj_priv->tiling_mode != I915_TILING_X) { 1417 if (obj->tiling_mode != I915_TILING_X) {
1414 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n"); 1418 DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n");
1415 dev_priv->no_fbc_reason = FBC_NOT_TILED; 1419 dev_priv->no_fbc_reason = FBC_NOT_TILED;
1416 goto out_disable; 1420 goto out_disable;
@@ -1433,14 +1437,13 @@ out_disable:
1433 1437
1434int 1438int
1435intel_pin_and_fence_fb_obj(struct drm_device *dev, 1439intel_pin_and_fence_fb_obj(struct drm_device *dev,
1436 struct drm_gem_object *obj, 1440 struct drm_i915_gem_object *obj,
1437 bool pipelined) 1441 struct intel_ring_buffer *pipelined)
1438{ 1442{
1439 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1440 u32 alignment; 1443 u32 alignment;
1441 int ret; 1444 int ret;
1442 1445
1443 switch (obj_priv->tiling_mode) { 1446 switch (obj->tiling_mode) {
1444 case I915_TILING_NONE: 1447 case I915_TILING_NONE:
1445 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) 1448 if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
1446 alignment = 128 * 1024; 1449 alignment = 128 * 1024;
@@ -1461,7 +1464,7 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
1461 BUG(); 1464 BUG();
1462 } 1465 }
1463 1466
1464 ret = i915_gem_object_pin(obj, alignment); 1467 ret = i915_gem_object_pin(obj, alignment, true);
1465 if (ret) 1468 if (ret)
1466 return ret; 1469 return ret;
1467 1470
@@ -1474,9 +1477,8 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
1474 * framebuffer compression. For simplicity, we always install 1477 * framebuffer compression. For simplicity, we always install
1475 * a fence as the cost is not that onerous. 1478 * a fence as the cost is not that onerous.
1476 */ 1479 */
1477 if (obj_priv->fence_reg == I915_FENCE_REG_NONE && 1480 if (obj->tiling_mode != I915_TILING_NONE) {
1478 obj_priv->tiling_mode != I915_TILING_NONE) { 1481 ret = i915_gem_object_get_fence(obj, pipelined, false);
1479 ret = i915_gem_object_get_fence_reg(obj, false);
1480 if (ret) 1482 if (ret)
1481 goto err_unpin; 1483 goto err_unpin;
1482 } 1484 }
@@ -1497,8 +1499,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1497 struct drm_i915_private *dev_priv = dev->dev_private; 1499 struct drm_i915_private *dev_priv = dev->dev_private;
1498 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 1500 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1499 struct intel_framebuffer *intel_fb; 1501 struct intel_framebuffer *intel_fb;
1500 struct drm_i915_gem_object *obj_priv; 1502 struct drm_i915_gem_object *obj;
1501 struct drm_gem_object *obj;
1502 int plane = intel_crtc->plane; 1503 int plane = intel_crtc->plane;
1503 unsigned long Start, Offset; 1504 unsigned long Start, Offset;
1504 u32 dspcntr; 1505 u32 dspcntr;
@@ -1515,7 +1516,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1515 1516
1516 intel_fb = to_intel_framebuffer(fb); 1517 intel_fb = to_intel_framebuffer(fb);
1517 obj = intel_fb->obj; 1518 obj = intel_fb->obj;
1518 obj_priv = to_intel_bo(obj);
1519 1519
1520 reg = DSPCNTR(plane); 1520 reg = DSPCNTR(plane);
1521 dspcntr = I915_READ(reg); 1521 dspcntr = I915_READ(reg);
@@ -1540,7 +1540,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1540 return -EINVAL; 1540 return -EINVAL;
1541 } 1541 }
1542 if (INTEL_INFO(dev)->gen >= 4) { 1542 if (INTEL_INFO(dev)->gen >= 4) {
1543 if (obj_priv->tiling_mode != I915_TILING_NONE) 1543 if (obj->tiling_mode != I915_TILING_NONE)
1544 dspcntr |= DISPPLANE_TILED; 1544 dspcntr |= DISPPLANE_TILED;
1545 else 1545 else
1546 dspcntr &= ~DISPPLANE_TILED; 1546 dspcntr &= ~DISPPLANE_TILED;
@@ -1552,7 +1552,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1552 1552
1553 I915_WRITE(reg, dspcntr); 1553 I915_WRITE(reg, dspcntr);
1554 1554
1555 Start = obj_priv->gtt_offset; 1555 Start = obj->gtt_offset;
1556 Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8); 1556 Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8);
1557 1557
1558 DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", 1558 DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
@@ -1598,7 +1598,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
1598 mutex_lock(&dev->struct_mutex); 1598 mutex_lock(&dev->struct_mutex);
1599 ret = intel_pin_and_fence_fb_obj(dev, 1599 ret = intel_pin_and_fence_fb_obj(dev,
1600 to_intel_framebuffer(crtc->fb)->obj, 1600 to_intel_framebuffer(crtc->fb)->obj,
1601 false); 1601 NULL);
1602 if (ret != 0) { 1602 if (ret != 0) {
1603 mutex_unlock(&dev->struct_mutex); 1603 mutex_unlock(&dev->struct_mutex);
1604 return ret; 1604 return ret;
@@ -1606,18 +1606,17 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
1606 1606
1607 if (old_fb) { 1607 if (old_fb) {
1608 struct drm_i915_private *dev_priv = dev->dev_private; 1608 struct drm_i915_private *dev_priv = dev->dev_private;
1609 struct drm_gem_object *obj = to_intel_framebuffer(old_fb)->obj; 1609 struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj;
1610 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1611 1610
1612 wait_event(dev_priv->pending_flip_queue, 1611 wait_event(dev_priv->pending_flip_queue,
1613 atomic_read(&obj_priv->pending_flip) == 0); 1612 atomic_read(&obj->pending_flip) == 0);
1614 1613
1615 /* Big Hammer, we also need to ensure that any pending 1614 /* Big Hammer, we also need to ensure that any pending
1616 * MI_WAIT_FOR_EVENT inside a user batch buffer on the 1615 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
1617 * current scanout is retired before unpinning the old 1616 * current scanout is retired before unpinning the old
1618 * framebuffer. 1617 * framebuffer.
1619 */ 1618 */
1620 ret = i915_gem_object_flush_gpu(obj_priv, false); 1619 ret = i915_gem_object_flush_gpu(obj, false);
1621 if (ret) { 1620 if (ret) {
1622 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj); 1621 i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj);
1623 mutex_unlock(&dev->struct_mutex); 1622 mutex_unlock(&dev->struct_mutex);
@@ -1633,8 +1632,10 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
1633 return ret; 1632 return ret;
1634 } 1633 }
1635 1634
1636 if (old_fb) 1635 if (old_fb) {
1636 intel_wait_for_vblank(dev, intel_crtc->pipe);
1637 i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj); 1637 i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj);
1638 }
1638 1639
1639 mutex_unlock(&dev->struct_mutex); 1640 mutex_unlock(&dev->struct_mutex);
1640 1641
@@ -1996,31 +1997,31 @@ static void intel_flush_display_plane(struct drm_device *dev,
1996static void intel_clear_scanline_wait(struct drm_device *dev) 1997static void intel_clear_scanline_wait(struct drm_device *dev)
1997{ 1998{
1998 struct drm_i915_private *dev_priv = dev->dev_private; 1999 struct drm_i915_private *dev_priv = dev->dev_private;
2000 struct intel_ring_buffer *ring;
1999 u32 tmp; 2001 u32 tmp;
2000 2002
2001 if (IS_GEN2(dev)) 2003 if (IS_GEN2(dev))
2002 /* Can't break the hang on i8xx */ 2004 /* Can't break the hang on i8xx */
2003 return; 2005 return;
2004 2006
2005 tmp = I915_READ(PRB0_CTL); 2007 ring = LP_RING(dev_priv);
2006 if (tmp & RING_WAIT) { 2008 tmp = I915_READ_CTL(ring);
2007 I915_WRITE(PRB0_CTL, tmp); 2009 if (tmp & RING_WAIT)
2008 POSTING_READ(PRB0_CTL); 2010 I915_WRITE_CTL(ring, tmp);
2009 }
2010} 2011}
2011 2012
2012static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) 2013static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
2013{ 2014{
2014 struct drm_i915_gem_object *obj_priv; 2015 struct drm_i915_gem_object *obj;
2015 struct drm_i915_private *dev_priv; 2016 struct drm_i915_private *dev_priv;
2016 2017
2017 if (crtc->fb == NULL) 2018 if (crtc->fb == NULL)
2018 return; 2019 return;
2019 2020
2020 obj_priv = to_intel_bo(to_intel_framebuffer(crtc->fb)->obj); 2021 obj = to_intel_framebuffer(crtc->fb)->obj;
2021 dev_priv = crtc->dev->dev_private; 2022 dev_priv = crtc->dev->dev_private;
2022 wait_event(dev_priv->pending_flip_queue, 2023 wait_event(dev_priv->pending_flip_queue,
2023 atomic_read(&obj_priv->pending_flip) == 0); 2024 atomic_read(&obj->pending_flip) == 0);
2024} 2025}
2025 2026
2026static void ironlake_crtc_enable(struct drm_crtc *crtc) 2027static void ironlake_crtc_enable(struct drm_crtc *crtc)
@@ -2850,6 +2851,39 @@ static struct intel_watermark_params ironlake_cursor_srwm_info = {
2850 ILK_FIFO_LINE_SIZE 2851 ILK_FIFO_LINE_SIZE
2851}; 2852};
2852 2853
2854static struct intel_watermark_params sandybridge_display_wm_info = {
2855 SNB_DISPLAY_FIFO,
2856 SNB_DISPLAY_MAXWM,
2857 SNB_DISPLAY_DFTWM,
2858 2,
2859 SNB_FIFO_LINE_SIZE
2860};
2861
2862static struct intel_watermark_params sandybridge_cursor_wm_info = {
2863 SNB_CURSOR_FIFO,
2864 SNB_CURSOR_MAXWM,
2865 SNB_CURSOR_DFTWM,
2866 2,
2867 SNB_FIFO_LINE_SIZE
2868};
2869
2870static struct intel_watermark_params sandybridge_display_srwm_info = {
2871 SNB_DISPLAY_SR_FIFO,
2872 SNB_DISPLAY_MAX_SRWM,
2873 SNB_DISPLAY_DFT_SRWM,
2874 2,
2875 SNB_FIFO_LINE_SIZE
2876};
2877
2878static struct intel_watermark_params sandybridge_cursor_srwm_info = {
2879 SNB_CURSOR_SR_FIFO,
2880 SNB_CURSOR_MAX_SRWM,
2881 SNB_CURSOR_DFT_SRWM,
2882 2,
2883 SNB_FIFO_LINE_SIZE
2884};
2885
2886
2853/** 2887/**
2854 * intel_calculate_wm - calculate watermark level 2888 * intel_calculate_wm - calculate watermark level
2855 * @clock_in_khz: pixel clock 2889 * @clock_in_khz: pixel clock
@@ -3383,6 +3417,10 @@ static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused,
3383 3417
3384static bool ironlake_compute_wm0(struct drm_device *dev, 3418static bool ironlake_compute_wm0(struct drm_device *dev,
3385 int pipe, 3419 int pipe,
3420 const struct intel_watermark_params *display,
3421 int display_latency,
3422 const struct intel_watermark_params *cursor,
3423 int cursor_latency,
3386 int *plane_wm, 3424 int *plane_wm,
3387 int *cursor_wm) 3425 int *cursor_wm)
3388{ 3426{
@@ -3400,22 +3438,20 @@ static bool ironlake_compute_wm0(struct drm_device *dev,
3400 pixel_size = crtc->fb->bits_per_pixel / 8; 3438 pixel_size = crtc->fb->bits_per_pixel / 8;
3401 3439
3402 /* Use the small buffer method to calculate plane watermark */ 3440 /* Use the small buffer method to calculate plane watermark */
3403 entries = ((clock * pixel_size / 1000) * ILK_LP0_PLANE_LATENCY) / 1000; 3441 entries = ((clock * pixel_size / 1000) * display_latency * 100) / 1000;
3404 entries = DIV_ROUND_UP(entries, 3442 entries = DIV_ROUND_UP(entries, display->cacheline_size);
3405 ironlake_display_wm_info.cacheline_size); 3443 *plane_wm = entries + display->guard_size;
3406 *plane_wm = entries + ironlake_display_wm_info.guard_size; 3444 if (*plane_wm > (int)display->max_wm)
3407 if (*plane_wm > (int)ironlake_display_wm_info.max_wm) 3445 *plane_wm = display->max_wm;
3408 *plane_wm = ironlake_display_wm_info.max_wm;
3409 3446
3410 /* Use the large buffer method to calculate cursor watermark */ 3447 /* Use the large buffer method to calculate cursor watermark */
3411 line_time_us = ((htotal * 1000) / clock); 3448 line_time_us = ((htotal * 1000) / clock);
3412 line_count = (ILK_LP0_CURSOR_LATENCY / line_time_us + 1000) / 1000; 3449 line_count = (cursor_latency * 100 / line_time_us + 1000) / 1000;
3413 entries = line_count * 64 * pixel_size; 3450 entries = line_count * 64 * pixel_size;
3414 entries = DIV_ROUND_UP(entries, 3451 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
3415 ironlake_cursor_wm_info.cacheline_size); 3452 *cursor_wm = entries + cursor->guard_size;
3416 *cursor_wm = entries + ironlake_cursor_wm_info.guard_size; 3453 if (*cursor_wm > (int)cursor->max_wm)
3417 if (*cursor_wm > ironlake_cursor_wm_info.max_wm) 3454 *cursor_wm = (int)cursor->max_wm;
3418 *cursor_wm = ironlake_cursor_wm_info.max_wm;
3419 3455
3420 return true; 3456 return true;
3421} 3457}
@@ -3430,7 +3466,12 @@ static void ironlake_update_wm(struct drm_device *dev,
3430 int tmp; 3466 int tmp;
3431 3467
3432 enabled = 0; 3468 enabled = 0;
3433 if (ironlake_compute_wm0(dev, 0, &plane_wm, &cursor_wm)) { 3469 if (ironlake_compute_wm0(dev, 0,
3470 &ironlake_display_wm_info,
3471 ILK_LP0_PLANE_LATENCY,
3472 &ironlake_cursor_wm_info,
3473 ILK_LP0_CURSOR_LATENCY,
3474 &plane_wm, &cursor_wm)) {
3434 I915_WRITE(WM0_PIPEA_ILK, 3475 I915_WRITE(WM0_PIPEA_ILK,
3435 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); 3476 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
3436 DRM_DEBUG_KMS("FIFO watermarks For pipe A -" 3477 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
@@ -3439,7 +3480,12 @@ static void ironlake_update_wm(struct drm_device *dev,
3439 enabled++; 3480 enabled++;
3440 } 3481 }
3441 3482
3442 if (ironlake_compute_wm0(dev, 1, &plane_wm, &cursor_wm)) { 3483 if (ironlake_compute_wm0(dev, 1,
3484 &ironlake_display_wm_info,
3485 ILK_LP0_PLANE_LATENCY,
3486 &ironlake_cursor_wm_info,
3487 ILK_LP0_CURSOR_LATENCY,
3488 &plane_wm, &cursor_wm)) {
3443 I915_WRITE(WM0_PIPEB_ILK, 3489 I915_WRITE(WM0_PIPEB_ILK,
3444 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); 3490 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
3445 DRM_DEBUG_KMS("FIFO watermarks For pipe B -" 3491 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
@@ -3453,7 +3499,7 @@ static void ironlake_update_wm(struct drm_device *dev,
3453 * display plane is used. 3499 * display plane is used.
3454 */ 3500 */
3455 tmp = 0; 3501 tmp = 0;
3456 if (enabled == 1 && /* XXX disabled due to buggy implmentation? */ 0) { 3502 if (enabled == 1) {
3457 unsigned long line_time_us; 3503 unsigned long line_time_us;
3458 int small, large, plane_fbc; 3504 int small, large, plane_fbc;
3459 int sr_clock, entries; 3505 int sr_clock, entries;
@@ -3505,6 +3551,197 @@ static void ironlake_update_wm(struct drm_device *dev,
3505 /* XXX setup WM2 and WM3 */ 3551 /* XXX setup WM2 and WM3 */
3506} 3552}
3507 3553
3554/*
3555 * Check the wm result.
3556 *
3557 * If any calculated watermark values is larger than the maximum value that
3558 * can be programmed into the associated watermark register, that watermark
3559 * must be disabled.
3560 *
3561 * Also return true if all of those watermark values is 0, which is set by
3562 * sandybridge_compute_srwm, to indicate the latency is ZERO.
3563 */
3564static bool sandybridge_check_srwm(struct drm_device *dev, int level,
3565 int fbc_wm, int display_wm, int cursor_wm)
3566{
3567 struct drm_i915_private *dev_priv = dev->dev_private;
3568
3569 DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
3570 " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
3571
3572 if (fbc_wm > SNB_FBC_MAX_SRWM) {
3573 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
3574 fbc_wm, SNB_FBC_MAX_SRWM, level);
3575
3576 /* fbc has it's own way to disable FBC WM */
3577 I915_WRITE(DISP_ARB_CTL,
3578 I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
3579 return false;
3580 }
3581
3582 if (display_wm > SNB_DISPLAY_MAX_SRWM) {
3583 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
3584 display_wm, SNB_DISPLAY_MAX_SRWM, level);
3585 return false;
3586 }
3587
3588 if (cursor_wm > SNB_CURSOR_MAX_SRWM) {
3589 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
3590 cursor_wm, SNB_CURSOR_MAX_SRWM, level);
3591 return false;
3592 }
3593
3594 if (!(fbc_wm || display_wm || cursor_wm)) {
3595 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
3596 return false;
3597 }
3598
3599 return true;
3600}
3601
3602/*
3603 * Compute watermark values of WM[1-3],
3604 */
3605static bool sandybridge_compute_srwm(struct drm_device *dev, int level,
3606 int hdisplay, int htotal, int pixel_size,
3607 int clock, int latency_ns, int *fbc_wm,
3608 int *display_wm, int *cursor_wm)
3609{
3610
3611 unsigned long line_time_us;
3612 int small, large;
3613 int entries;
3614 int line_count, line_size;
3615
3616 if (!latency_ns) {
3617 *fbc_wm = *display_wm = *cursor_wm = 0;
3618 return false;
3619 }
3620
3621 line_time_us = (htotal * 1000) / clock;
3622 line_count = (latency_ns / line_time_us + 1000) / 1000;
3623 line_size = hdisplay * pixel_size;
3624
3625 /* Use the minimum of the small and large buffer method for primary */
3626 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
3627 large = line_count * line_size;
3628
3629 entries = DIV_ROUND_UP(min(small, large),
3630 sandybridge_display_srwm_info.cacheline_size);
3631 *display_wm = entries + sandybridge_display_srwm_info.guard_size;
3632
3633 /*
3634 * Spec said:
3635 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
3636 */
3637 *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
3638
3639 /* calculate the self-refresh watermark for display cursor */
3640 entries = line_count * pixel_size * 64;
3641 entries = DIV_ROUND_UP(entries,
3642 sandybridge_cursor_srwm_info.cacheline_size);
3643 *cursor_wm = entries + sandybridge_cursor_srwm_info.guard_size;
3644
3645 return sandybridge_check_srwm(dev, level,
3646 *fbc_wm, *display_wm, *cursor_wm);
3647}
3648
3649static void sandybridge_update_wm(struct drm_device *dev,
3650 int planea_clock, int planeb_clock,
3651 int hdisplay, int htotal,
3652 int pixel_size)
3653{
3654 struct drm_i915_private *dev_priv = dev->dev_private;
3655 int latency = SNB_READ_WM0_LATENCY();
3656 int fbc_wm, plane_wm, cursor_wm, enabled;
3657 int clock;
3658
3659 enabled = 0;
3660 if (ironlake_compute_wm0(dev, 0,
3661 &sandybridge_display_wm_info, latency,
3662 &sandybridge_cursor_wm_info, latency,
3663 &plane_wm, &cursor_wm)) {
3664 I915_WRITE(WM0_PIPEA_ILK,
3665 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
3666 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
3667 " plane %d, " "cursor: %d\n",
3668 plane_wm, cursor_wm);
3669 enabled++;
3670 }
3671
3672 if (ironlake_compute_wm0(dev, 1,
3673 &sandybridge_display_wm_info, latency,
3674 &sandybridge_cursor_wm_info, latency,
3675 &plane_wm, &cursor_wm)) {
3676 I915_WRITE(WM0_PIPEB_ILK,
3677 (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
3678 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
3679 " plane %d, cursor: %d\n",
3680 plane_wm, cursor_wm);
3681 enabled++;
3682 }
3683
3684 /*
3685 * Calculate and update the self-refresh watermark only when one
3686 * display plane is used.
3687 *
3688 * SNB support 3 levels of watermark.
3689 *
3690 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
3691 * and disabled in the descending order
3692 *
3693 */
3694 I915_WRITE(WM3_LP_ILK, 0);
3695 I915_WRITE(WM2_LP_ILK, 0);
3696 I915_WRITE(WM1_LP_ILK, 0);
3697
3698 if (enabled != 1)
3699 return;
3700
3701 clock = planea_clock ? planea_clock : planeb_clock;
3702
3703 /* WM1 */
3704 if (!sandybridge_compute_srwm(dev, 1, hdisplay, htotal, pixel_size,
3705 clock, SNB_READ_WM1_LATENCY() * 500,
3706 &fbc_wm, &plane_wm, &cursor_wm))
3707 return;
3708
3709 I915_WRITE(WM1_LP_ILK,
3710 WM1_LP_SR_EN |
3711 (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
3712 (fbc_wm << WM1_LP_FBC_SHIFT) |
3713 (plane_wm << WM1_LP_SR_SHIFT) |
3714 cursor_wm);
3715
3716 /* WM2 */
3717 if (!sandybridge_compute_srwm(dev, 2,
3718 hdisplay, htotal, pixel_size,
3719 clock, SNB_READ_WM2_LATENCY() * 500,
3720 &fbc_wm, &plane_wm, &cursor_wm))
3721 return;
3722
3723 I915_WRITE(WM2_LP_ILK,
3724 WM2_LP_EN |
3725 (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
3726 (fbc_wm << WM1_LP_FBC_SHIFT) |
3727 (plane_wm << WM1_LP_SR_SHIFT) |
3728 cursor_wm);
3729
3730 /* WM3 */
3731 if (!sandybridge_compute_srwm(dev, 3,
3732 hdisplay, htotal, pixel_size,
3733 clock, SNB_READ_WM3_LATENCY() * 500,
3734 &fbc_wm, &plane_wm, &cursor_wm))
3735 return;
3736
3737 I915_WRITE(WM3_LP_ILK,
3738 WM3_LP_EN |
3739 (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
3740 (fbc_wm << WM1_LP_FBC_SHIFT) |
3741 (plane_wm << WM1_LP_SR_SHIFT) |
3742 cursor_wm);
3743}
3744
3508/** 3745/**
3509 * intel_update_watermarks - update FIFO watermark values based on current modes 3746 * intel_update_watermarks - update FIFO watermark values based on current modes
3510 * 3747 *
@@ -3660,7 +3897,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
3660 * refclk, or FALSE. The returned values represent the clock equation: 3897 * refclk, or FALSE. The returned values represent the clock equation:
3661 * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. 3898 * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
3662 */ 3899 */
3663 limit = intel_limit(crtc); 3900 limit = intel_limit(crtc, refclk);
3664 ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock); 3901 ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock);
3665 if (!ok) { 3902 if (!ok) {
3666 DRM_ERROR("Couldn't find PLL settings for mode!\n"); 3903 DRM_ERROR("Couldn't find PLL settings for mode!\n");
@@ -3857,6 +4094,22 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
3857 reduced_clock.m2; 4094 reduced_clock.m2;
3858 } 4095 }
3859 4096
4097 /* Enable autotuning of the PLL clock (if permissible) */
4098 if (HAS_PCH_SPLIT(dev)) {
4099 int factor = 21;
4100
4101 if (is_lvds) {
4102 if ((dev_priv->lvds_use_ssc &&
4103 dev_priv->lvds_ssc_freq == 100) ||
4104 (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP)
4105 factor = 25;
4106 } else if (is_sdvo && is_tv)
4107 factor = 20;
4108
4109 if (clock.m1 < factor * clock.n)
4110 fp |= FP_CB_TUNE;
4111 }
4112
3860 dpll = 0; 4113 dpll = 0;
3861 if (!HAS_PCH_SPLIT(dev)) 4114 if (!HAS_PCH_SPLIT(dev))
3862 dpll = DPLL_VGA_MODE_DIS; 4115 dpll = DPLL_VGA_MODE_DIS;
@@ -4071,7 +4324,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
4071 } 4324 }
4072 4325
4073 if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) { 4326 if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
4074 I915_WRITE(fp_reg, fp);
4075 I915_WRITE(dpll_reg, dpll); 4327 I915_WRITE(dpll_reg, dpll);
4076 4328
4077 /* Wait for the clocks to stabilize. */ 4329 /* Wait for the clocks to stabilize. */
@@ -4089,13 +4341,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
4089 } 4341 }
4090 I915_WRITE(DPLL_MD(pipe), temp); 4342 I915_WRITE(DPLL_MD(pipe), temp);
4091 } else { 4343 } else {
4092 /* write it again -- the BIOS does, after all */ 4344 /* The pixel multiplier can only be updated once the
4345 * DPLL is enabled and the clocks are stable.
4346 *
4347 * So write it again.
4348 */
4093 I915_WRITE(dpll_reg, dpll); 4349 I915_WRITE(dpll_reg, dpll);
4094 } 4350 }
4095
4096 /* Wait for the clocks to stabilize. */
4097 POSTING_READ(dpll_reg);
4098 udelay(150);
4099 } 4351 }
4100 4352
4101 intel_crtc->lowfreq_avail = false; 4353 intel_crtc->lowfreq_avail = false;
@@ -4331,15 +4583,14 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
4331} 4583}
4332 4584
4333static int intel_crtc_cursor_set(struct drm_crtc *crtc, 4585static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4334 struct drm_file *file_priv, 4586 struct drm_file *file,
4335 uint32_t handle, 4587 uint32_t handle,
4336 uint32_t width, uint32_t height) 4588 uint32_t width, uint32_t height)
4337{ 4589{
4338 struct drm_device *dev = crtc->dev; 4590 struct drm_device *dev = crtc->dev;
4339 struct drm_i915_private *dev_priv = dev->dev_private; 4591 struct drm_i915_private *dev_priv = dev->dev_private;
4340 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 4592 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4341 struct drm_gem_object *bo; 4593 struct drm_i915_gem_object *obj;
4342 struct drm_i915_gem_object *obj_priv;
4343 uint32_t addr; 4594 uint32_t addr;
4344 int ret; 4595 int ret;
4345 4596
@@ -4349,7 +4600,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4349 if (!handle) { 4600 if (!handle) {
4350 DRM_DEBUG_KMS("cursor off\n"); 4601 DRM_DEBUG_KMS("cursor off\n");
4351 addr = 0; 4602 addr = 0;
4352 bo = NULL; 4603 obj = NULL;
4353 mutex_lock(&dev->struct_mutex); 4604 mutex_lock(&dev->struct_mutex);
4354 goto finish; 4605 goto finish;
4355 } 4606 }
@@ -4360,13 +4611,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4360 return -EINVAL; 4611 return -EINVAL;
4361 } 4612 }
4362 4613
4363 bo = drm_gem_object_lookup(dev, file_priv, handle); 4614 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
4364 if (!bo) 4615 if (!obj)
4365 return -ENOENT; 4616 return -ENOENT;
4366 4617
4367 obj_priv = to_intel_bo(bo); 4618 if (obj->base.size < width * height * 4) {
4368
4369 if (bo->size < width * height * 4) {
4370 DRM_ERROR("buffer is to small\n"); 4619 DRM_ERROR("buffer is to small\n");
4371 ret = -ENOMEM; 4620 ret = -ENOMEM;
4372 goto fail; 4621 goto fail;
@@ -4375,29 +4624,41 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4375 /* we only need to pin inside GTT if cursor is non-phy */ 4624 /* we only need to pin inside GTT if cursor is non-phy */
4376 mutex_lock(&dev->struct_mutex); 4625 mutex_lock(&dev->struct_mutex);
4377 if (!dev_priv->info->cursor_needs_physical) { 4626 if (!dev_priv->info->cursor_needs_physical) {
4378 ret = i915_gem_object_pin(bo, PAGE_SIZE); 4627 if (obj->tiling_mode) {
4628 DRM_ERROR("cursor cannot be tiled\n");
4629 ret = -EINVAL;
4630 goto fail_locked;
4631 }
4632
4633 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
4379 if (ret) { 4634 if (ret) {
4380 DRM_ERROR("failed to pin cursor bo\n"); 4635 DRM_ERROR("failed to pin cursor bo\n");
4381 goto fail_locked; 4636 goto fail_locked;
4382 } 4637 }
4383 4638
4384 ret = i915_gem_object_set_to_gtt_domain(bo, 0); 4639 ret = i915_gem_object_set_to_gtt_domain(obj, 0);
4385 if (ret) { 4640 if (ret) {
4386 DRM_ERROR("failed to move cursor bo into the GTT\n"); 4641 DRM_ERROR("failed to move cursor bo into the GTT\n");
4387 goto fail_unpin; 4642 goto fail_unpin;
4388 } 4643 }
4389 4644
4390 addr = obj_priv->gtt_offset; 4645 ret = i915_gem_object_put_fence(obj);
4646 if (ret) {
4647 DRM_ERROR("failed to move cursor bo into the GTT\n");
4648 goto fail_unpin;
4649 }
4650
4651 addr = obj->gtt_offset;
4391 } else { 4652 } else {
4392 int align = IS_I830(dev) ? 16 * 1024 : 256; 4653 int align = IS_I830(dev) ? 16 * 1024 : 256;
4393 ret = i915_gem_attach_phys_object(dev, bo, 4654 ret = i915_gem_attach_phys_object(dev, obj,
4394 (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1, 4655 (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
4395 align); 4656 align);
4396 if (ret) { 4657 if (ret) {
4397 DRM_ERROR("failed to attach phys object\n"); 4658 DRM_ERROR("failed to attach phys object\n");
4398 goto fail_locked; 4659 goto fail_locked;
4399 } 4660 }
4400 addr = obj_priv->phys_obj->handle->busaddr; 4661 addr = obj->phys_obj->handle->busaddr;
4401 } 4662 }
4402 4663
4403 if (IS_GEN2(dev)) 4664 if (IS_GEN2(dev))
@@ -4406,17 +4667,17 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4406 finish: 4667 finish:
4407 if (intel_crtc->cursor_bo) { 4668 if (intel_crtc->cursor_bo) {
4408 if (dev_priv->info->cursor_needs_physical) { 4669 if (dev_priv->info->cursor_needs_physical) {
4409 if (intel_crtc->cursor_bo != bo) 4670 if (intel_crtc->cursor_bo != obj)
4410 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo); 4671 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
4411 } else 4672 } else
4412 i915_gem_object_unpin(intel_crtc->cursor_bo); 4673 i915_gem_object_unpin(intel_crtc->cursor_bo);
4413 drm_gem_object_unreference(intel_crtc->cursor_bo); 4674 drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
4414 } 4675 }
4415 4676
4416 mutex_unlock(&dev->struct_mutex); 4677 mutex_unlock(&dev->struct_mutex);
4417 4678
4418 intel_crtc->cursor_addr = addr; 4679 intel_crtc->cursor_addr = addr;
4419 intel_crtc->cursor_bo = bo; 4680 intel_crtc->cursor_bo = obj;
4420 intel_crtc->cursor_width = width; 4681 intel_crtc->cursor_width = width;
4421 intel_crtc->cursor_height = height; 4682 intel_crtc->cursor_height = height;
4422 4683
@@ -4424,11 +4685,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
4424 4685
4425 return 0; 4686 return 0;
4426fail_unpin: 4687fail_unpin:
4427 i915_gem_object_unpin(bo); 4688 i915_gem_object_unpin(obj);
4428fail_locked: 4689fail_locked:
4429 mutex_unlock(&dev->struct_mutex); 4690 mutex_unlock(&dev->struct_mutex);
4430fail: 4691fail:
4431 drm_gem_object_unreference_unlocked(bo); 4692 drm_gem_object_unreference_unlocked(&obj->base);
4432 return ret; 4693 return ret;
4433} 4694}
4434 4695
@@ -4739,8 +5000,14 @@ static void intel_gpu_idle_timer(unsigned long arg)
4739 struct drm_device *dev = (struct drm_device *)arg; 5000 struct drm_device *dev = (struct drm_device *)arg;
4740 drm_i915_private_t *dev_priv = dev->dev_private; 5001 drm_i915_private_t *dev_priv = dev->dev_private;
4741 5002
4742 dev_priv->busy = false; 5003 if (!list_empty(&dev_priv->mm.active_list)) {
5004 /* Still processing requests, so just re-arm the timer. */
5005 mod_timer(&dev_priv->idle_timer, jiffies +
5006 msecs_to_jiffies(GPU_IDLE_TIMEOUT));
5007 return;
5008 }
4743 5009
5010 dev_priv->busy = false;
4744 queue_work(dev_priv->wq, &dev_priv->idle_work); 5011 queue_work(dev_priv->wq, &dev_priv->idle_work);
4745} 5012}
4746 5013
@@ -4751,9 +5018,17 @@ static void intel_crtc_idle_timer(unsigned long arg)
4751 struct intel_crtc *intel_crtc = (struct intel_crtc *)arg; 5018 struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
4752 struct drm_crtc *crtc = &intel_crtc->base; 5019 struct drm_crtc *crtc = &intel_crtc->base;
4753 drm_i915_private_t *dev_priv = crtc->dev->dev_private; 5020 drm_i915_private_t *dev_priv = crtc->dev->dev_private;
5021 struct intel_framebuffer *intel_fb;
4754 5022
4755 intel_crtc->busy = false; 5023 intel_fb = to_intel_framebuffer(crtc->fb);
5024 if (intel_fb && intel_fb->obj->active) {
5025 /* The framebuffer is still being accessed by the GPU. */
5026 mod_timer(&intel_crtc->idle_timer, jiffies +
5027 msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
5028 return;
5029 }
4756 5030
5031 intel_crtc->busy = false;
4757 queue_work(dev_priv->wq, &dev_priv->idle_work); 5032 queue_work(dev_priv->wq, &dev_priv->idle_work);
4758} 5033}
4759 5034
@@ -4888,7 +5163,7 @@ static void intel_idle_update(struct work_struct *work)
4888 * buffer), we'll also mark the display as busy, so we know to increase its 5163 * buffer), we'll also mark the display as busy, so we know to increase its
4889 * clock frequency. 5164 * clock frequency.
4890 */ 5165 */
4891void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj) 5166void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
4892{ 5167{
4893 drm_i915_private_t *dev_priv = dev->dev_private; 5168 drm_i915_private_t *dev_priv = dev->dev_private;
4894 struct drm_crtc *crtc = NULL; 5169 struct drm_crtc *crtc = NULL;
@@ -4969,8 +5244,9 @@ static void intel_unpin_work_fn(struct work_struct *__work)
4969 5244
4970 mutex_lock(&work->dev->struct_mutex); 5245 mutex_lock(&work->dev->struct_mutex);
4971 i915_gem_object_unpin(work->old_fb_obj); 5246 i915_gem_object_unpin(work->old_fb_obj);
4972 drm_gem_object_unreference(work->pending_flip_obj); 5247 drm_gem_object_unreference(&work->pending_flip_obj->base);
4973 drm_gem_object_unreference(work->old_fb_obj); 5248 drm_gem_object_unreference(&work->old_fb_obj->base);
5249
4974 mutex_unlock(&work->dev->struct_mutex); 5250 mutex_unlock(&work->dev->struct_mutex);
4975 kfree(work); 5251 kfree(work);
4976} 5252}
@@ -4981,15 +5257,17 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
4981 drm_i915_private_t *dev_priv = dev->dev_private; 5257 drm_i915_private_t *dev_priv = dev->dev_private;
4982 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5258 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4983 struct intel_unpin_work *work; 5259 struct intel_unpin_work *work;
4984 struct drm_i915_gem_object *obj_priv; 5260 struct drm_i915_gem_object *obj;
4985 struct drm_pending_vblank_event *e; 5261 struct drm_pending_vblank_event *e;
4986 struct timeval now; 5262 struct timeval tnow, tvbl;
4987 unsigned long flags; 5263 unsigned long flags;
4988 5264
4989 /* Ignore early vblank irqs */ 5265 /* Ignore early vblank irqs */
4990 if (intel_crtc == NULL) 5266 if (intel_crtc == NULL)
4991 return; 5267 return;
4992 5268
5269 do_gettimeofday(&tnow);
5270
4993 spin_lock_irqsave(&dev->event_lock, flags); 5271 spin_lock_irqsave(&dev->event_lock, flags);
4994 work = intel_crtc->unpin_work; 5272 work = intel_crtc->unpin_work;
4995 if (work == NULL || !work->pending) { 5273 if (work == NULL || !work->pending) {
@@ -4998,26 +5276,49 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
4998 } 5276 }
4999 5277
5000 intel_crtc->unpin_work = NULL; 5278 intel_crtc->unpin_work = NULL;
5001 drm_vblank_put(dev, intel_crtc->pipe);
5002 5279
5003 if (work->event) { 5280 if (work->event) {
5004 e = work->event; 5281 e = work->event;
5005 do_gettimeofday(&now); 5282 e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl);
5006 e->event.sequence = drm_vblank_count(dev, intel_crtc->pipe); 5283
5007 e->event.tv_sec = now.tv_sec; 5284 /* Called before vblank count and timestamps have
5008 e->event.tv_usec = now.tv_usec; 5285 * been updated for the vblank interval of flip
5286 * completion? Need to increment vblank count and
5287 * add one videorefresh duration to returned timestamp
5288 * to account for this. We assume this happened if we
5289 * get called over 0.9 frame durations after the last
5290 * timestamped vblank.
5291 *
5292 * This calculation can not be used with vrefresh rates
5293 * below 5Hz (10Hz to be on the safe side) without
5294 * promoting to 64 integers.
5295 */
5296 if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) >
5297 9 * crtc->framedur_ns) {
5298 e->event.sequence++;
5299 tvbl = ns_to_timeval(timeval_to_ns(&tvbl) +
5300 crtc->framedur_ns);
5301 }
5302
5303 e->event.tv_sec = tvbl.tv_sec;
5304 e->event.tv_usec = tvbl.tv_usec;
5305
5009 list_add_tail(&e->base.link, 5306 list_add_tail(&e->base.link,
5010 &e->base.file_priv->event_list); 5307 &e->base.file_priv->event_list);
5011 wake_up_interruptible(&e->base.file_priv->event_wait); 5308 wake_up_interruptible(&e->base.file_priv->event_wait);
5012 } 5309 }
5013 5310
5311 drm_vblank_put(dev, intel_crtc->pipe);
5312
5014 spin_unlock_irqrestore(&dev->event_lock, flags); 5313 spin_unlock_irqrestore(&dev->event_lock, flags);
5015 5314
5016 obj_priv = to_intel_bo(work->old_fb_obj); 5315 obj = work->old_fb_obj;
5316
5017 atomic_clear_mask(1 << intel_crtc->plane, 5317 atomic_clear_mask(1 << intel_crtc->plane,
5018 &obj_priv->pending_flip.counter); 5318 &obj->pending_flip.counter);
5019 if (atomic_read(&obj_priv->pending_flip) == 0) 5319 if (atomic_read(&obj->pending_flip) == 0)
5020 wake_up(&dev_priv->pending_flip_queue); 5320 wake_up(&dev_priv->pending_flip_queue);
5321
5021 schedule_work(&work->work); 5322 schedule_work(&work->work);
5022 5323
5023 trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); 5324 trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
@@ -5063,8 +5364,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5063 struct drm_device *dev = crtc->dev; 5364 struct drm_device *dev = crtc->dev;
5064 struct drm_i915_private *dev_priv = dev->dev_private; 5365 struct drm_i915_private *dev_priv = dev->dev_private;
5065 struct intel_framebuffer *intel_fb; 5366 struct intel_framebuffer *intel_fb;
5066 struct drm_i915_gem_object *obj_priv; 5367 struct drm_i915_gem_object *obj;
5067 struct drm_gem_object *obj;
5068 struct intel_crtc *intel_crtc = to_intel_crtc(crtc); 5368 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5069 struct intel_unpin_work *work; 5369 struct intel_unpin_work *work;
5070 unsigned long flags, offset; 5370 unsigned long flags, offset;
@@ -5098,13 +5398,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5098 obj = intel_fb->obj; 5398 obj = intel_fb->obj;
5099 5399
5100 mutex_lock(&dev->struct_mutex); 5400 mutex_lock(&dev->struct_mutex);
5101 ret = intel_pin_and_fence_fb_obj(dev, obj, true); 5401 ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
5102 if (ret) 5402 if (ret)
5103 goto cleanup_work; 5403 goto cleanup_work;
5104 5404
5105 /* Reference the objects for the scheduled work. */ 5405 /* Reference the objects for the scheduled work. */
5106 drm_gem_object_reference(work->old_fb_obj); 5406 drm_gem_object_reference(&work->old_fb_obj->base);
5107 drm_gem_object_reference(obj); 5407 drm_gem_object_reference(&obj->base);
5108 5408
5109 crtc->fb = fb; 5409 crtc->fb = fb;
5110 5410
@@ -5112,22 +5412,16 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5112 if (ret) 5412 if (ret)
5113 goto cleanup_objs; 5413 goto cleanup_objs;
5114 5414
5115 /* Block clients from rendering to the new back buffer until
5116 * the flip occurs and the object is no longer visible.
5117 */
5118 atomic_add(1 << intel_crtc->plane,
5119 &to_intel_bo(work->old_fb_obj)->pending_flip);
5120
5121 work->pending_flip_obj = obj;
5122 obj_priv = to_intel_bo(obj);
5123
5124 if (IS_GEN3(dev) || IS_GEN2(dev)) { 5415 if (IS_GEN3(dev) || IS_GEN2(dev)) {
5125 u32 flip_mask; 5416 u32 flip_mask;
5126 5417
5127 /* Can't queue multiple flips, so wait for the previous 5418 /* Can't queue multiple flips, so wait for the previous
5128 * one to finish before executing the next. 5419 * one to finish before executing the next.
5129 */ 5420 */
5130 BEGIN_LP_RING(2); 5421 ret = BEGIN_LP_RING(2);
5422 if (ret)
5423 goto cleanup_objs;
5424
5131 if (intel_crtc->plane) 5425 if (intel_crtc->plane)
5132 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; 5426 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
5133 else 5427 else
@@ -5137,18 +5431,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5137 ADVANCE_LP_RING(); 5431 ADVANCE_LP_RING();
5138 } 5432 }
5139 5433
5434 work->pending_flip_obj = obj;
5435
5140 work->enable_stall_check = true; 5436 work->enable_stall_check = true;
5141 5437
5142 /* Offset into the new buffer for cases of shared fbs between CRTCs */ 5438 /* Offset into the new buffer for cases of shared fbs between CRTCs */
5143 offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8; 5439 offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8;
5144 5440
5145 BEGIN_LP_RING(4); 5441 ret = BEGIN_LP_RING(4);
5146 switch(INTEL_INFO(dev)->gen) { 5442 if (ret)
5443 goto cleanup_objs;
5444
5445 /* Block clients from rendering to the new back buffer until
5446 * the flip occurs and the object is no longer visible.
5447 */
5448 atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip);
5449
5450 switch (INTEL_INFO(dev)->gen) {
5147 case 2: 5451 case 2:
5148 OUT_RING(MI_DISPLAY_FLIP | 5452 OUT_RING(MI_DISPLAY_FLIP |
5149 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5453 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5150 OUT_RING(fb->pitch); 5454 OUT_RING(fb->pitch);
5151 OUT_RING(obj_priv->gtt_offset + offset); 5455 OUT_RING(obj->gtt_offset + offset);
5152 OUT_RING(MI_NOOP); 5456 OUT_RING(MI_NOOP);
5153 break; 5457 break;
5154 5458
@@ -5156,7 +5460,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5156 OUT_RING(MI_DISPLAY_FLIP_I915 | 5460 OUT_RING(MI_DISPLAY_FLIP_I915 |
5157 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5461 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5158 OUT_RING(fb->pitch); 5462 OUT_RING(fb->pitch);
5159 OUT_RING(obj_priv->gtt_offset + offset); 5463 OUT_RING(obj->gtt_offset + offset);
5160 OUT_RING(MI_NOOP); 5464 OUT_RING(MI_NOOP);
5161 break; 5465 break;
5162 5466
@@ -5169,7 +5473,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5169 OUT_RING(MI_DISPLAY_FLIP | 5473 OUT_RING(MI_DISPLAY_FLIP |
5170 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5474 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5171 OUT_RING(fb->pitch); 5475 OUT_RING(fb->pitch);
5172 OUT_RING(obj_priv->gtt_offset | obj_priv->tiling_mode); 5476 OUT_RING(obj->gtt_offset | obj->tiling_mode);
5173 5477
5174 /* XXX Enabling the panel-fitter across page-flip is so far 5478 /* XXX Enabling the panel-fitter across page-flip is so far
5175 * untested on non-native modes, so ignore it for now. 5479 * untested on non-native modes, so ignore it for now.
@@ -5183,8 +5487,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5183 case 6: 5487 case 6:
5184 OUT_RING(MI_DISPLAY_FLIP | 5488 OUT_RING(MI_DISPLAY_FLIP |
5185 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); 5489 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
5186 OUT_RING(fb->pitch | obj_priv->tiling_mode); 5490 OUT_RING(fb->pitch | obj->tiling_mode);
5187 OUT_RING(obj_priv->gtt_offset); 5491 OUT_RING(obj->gtt_offset);
5188 5492
5189 pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE; 5493 pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
5190 pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff; 5494 pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff;
@@ -5200,8 +5504,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
5200 return 0; 5504 return 0;
5201 5505
5202cleanup_objs: 5506cleanup_objs:
5203 drm_gem_object_unreference(work->old_fb_obj); 5507 drm_gem_object_unreference(&work->old_fb_obj->base);
5204 drm_gem_object_unreference(obj); 5508 drm_gem_object_unreference(&obj->base);
5205cleanup_work: 5509cleanup_work:
5206 mutex_unlock(&dev->struct_mutex); 5510 mutex_unlock(&dev->struct_mutex);
5207 5511
@@ -5338,7 +5642,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
5338} 5642}
5339 5643
5340int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, 5644int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
5341 struct drm_file *file_priv) 5645 struct drm_file *file)
5342{ 5646{
5343 drm_i915_private_t *dev_priv = dev->dev_private; 5647 drm_i915_private_t *dev_priv = dev->dev_private;
5344 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data; 5648 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
@@ -5505,19 +5809,19 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
5505 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 5809 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
5506 5810
5507 drm_framebuffer_cleanup(fb); 5811 drm_framebuffer_cleanup(fb);
5508 drm_gem_object_unreference_unlocked(intel_fb->obj); 5812 drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
5509 5813
5510 kfree(intel_fb); 5814 kfree(intel_fb);
5511} 5815}
5512 5816
5513static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, 5817static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
5514 struct drm_file *file_priv, 5818 struct drm_file *file,
5515 unsigned int *handle) 5819 unsigned int *handle)
5516{ 5820{
5517 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 5821 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
5518 struct drm_gem_object *object = intel_fb->obj; 5822 struct drm_i915_gem_object *obj = intel_fb->obj;
5519 5823
5520 return drm_gem_handle_create(file_priv, object, handle); 5824 return drm_gem_handle_create(file, &obj->base, handle);
5521} 5825}
5522 5826
5523static const struct drm_framebuffer_funcs intel_fb_funcs = { 5827static const struct drm_framebuffer_funcs intel_fb_funcs = {
@@ -5528,12 +5832,11 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
5528int intel_framebuffer_init(struct drm_device *dev, 5832int intel_framebuffer_init(struct drm_device *dev,
5529 struct intel_framebuffer *intel_fb, 5833 struct intel_framebuffer *intel_fb,
5530 struct drm_mode_fb_cmd *mode_cmd, 5834 struct drm_mode_fb_cmd *mode_cmd,
5531 struct drm_gem_object *obj) 5835 struct drm_i915_gem_object *obj)
5532{ 5836{
5533 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5534 int ret; 5837 int ret;
5535 5838
5536 if (obj_priv->tiling_mode == I915_TILING_Y) 5839 if (obj->tiling_mode == I915_TILING_Y)
5537 return -EINVAL; 5840 return -EINVAL;
5538 5841
5539 if (mode_cmd->pitch & 63) 5842 if (mode_cmd->pitch & 63)
@@ -5565,11 +5868,11 @@ intel_user_framebuffer_create(struct drm_device *dev,
5565 struct drm_file *filp, 5868 struct drm_file *filp,
5566 struct drm_mode_fb_cmd *mode_cmd) 5869 struct drm_mode_fb_cmd *mode_cmd)
5567{ 5870{
5568 struct drm_gem_object *obj; 5871 struct drm_i915_gem_object *obj;
5569 struct intel_framebuffer *intel_fb; 5872 struct intel_framebuffer *intel_fb;
5570 int ret; 5873 int ret;
5571 5874
5572 obj = drm_gem_object_lookup(dev, filp, mode_cmd->handle); 5875 obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle));
5573 if (!obj) 5876 if (!obj)
5574 return ERR_PTR(-ENOENT); 5877 return ERR_PTR(-ENOENT);
5575 5878
@@ -5577,10 +5880,9 @@ intel_user_framebuffer_create(struct drm_device *dev,
5577 if (!intel_fb) 5880 if (!intel_fb)
5578 return ERR_PTR(-ENOMEM); 5881 return ERR_PTR(-ENOMEM);
5579 5882
5580 ret = intel_framebuffer_init(dev, intel_fb, 5883 ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
5581 mode_cmd, obj);
5582 if (ret) { 5884 if (ret) {
5583 drm_gem_object_unreference_unlocked(obj); 5885 drm_gem_object_unreference_unlocked(&obj->base);
5584 kfree(intel_fb); 5886 kfree(intel_fb);
5585 return ERR_PTR(ret); 5887 return ERR_PTR(ret);
5586 } 5888 }
@@ -5593,10 +5895,10 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
5593 .output_poll_changed = intel_fb_output_poll_changed, 5895 .output_poll_changed = intel_fb_output_poll_changed,
5594}; 5896};
5595 5897
5596static struct drm_gem_object * 5898static struct drm_i915_gem_object *
5597intel_alloc_context_page(struct drm_device *dev) 5899intel_alloc_context_page(struct drm_device *dev)
5598{ 5900{
5599 struct drm_gem_object *ctx; 5901 struct drm_i915_gem_object *ctx;
5600 int ret; 5902 int ret;
5601 5903
5602 ctx = i915_gem_alloc_object(dev, 4096); 5904 ctx = i915_gem_alloc_object(dev, 4096);
@@ -5606,7 +5908,7 @@ intel_alloc_context_page(struct drm_device *dev)
5606 } 5908 }
5607 5909
5608 mutex_lock(&dev->struct_mutex); 5910 mutex_lock(&dev->struct_mutex);
5609 ret = i915_gem_object_pin(ctx, 4096); 5911 ret = i915_gem_object_pin(ctx, 4096, true);
5610 if (ret) { 5912 if (ret) {
5611 DRM_ERROR("failed to pin power context: %d\n", ret); 5913 DRM_ERROR("failed to pin power context: %d\n", ret);
5612 goto err_unref; 5914 goto err_unref;
@@ -5624,7 +5926,7 @@ intel_alloc_context_page(struct drm_device *dev)
5624err_unpin: 5926err_unpin:
5625 i915_gem_object_unpin(ctx); 5927 i915_gem_object_unpin(ctx);
5626err_unref: 5928err_unref:
5627 drm_gem_object_unreference(ctx); 5929 drm_gem_object_unreference(&ctx->base);
5628 mutex_unlock(&dev->struct_mutex); 5930 mutex_unlock(&dev->struct_mutex);
5629 return NULL; 5931 return NULL;
5630} 5932}
@@ -5736,6 +6038,25 @@ void ironlake_disable_drps(struct drm_device *dev)
5736 6038
5737} 6039}
5738 6040
6041void gen6_set_rps(struct drm_device *dev, u8 val)
6042{
6043 struct drm_i915_private *dev_priv = dev->dev_private;
6044 u32 swreq;
6045
6046 swreq = (val & 0x3ff) << 25;
6047 I915_WRITE(GEN6_RPNSWREQ, swreq);
6048}
6049
6050void gen6_disable_rps(struct drm_device *dev)
6051{
6052 struct drm_i915_private *dev_priv = dev->dev_private;
6053
6054 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6055 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
6056 I915_WRITE(GEN6_PMIER, 0);
6057 I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
6058}
6059
5739static unsigned long intel_pxfreq(u32 vidfreq) 6060static unsigned long intel_pxfreq(u32 vidfreq)
5740{ 6061{
5741 unsigned long freq; 6062 unsigned long freq;
@@ -5822,7 +6143,96 @@ void intel_init_emon(struct drm_device *dev)
5822 dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK); 6143 dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
5823} 6144}
5824 6145
5825void intel_init_clock_gating(struct drm_device *dev) 6146void gen6_enable_rps(struct drm_i915_private *dev_priv)
6147{
6148 int i;
6149
6150 /* Here begins a magic sequence of register writes to enable
6151 * auto-downclocking.
6152 *
6153 * Perhaps there might be some value in exposing these to
6154 * userspace...
6155 */
6156 I915_WRITE(GEN6_RC_STATE, 0);
6157 __gen6_force_wake_get(dev_priv);
6158
6159 /* disable the counters and set deterministic thresholds */
6160 I915_WRITE(GEN6_RC_CONTROL, 0);
6161
6162 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
6163 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
6164 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
6165 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6166 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6167
6168 for (i = 0; i < I915_NUM_RINGS; i++)
6169 I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10);
6170
6171 I915_WRITE(GEN6_RC_SLEEP, 0);
6172 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
6173 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
6174 I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
6175 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
6176
6177 I915_WRITE(GEN6_RC_CONTROL,
6178 GEN6_RC_CTL_RC6p_ENABLE |
6179 GEN6_RC_CTL_RC6_ENABLE |
6180 GEN6_RC_CTL_EI_MODE(1) |
6181 GEN6_RC_CTL_HW_ENABLE);
6182
6183 I915_WRITE(GEN6_RPNSWREQ,
6184 GEN6_FREQUENCY(10) |
6185 GEN6_OFFSET(0) |
6186 GEN6_AGGRESSIVE_TURBO);
6187 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6188 GEN6_FREQUENCY(12));
6189
6190 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
6191 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
6192 18 << 24 |
6193 6 << 16);
6194 I915_WRITE(GEN6_RP_UP_THRESHOLD, 90000);
6195 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 100000);
6196 I915_WRITE(GEN6_RP_UP_EI, 100000);
6197 I915_WRITE(GEN6_RP_DOWN_EI, 300000);
6198 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6199 I915_WRITE(GEN6_RP_CONTROL,
6200 GEN6_RP_MEDIA_TURBO |
6201 GEN6_RP_USE_NORMAL_FREQ |
6202 GEN6_RP_MEDIA_IS_GFX |
6203 GEN6_RP_ENABLE |
6204 GEN6_RP_UP_BUSY_MAX |
6205 GEN6_RP_DOWN_BUSY_MIN);
6206
6207 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6208 500))
6209 DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
6210
6211 I915_WRITE(GEN6_PCODE_DATA, 0);
6212 I915_WRITE(GEN6_PCODE_MAILBOX,
6213 GEN6_PCODE_READY |
6214 GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
6215 if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6216 500))
6217 DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
6218
6219 /* requires MSI enabled */
6220 I915_WRITE(GEN6_PMIER,
6221 GEN6_PM_MBOX_EVENT |
6222 GEN6_PM_THERMAL_EVENT |
6223 GEN6_PM_RP_DOWN_TIMEOUT |
6224 GEN6_PM_RP_UP_THRESHOLD |
6225 GEN6_PM_RP_DOWN_THRESHOLD |
6226 GEN6_PM_RP_UP_EI_EXPIRED |
6227 GEN6_PM_RP_DOWN_EI_EXPIRED);
6228 I915_WRITE(GEN6_PMIMR, 0);
6229 /* enable all PM interrupts */
6230 I915_WRITE(GEN6_PMINTRMSK, 0);
6231
6232 __gen6_force_wake_put(dev_priv);
6233}
6234
6235void intel_enable_clock_gating(struct drm_device *dev)
5826{ 6236{
5827 struct drm_i915_private *dev_priv = dev->dev_private; 6237 struct drm_i915_private *dev_priv = dev->dev_private;
5828 6238
@@ -5872,9 +6282,9 @@ void intel_init_clock_gating(struct drm_device *dev)
5872 I915_WRITE(DISP_ARB_CTL, 6282 I915_WRITE(DISP_ARB_CTL,
5873 (I915_READ(DISP_ARB_CTL) | 6283 (I915_READ(DISP_ARB_CTL) |
5874 DISP_FBC_WM_DIS)); 6284 DISP_FBC_WM_DIS));
5875 I915_WRITE(WM3_LP_ILK, 0); 6285 I915_WRITE(WM3_LP_ILK, 0);
5876 I915_WRITE(WM2_LP_ILK, 0); 6286 I915_WRITE(WM2_LP_ILK, 0);
5877 I915_WRITE(WM1_LP_ILK, 0); 6287 I915_WRITE(WM1_LP_ILK, 0);
5878 } 6288 }
5879 /* 6289 /*
5880 * Based on the document from hardware guys the following bits 6290 * Based on the document from hardware guys the following bits
@@ -5896,7 +6306,49 @@ void intel_init_clock_gating(struct drm_device *dev)
5896 ILK_DPFC_DIS2 | 6306 ILK_DPFC_DIS2 |
5897 ILK_CLK_FBC); 6307 ILK_CLK_FBC);
5898 } 6308 }
5899 return; 6309
6310 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6311 I915_READ(ILK_DISPLAY_CHICKEN2) |
6312 ILK_ELPIN_409_SELECT);
6313
6314 if (IS_GEN5(dev)) {
6315 I915_WRITE(_3D_CHICKEN2,
6316 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
6317 _3D_CHICKEN2_WM_READ_PIPELINED);
6318 }
6319
6320 if (IS_GEN6(dev)) {
6321 I915_WRITE(WM3_LP_ILK, 0);
6322 I915_WRITE(WM2_LP_ILK, 0);
6323 I915_WRITE(WM1_LP_ILK, 0);
6324
6325 /*
6326 * According to the spec the following bits should be
6327 * set in order to enable memory self-refresh and fbc:
6328 * The bit21 and bit22 of 0x42000
6329 * The bit21 and bit22 of 0x42004
6330 * The bit5 and bit7 of 0x42020
6331 * The bit14 of 0x70180
6332 * The bit14 of 0x71180
6333 */
6334 I915_WRITE(ILK_DISPLAY_CHICKEN1,
6335 I915_READ(ILK_DISPLAY_CHICKEN1) |
6336 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
6337 I915_WRITE(ILK_DISPLAY_CHICKEN2,
6338 I915_READ(ILK_DISPLAY_CHICKEN2) |
6339 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
6340 I915_WRITE(ILK_DSPCLK_GATE,
6341 I915_READ(ILK_DSPCLK_GATE) |
6342 ILK_DPARB_CLK_GATE |
6343 ILK_DPFD_CLK_GATE);
6344
6345 I915_WRITE(DSPACNTR,
6346 I915_READ(DSPACNTR) |
6347 DISPPLANE_TRICKLE_FEED_DISABLE);
6348 I915_WRITE(DSPBCNTR,
6349 I915_READ(DSPBCNTR) |
6350 DISPPLANE_TRICKLE_FEED_DISABLE);
6351 }
5900 } else if (IS_G4X(dev)) { 6352 } else if (IS_G4X(dev)) {
5901 uint32_t dspclk_gate; 6353 uint32_t dspclk_gate;
5902 I915_WRITE(RENCLK_GATE_D1, 0); 6354 I915_WRITE(RENCLK_GATE_D1, 0);
@@ -5939,20 +6391,18 @@ void intel_init_clock_gating(struct drm_device *dev)
5939 * GPU can automatically power down the render unit if given a page 6391 * GPU can automatically power down the render unit if given a page
5940 * to save state. 6392 * to save state.
5941 */ 6393 */
5942 if (IS_IRONLAKE_M(dev)) { 6394 if (IS_IRONLAKE_M(dev) && 0) { /* XXX causes a failure during suspend */
5943 if (dev_priv->renderctx == NULL) 6395 if (dev_priv->renderctx == NULL)
5944 dev_priv->renderctx = intel_alloc_context_page(dev); 6396 dev_priv->renderctx = intel_alloc_context_page(dev);
5945 if (dev_priv->renderctx) { 6397 if (dev_priv->renderctx) {
5946 struct drm_i915_gem_object *obj_priv; 6398 struct drm_i915_gem_object *obj = dev_priv->renderctx;
5947 obj_priv = to_intel_bo(dev_priv->renderctx); 6399 if (BEGIN_LP_RING(4) == 0) {
5948 if (obj_priv) {
5949 BEGIN_LP_RING(4);
5950 OUT_RING(MI_SET_CONTEXT); 6400 OUT_RING(MI_SET_CONTEXT);
5951 OUT_RING(obj_priv->gtt_offset | 6401 OUT_RING(obj->gtt_offset |
5952 MI_MM_SPACE_GTT | 6402 MI_MM_SPACE_GTT |
5953 MI_SAVE_EXT_STATE_EN | 6403 MI_SAVE_EXT_STATE_EN |
5954 MI_RESTORE_EXT_STATE_EN | 6404 MI_RESTORE_EXT_STATE_EN |
5955 MI_RESTORE_INHIBIT); 6405 MI_RESTORE_INHIBIT);
5956 OUT_RING(MI_NOOP); 6406 OUT_RING(MI_NOOP);
5957 OUT_RING(MI_FLUSH); 6407 OUT_RING(MI_FLUSH);
5958 ADVANCE_LP_RING(); 6408 ADVANCE_LP_RING();
@@ -5962,29 +6412,45 @@ void intel_init_clock_gating(struct drm_device *dev)
5962 "Disable RC6\n"); 6412 "Disable RC6\n");
5963 } 6413 }
5964 6414
5965 if (I915_HAS_RC6(dev) && drm_core_check_feature(dev, DRIVER_MODESET)) { 6415 if (IS_GEN4(dev) && IS_MOBILE(dev)) {
5966 struct drm_i915_gem_object *obj_priv = NULL; 6416 if (dev_priv->pwrctx == NULL)
5967 6417 dev_priv->pwrctx = intel_alloc_context_page(dev);
5968 if (dev_priv->pwrctx) { 6418 if (dev_priv->pwrctx) {
5969 obj_priv = to_intel_bo(dev_priv->pwrctx); 6419 struct drm_i915_gem_object *obj = dev_priv->pwrctx;
5970 } else { 6420 I915_WRITE(PWRCTXA, obj->gtt_offset | PWRCTX_EN);
5971 struct drm_gem_object *pwrctx;
5972
5973 pwrctx = intel_alloc_context_page(dev);
5974 if (pwrctx) {
5975 dev_priv->pwrctx = pwrctx;
5976 obj_priv = to_intel_bo(pwrctx);
5977 }
5978 }
5979
5980 if (obj_priv) {
5981 I915_WRITE(PWRCTXA, obj_priv->gtt_offset | PWRCTX_EN);
5982 I915_WRITE(MCHBAR_RENDER_STANDBY, 6421 I915_WRITE(MCHBAR_RENDER_STANDBY,
5983 I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT); 6422 I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT);
5984 } 6423 }
5985 } 6424 }
5986} 6425}
5987 6426
6427void intel_disable_clock_gating(struct drm_device *dev)
6428{
6429 struct drm_i915_private *dev_priv = dev->dev_private;
6430
6431 if (dev_priv->renderctx) {
6432 struct drm_i915_gem_object *obj = dev_priv->renderctx;
6433
6434 I915_WRITE(CCID, 0);
6435 POSTING_READ(CCID);
6436
6437 i915_gem_object_unpin(obj);
6438 drm_gem_object_unreference(&obj->base);
6439 dev_priv->renderctx = NULL;
6440 }
6441
6442 if (dev_priv->pwrctx) {
6443 struct drm_i915_gem_object *obj = dev_priv->pwrctx;
6444
6445 I915_WRITE(PWRCTXA, 0);
6446 POSTING_READ(PWRCTXA);
6447
6448 i915_gem_object_unpin(obj);
6449 drm_gem_object_unreference(&obj->base);
6450 dev_priv->pwrctx = NULL;
6451 }
6452}
6453
5988/* Set up chip specific display functions */ 6454/* Set up chip specific display functions */
5989static void intel_init_display(struct drm_device *dev) 6455static void intel_init_display(struct drm_device *dev)
5990{ 6456{
@@ -5997,7 +6463,7 @@ static void intel_init_display(struct drm_device *dev)
5997 dev_priv->display.dpms = i9xx_crtc_dpms; 6463 dev_priv->display.dpms = i9xx_crtc_dpms;
5998 6464
5999 if (I915_HAS_FBC(dev)) { 6465 if (I915_HAS_FBC(dev)) {
6000 if (IS_IRONLAKE_M(dev)) { 6466 if (HAS_PCH_SPLIT(dev)) {
6001 dev_priv->display.fbc_enabled = ironlake_fbc_enabled; 6467 dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
6002 dev_priv->display.enable_fbc = ironlake_enable_fbc; 6468 dev_priv->display.enable_fbc = ironlake_enable_fbc;
6003 dev_priv->display.disable_fbc = ironlake_disable_fbc; 6469 dev_priv->display.disable_fbc = ironlake_disable_fbc;
@@ -6046,6 +6512,14 @@ static void intel_init_display(struct drm_device *dev)
6046 "Disable CxSR\n"); 6512 "Disable CxSR\n");
6047 dev_priv->display.update_wm = NULL; 6513 dev_priv->display.update_wm = NULL;
6048 } 6514 }
6515 } else if (IS_GEN6(dev)) {
6516 if (SNB_READ_WM0_LATENCY()) {
6517 dev_priv->display.update_wm = sandybridge_update_wm;
6518 } else {
6519 DRM_DEBUG_KMS("Failed to read display plane latency. "
6520 "Disable CxSR\n");
6521 dev_priv->display.update_wm = NULL;
6522 }
6049 } else 6523 } else
6050 dev_priv->display.update_wm = NULL; 6524 dev_priv->display.update_wm = NULL;
6051 } else if (IS_PINEVIEW(dev)) { 6525 } else if (IS_PINEVIEW(dev)) {
@@ -6211,7 +6685,7 @@ void intel_modeset_init(struct drm_device *dev)
6211 6685
6212 intel_setup_outputs(dev); 6686 intel_setup_outputs(dev);
6213 6687
6214 intel_init_clock_gating(dev); 6688 intel_enable_clock_gating(dev);
6215 6689
6216 /* Just disable it once at startup */ 6690 /* Just disable it once at startup */
6217 i915_disable_vga(dev); 6691 i915_disable_vga(dev);
@@ -6221,6 +6695,9 @@ void intel_modeset_init(struct drm_device *dev)
6221 intel_init_emon(dev); 6695 intel_init_emon(dev);
6222 } 6696 }
6223 6697
6698 if (IS_GEN6(dev))
6699 gen6_enable_rps(dev_priv);
6700
6224 INIT_WORK(&dev_priv->idle_work, intel_idle_update); 6701 INIT_WORK(&dev_priv->idle_work, intel_idle_update);
6225 setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer, 6702 setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
6226 (unsigned long)dev); 6703 (unsigned long)dev);
@@ -6252,28 +6729,12 @@ void intel_modeset_cleanup(struct drm_device *dev)
6252 if (dev_priv->display.disable_fbc) 6729 if (dev_priv->display.disable_fbc)
6253 dev_priv->display.disable_fbc(dev); 6730 dev_priv->display.disable_fbc(dev);
6254 6731
6255 if (dev_priv->renderctx) {
6256 struct drm_i915_gem_object *obj_priv;
6257
6258 obj_priv = to_intel_bo(dev_priv->renderctx);
6259 I915_WRITE(CCID, obj_priv->gtt_offset &~ CCID_EN);
6260 I915_READ(CCID);
6261 i915_gem_object_unpin(dev_priv->renderctx);
6262 drm_gem_object_unreference(dev_priv->renderctx);
6263 }
6264
6265 if (dev_priv->pwrctx) {
6266 struct drm_i915_gem_object *obj_priv;
6267
6268 obj_priv = to_intel_bo(dev_priv->pwrctx);
6269 I915_WRITE(PWRCTXA, obj_priv->gtt_offset &~ PWRCTX_EN);
6270 I915_READ(PWRCTXA);
6271 i915_gem_object_unpin(dev_priv->pwrctx);
6272 drm_gem_object_unreference(dev_priv->pwrctx);
6273 }
6274
6275 if (IS_IRONLAKE_M(dev)) 6732 if (IS_IRONLAKE_M(dev))
6276 ironlake_disable_drps(dev); 6733 ironlake_disable_drps(dev);
6734 if (IS_GEN6(dev))
6735 gen6_disable_rps(dev);
6736
6737 intel_disable_clock_gating(dev);
6277 6738
6278 mutex_unlock(&dev->struct_mutex); 6739 mutex_unlock(&dev->struct_mutex);
6279 6740
@@ -6325,3 +6786,113 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
6325 pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl); 6786 pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl);
6326 return 0; 6787 return 0;
6327} 6788}
6789
6790#ifdef CONFIG_DEBUG_FS
6791#include <linux/seq_file.h>
6792
6793struct intel_display_error_state {
6794 struct intel_cursor_error_state {
6795 u32 control;
6796 u32 position;
6797 u32 base;
6798 u32 size;
6799 } cursor[2];
6800
6801 struct intel_pipe_error_state {
6802 u32 conf;
6803 u32 source;
6804
6805 u32 htotal;
6806 u32 hblank;
6807 u32 hsync;
6808 u32 vtotal;
6809 u32 vblank;
6810 u32 vsync;
6811 } pipe[2];
6812
6813 struct intel_plane_error_state {
6814 u32 control;
6815 u32 stride;
6816 u32 size;
6817 u32 pos;
6818 u32 addr;
6819 u32 surface;
6820 u32 tile_offset;
6821 } plane[2];
6822};
6823
6824struct intel_display_error_state *
6825intel_display_capture_error_state(struct drm_device *dev)
6826{
6827 drm_i915_private_t *dev_priv = dev->dev_private;
6828 struct intel_display_error_state *error;
6829 int i;
6830
6831 error = kmalloc(sizeof(*error), GFP_ATOMIC);
6832 if (error == NULL)
6833 return NULL;
6834
6835 for (i = 0; i < 2; i++) {
6836 error->cursor[i].control = I915_READ(CURCNTR(i));
6837 error->cursor[i].position = I915_READ(CURPOS(i));
6838 error->cursor[i].base = I915_READ(CURBASE(i));
6839
6840 error->plane[i].control = I915_READ(DSPCNTR(i));
6841 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
6842 error->plane[i].size = I915_READ(DSPSIZE(i));
6843 error->plane[i].pos= I915_READ(DSPPOS(i));
6844 error->plane[i].addr = I915_READ(DSPADDR(i));
6845 if (INTEL_INFO(dev)->gen >= 4) {
6846 error->plane[i].surface = I915_READ(DSPSURF(i));
6847 error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
6848 }
6849
6850 error->pipe[i].conf = I915_READ(PIPECONF(i));
6851 error->pipe[i].source = I915_READ(PIPESRC(i));
6852 error->pipe[i].htotal = I915_READ(HTOTAL(i));
6853 error->pipe[i].hblank = I915_READ(HBLANK(i));
6854 error->pipe[i].hsync = I915_READ(HSYNC(i));
6855 error->pipe[i].vtotal = I915_READ(VTOTAL(i));
6856 error->pipe[i].vblank = I915_READ(VBLANK(i));
6857 error->pipe[i].vsync = I915_READ(VSYNC(i));
6858 }
6859
6860 return error;
6861}
6862
6863void
6864intel_display_print_error_state(struct seq_file *m,
6865 struct drm_device *dev,
6866 struct intel_display_error_state *error)
6867{
6868 int i;
6869
6870 for (i = 0; i < 2; i++) {
6871 seq_printf(m, "Pipe [%d]:\n", i);
6872 seq_printf(m, " CONF: %08x\n", error->pipe[i].conf);
6873 seq_printf(m, " SRC: %08x\n", error->pipe[i].source);
6874 seq_printf(m, " HTOTAL: %08x\n", error->pipe[i].htotal);
6875 seq_printf(m, " HBLANK: %08x\n", error->pipe[i].hblank);
6876 seq_printf(m, " HSYNC: %08x\n", error->pipe[i].hsync);
6877 seq_printf(m, " VTOTAL: %08x\n", error->pipe[i].vtotal);
6878 seq_printf(m, " VBLANK: %08x\n", error->pipe[i].vblank);
6879 seq_printf(m, " VSYNC: %08x\n", error->pipe[i].vsync);
6880
6881 seq_printf(m, "Plane [%d]:\n", i);
6882 seq_printf(m, " CNTR: %08x\n", error->plane[i].control);
6883 seq_printf(m, " STRIDE: %08x\n", error->plane[i].stride);
6884 seq_printf(m, " SIZE: %08x\n", error->plane[i].size);
6885 seq_printf(m, " POS: %08x\n", error->plane[i].pos);
6886 seq_printf(m, " ADDR: %08x\n", error->plane[i].addr);
6887 if (INTEL_INFO(dev)->gen >= 4) {
6888 seq_printf(m, " SURF: %08x\n", error->plane[i].surface);
6889 seq_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset);
6890 }
6891
6892 seq_printf(m, "Cursor [%d]:\n", i);
6893 seq_printf(m, " CNTR: %08x\n", error->cursor[i].control);
6894 seq_printf(m, " POS: %08x\n", error->cursor[i].position);
6895 seq_printf(m, " BASE: %08x\n", error->cursor[i].base);
6896 }
6897}
6898#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 864417cffe9a..1dc60408d5b8 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -1442,8 +1442,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
1442 /* Changes to enable or select take place the vblank 1442 /* Changes to enable or select take place the vblank
1443 * after being written. 1443 * after being written.
1444 */ 1444 */
1445 intel_wait_for_vblank(intel_dp->base.base.dev, 1445 intel_wait_for_vblank(dev, intel_crtc->pipe);
1446 intel_crtc->pipe);
1447 } 1446 }
1448 1447
1449 I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); 1448 I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index e52c6125bb1f..d782ad9fd6db 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -127,7 +127,7 @@ intel_mode_get_pixel_multiplier(const struct drm_display_mode *mode)
127 127
128struct intel_framebuffer { 128struct intel_framebuffer {
129 struct drm_framebuffer base; 129 struct drm_framebuffer base;
130 struct drm_gem_object *obj; 130 struct drm_i915_gem_object *obj;
131}; 131};
132 132
133struct intel_fbdev { 133struct intel_fbdev {
@@ -166,7 +166,7 @@ struct intel_crtc {
166 struct intel_unpin_work *unpin_work; 166 struct intel_unpin_work *unpin_work;
167 int fdi_lanes; 167 int fdi_lanes;
168 168
169 struct drm_gem_object *cursor_bo; 169 struct drm_i915_gem_object *cursor_bo;
170 uint32_t cursor_addr; 170 uint32_t cursor_addr;
171 int16_t cursor_x, cursor_y; 171 int16_t cursor_x, cursor_y;
172 int16_t cursor_width, cursor_height; 172 int16_t cursor_width, cursor_height;
@@ -220,8 +220,8 @@ intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
220struct intel_unpin_work { 220struct intel_unpin_work {
221 struct work_struct work; 221 struct work_struct work;
222 struct drm_device *dev; 222 struct drm_device *dev;
223 struct drm_gem_object *old_fb_obj; 223 struct drm_i915_gem_object *old_fb_obj;
224 struct drm_gem_object *pending_flip_obj; 224 struct drm_i915_gem_object *pending_flip_obj;
225 struct drm_pending_vblank_event *event; 225 struct drm_pending_vblank_event *event;
226 int pending; 226 int pending;
227 bool enable_stall_check; 227 bool enable_stall_check;
@@ -236,7 +236,8 @@ void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
236extern bool intel_sdvo_init(struct drm_device *dev, int output_device); 236extern bool intel_sdvo_init(struct drm_device *dev, int output_device);
237extern void intel_dvo_init(struct drm_device *dev); 237extern void intel_dvo_init(struct drm_device *dev);
238extern void intel_tv_init(struct drm_device *dev); 238extern void intel_tv_init(struct drm_device *dev);
239extern void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj); 239extern void intel_mark_busy(struct drm_device *dev,
240 struct drm_i915_gem_object *obj);
240extern bool intel_lvds_init(struct drm_device *dev); 241extern bool intel_lvds_init(struct drm_device *dev);
241extern void intel_dp_init(struct drm_device *dev, int dp_reg); 242extern void intel_dp_init(struct drm_device *dev, int dp_reg);
242void 243void
@@ -293,19 +294,22 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
293 u16 blue, int regno); 294 u16 blue, int regno);
294extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green, 295extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
295 u16 *blue, int regno); 296 u16 *blue, int regno);
296extern void intel_init_clock_gating(struct drm_device *dev); 297extern void intel_enable_clock_gating(struct drm_device *dev);
298extern void intel_disable_clock_gating(struct drm_device *dev);
297extern void ironlake_enable_drps(struct drm_device *dev); 299extern void ironlake_enable_drps(struct drm_device *dev);
298extern void ironlake_disable_drps(struct drm_device *dev); 300extern void ironlake_disable_drps(struct drm_device *dev);
301extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
302extern void gen6_disable_rps(struct drm_device *dev);
299extern void intel_init_emon(struct drm_device *dev); 303extern void intel_init_emon(struct drm_device *dev);
300 304
301extern int intel_pin_and_fence_fb_obj(struct drm_device *dev, 305extern int intel_pin_and_fence_fb_obj(struct drm_device *dev,
302 struct drm_gem_object *obj, 306 struct drm_i915_gem_object *obj,
303 bool pipelined); 307 struct intel_ring_buffer *pipelined);
304 308
305extern int intel_framebuffer_init(struct drm_device *dev, 309extern int intel_framebuffer_init(struct drm_device *dev,
306 struct intel_framebuffer *ifb, 310 struct intel_framebuffer *ifb,
307 struct drm_mode_fb_cmd *mode_cmd, 311 struct drm_mode_fb_cmd *mode_cmd,
308 struct drm_gem_object *obj); 312 struct drm_i915_gem_object *obj);
309extern int intel_fbdev_init(struct drm_device *dev); 313extern int intel_fbdev_init(struct drm_device *dev);
310extern void intel_fbdev_fini(struct drm_device *dev); 314extern void intel_fbdev_fini(struct drm_device *dev);
311 315
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index af2a1dddc28e..701e830d0012 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -65,10 +65,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
65 struct fb_info *info; 65 struct fb_info *info;
66 struct drm_framebuffer *fb; 66 struct drm_framebuffer *fb;
67 struct drm_mode_fb_cmd mode_cmd; 67 struct drm_mode_fb_cmd mode_cmd;
68 struct drm_gem_object *fbo = NULL; 68 struct drm_i915_gem_object *obj;
69 struct drm_i915_gem_object *obj_priv;
70 struct device *device = &dev->pdev->dev; 69 struct device *device = &dev->pdev->dev;
71 int size, ret, mmio_bar = IS_GEN2(dev) ? 1 : 0; 70 int size, ret;
72 71
73 /* we don't do packed 24bpp */ 72 /* we don't do packed 24bpp */
74 if (sizes->surface_bpp == 24) 73 if (sizes->surface_bpp == 24)
@@ -83,18 +82,17 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
83 82
84 size = mode_cmd.pitch * mode_cmd.height; 83 size = mode_cmd.pitch * mode_cmd.height;
85 size = ALIGN(size, PAGE_SIZE); 84 size = ALIGN(size, PAGE_SIZE);
86 fbo = i915_gem_alloc_object(dev, size); 85 obj = i915_gem_alloc_object(dev, size);
87 if (!fbo) { 86 if (!obj) {
88 DRM_ERROR("failed to allocate framebuffer\n"); 87 DRM_ERROR("failed to allocate framebuffer\n");
89 ret = -ENOMEM; 88 ret = -ENOMEM;
90 goto out; 89 goto out;
91 } 90 }
92 obj_priv = to_intel_bo(fbo);
93 91
94 mutex_lock(&dev->struct_mutex); 92 mutex_lock(&dev->struct_mutex);
95 93
96 /* Flush everything out, we'll be doing GTT only from now on */ 94 /* Flush everything out, we'll be doing GTT only from now on */
97 ret = intel_pin_and_fence_fb_obj(dev, fbo, false); 95 ret = intel_pin_and_fence_fb_obj(dev, obj, false);
98 if (ret) { 96 if (ret) {
99 DRM_ERROR("failed to pin fb: %d\n", ret); 97 DRM_ERROR("failed to pin fb: %d\n", ret);
100 goto out_unref; 98 goto out_unref;
@@ -108,7 +106,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
108 106
109 info->par = ifbdev; 107 info->par = ifbdev;
110 108
111 ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, fbo); 109 ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
112 if (ret) 110 if (ret)
113 goto out_unpin; 111 goto out_unpin;
114 112
@@ -134,11 +132,10 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
134 else 132 else
135 info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0); 133 info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
136 134
137 info->fix.smem_start = dev->mode_config.fb_base + obj_priv->gtt_offset; 135 info->fix.smem_start = dev->mode_config.fb_base + obj->gtt_offset;
138 info->fix.smem_len = size; 136 info->fix.smem_len = size;
139 137
140 info->screen_base = ioremap_wc(dev->agp->base + obj_priv->gtt_offset, 138 info->screen_base = ioremap_wc(dev->agp->base + obj->gtt_offset, size);
141 size);
142 if (!info->screen_base) { 139 if (!info->screen_base) {
143 ret = -ENOSPC; 140 ret = -ENOSPC;
144 goto out_unpin; 141 goto out_unpin;
@@ -153,13 +150,8 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
153 150
154// memset(info->screen_base, 0, size); 151// memset(info->screen_base, 0, size);
155 152
156 drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
157 drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height); 153 drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height);
158 154
159 /* FIXME: we really shouldn't expose mmio space at all */
160 info->fix.mmio_start = pci_resource_start(dev->pdev, mmio_bar);
161 info->fix.mmio_len = pci_resource_len(dev->pdev, mmio_bar);
162
163 info->pixmap.size = 64*1024; 155 info->pixmap.size = 64*1024;
164 info->pixmap.buf_align = 8; 156 info->pixmap.buf_align = 8;
165 info->pixmap.access_align = 32; 157 info->pixmap.access_align = 32;
@@ -168,7 +160,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
168 160
169 DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n", 161 DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n",
170 fb->width, fb->height, 162 fb->width, fb->height,
171 obj_priv->gtt_offset, fbo); 163 obj->gtt_offset, obj);
172 164
173 165
174 mutex_unlock(&dev->struct_mutex); 166 mutex_unlock(&dev->struct_mutex);
@@ -176,9 +168,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
176 return 0; 168 return 0;
177 169
178out_unpin: 170out_unpin:
179 i915_gem_object_unpin(fbo); 171 i915_gem_object_unpin(obj);
180out_unref: 172out_unref:
181 drm_gem_object_unreference(fbo); 173 drm_gem_object_unreference(&obj->base);
182 mutex_unlock(&dev->struct_mutex); 174 mutex_unlock(&dev->struct_mutex);
183out: 175out:
184 return ret; 176 return ret;
@@ -225,7 +217,7 @@ static void intel_fbdev_destroy(struct drm_device *dev,
225 217
226 drm_framebuffer_cleanup(&ifb->base); 218 drm_framebuffer_cleanup(&ifb->base);
227 if (ifb->obj) { 219 if (ifb->obj) {
228 drm_gem_object_unreference_unlocked(ifb->obj); 220 drm_gem_object_unreference_unlocked(&ifb->obj->base);
229 ifb->obj = NULL; 221 ifb->obj = NULL;
230 } 222 }
231} 223}
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 3dba086e7eea..58040f68ed7a 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -85,8 +85,9 @@ static u32 get_reserved(struct intel_gpio *gpio)
85 85
86 /* On most chips, these bits must be preserved in software. */ 86 /* On most chips, these bits must be preserved in software. */
87 if (!IS_I830(dev) && !IS_845G(dev)) 87 if (!IS_I830(dev) && !IS_845G(dev))
88 reserved = I915_READ(gpio->reg) & (GPIO_DATA_PULLUP_DISABLE | 88 reserved = I915_READ_NOTRACE(gpio->reg) &
89 GPIO_CLOCK_PULLUP_DISABLE); 89 (GPIO_DATA_PULLUP_DISABLE |
90 GPIO_CLOCK_PULLUP_DISABLE);
90 91
91 return reserved; 92 return reserved;
92} 93}
@@ -96,9 +97,9 @@ static int get_clock(void *data)
96 struct intel_gpio *gpio = data; 97 struct intel_gpio *gpio = data;
97 struct drm_i915_private *dev_priv = gpio->dev_priv; 98 struct drm_i915_private *dev_priv = gpio->dev_priv;
98 u32 reserved = get_reserved(gpio); 99 u32 reserved = get_reserved(gpio);
99 I915_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK); 100 I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
100 I915_WRITE(gpio->reg, reserved); 101 I915_WRITE_NOTRACE(gpio->reg, reserved);
101 return (I915_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0; 102 return (I915_READ_NOTRACE(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
102} 103}
103 104
104static int get_data(void *data) 105static int get_data(void *data)
@@ -106,9 +107,9 @@ static int get_data(void *data)
106 struct intel_gpio *gpio = data; 107 struct intel_gpio *gpio = data;
107 struct drm_i915_private *dev_priv = gpio->dev_priv; 108 struct drm_i915_private *dev_priv = gpio->dev_priv;
108 u32 reserved = get_reserved(gpio); 109 u32 reserved = get_reserved(gpio);
109 I915_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK); 110 I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
110 I915_WRITE(gpio->reg, reserved); 111 I915_WRITE_NOTRACE(gpio->reg, reserved);
111 return (I915_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0; 112 return (I915_READ_NOTRACE(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
112} 113}
113 114
114static void set_clock(void *data, int state_high) 115static void set_clock(void *data, int state_high)
@@ -124,7 +125,7 @@ static void set_clock(void *data, int state_high)
124 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK | 125 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
125 GPIO_CLOCK_VAL_MASK; 126 GPIO_CLOCK_VAL_MASK;
126 127
127 I915_WRITE(gpio->reg, reserved | clock_bits); 128 I915_WRITE_NOTRACE(gpio->reg, reserved | clock_bits);
128 POSTING_READ(gpio->reg); 129 POSTING_READ(gpio->reg);
129} 130}
130 131
@@ -141,7 +142,7 @@ static void set_data(void *data, int state_high)
141 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK | 142 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
142 GPIO_DATA_VAL_MASK; 143 GPIO_DATA_VAL_MASK;
143 144
144 I915_WRITE(gpio->reg, reserved | data_bits); 145 I915_WRITE_NOTRACE(gpio->reg, reserved | data_bits);
145 POSTING_READ(gpio->reg); 146 POSTING_READ(gpio->reg);
146} 147}
147 148
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 25bcedf386fd..aa2307080be2 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -304,14 +304,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
304 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; 304 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
305 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; 305 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
306 306
307 pfit_control |= PFIT_ENABLE;
308 /* 965+ is easy, it does everything in hw */ 307 /* 965+ is easy, it does everything in hw */
309 if (scaled_width > scaled_height) 308 if (scaled_width > scaled_height)
310 pfit_control |= PFIT_SCALING_PILLAR; 309 pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR;
311 else if (scaled_width < scaled_height) 310 else if (scaled_width < scaled_height)
312 pfit_control |= PFIT_SCALING_LETTER; 311 pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER;
313 else 312 else if (adjusted_mode->hdisplay != mode->hdisplay)
314 pfit_control |= PFIT_SCALING_AUTO; 313 pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO;
315 } else { 314 } else {
316 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; 315 u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay;
317 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; 316 u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay;
@@ -358,13 +357,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
358 * Full scaling, even if it changes the aspect ratio. 357 * Full scaling, even if it changes the aspect ratio.
359 * Fortunately this is all done for us in hw. 358 * Fortunately this is all done for us in hw.
360 */ 359 */
361 pfit_control |= PFIT_ENABLE; 360 if (mode->vdisplay != adjusted_mode->vdisplay ||
362 if (INTEL_INFO(dev)->gen >= 4) 361 mode->hdisplay != adjusted_mode->hdisplay) {
363 pfit_control |= PFIT_SCALING_AUTO; 362 pfit_control |= PFIT_ENABLE;
364 else 363 if (INTEL_INFO(dev)->gen >= 4)
365 pfit_control |= (VERT_AUTO_SCALE | HORIZ_AUTO_SCALE | 364 pfit_control |= PFIT_SCALING_AUTO;
366 VERT_INTERP_BILINEAR | 365 else
367 HORIZ_INTERP_BILINEAR); 366 pfit_control |= (VERT_AUTO_SCALE |
367 VERT_INTERP_BILINEAR |
368 HORIZ_AUTO_SCALE |
369 HORIZ_INTERP_BILINEAR);
370 }
368 break; 371 break;
369 372
370 default: 373 default:
@@ -914,6 +917,8 @@ bool intel_lvds_init(struct drm_device *dev)
914 917
915 intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT); 918 intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT);
916 intel_encoder->crtc_mask = (1 << 1); 919 intel_encoder->crtc_mask = (1 << 1);
920 if (INTEL_INFO(dev)->gen >= 5)
921 intel_encoder->crtc_mask |= (1 << 0);
917 drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs); 922 drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs);
918 drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs); 923 drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs);
919 connector->display_info.subpixel_order = SubPixelHorizontalRGB; 924 connector->display_info.subpixel_order = SubPixelHorizontalRGB;
@@ -1019,10 +1024,18 @@ bool intel_lvds_init(struct drm_device *dev)
1019out: 1024out:
1020 if (HAS_PCH_SPLIT(dev)) { 1025 if (HAS_PCH_SPLIT(dev)) {
1021 u32 pwm; 1026 u32 pwm;
1022 /* make sure PWM is enabled */ 1027
1028 pipe = (I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) ? 1 : 0;
1029
1030 /* make sure PWM is enabled and locked to the LVDS pipe */
1023 pwm = I915_READ(BLC_PWM_CPU_CTL2); 1031 pwm = I915_READ(BLC_PWM_CPU_CTL2);
1024 pwm |= (PWM_ENABLE | PWM_PIPE_B); 1032 if (pipe == 0 && (pwm & PWM_PIPE_B))
1025 I915_WRITE(BLC_PWM_CPU_CTL2, pwm); 1033 I915_WRITE(BLC_PWM_CPU_CTL2, pwm & ~PWM_ENABLE);
1034 if (pipe)
1035 pwm |= PWM_PIPE_B;
1036 else
1037 pwm &= ~PWM_PIPE_B;
1038 I915_WRITE(BLC_PWM_CPU_CTL2, pwm | PWM_ENABLE);
1026 1039
1027 pwm = I915_READ(BLC_PWM_PCH_CTL1); 1040 pwm = I915_READ(BLC_PWM_PCH_CTL1);
1028 pwm |= PWM_PCH_ENABLE; 1041 pwm |= PWM_PCH_ENABLE;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 9b0d9a867aea..f295a7aaadf9 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -273,14 +273,8 @@ void intel_opregion_enable_asle(struct drm_device *dev)
273 struct opregion_asle *asle = dev_priv->opregion.asle; 273 struct opregion_asle *asle = dev_priv->opregion.asle;
274 274
275 if (asle) { 275 if (asle) {
276 if (IS_MOBILE(dev)) { 276 if (IS_MOBILE(dev))
277 unsigned long irqflags;
278
279 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
280 intel_enable_asle(dev); 277 intel_enable_asle(dev);
281 spin_unlock_irqrestore(&dev_priv->user_irq_lock,
282 irqflags);
283 }
284 278
285 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN | 279 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
286 ASLE_PFMB_EN; 280 ASLE_PFMB_EN;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 02ff0a481f47..3fbb98b948d6 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -221,15 +221,16 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
221 int ret; 221 int ret;
222 222
223 BUG_ON(overlay->last_flip_req); 223 BUG_ON(overlay->last_flip_req);
224 overlay->last_flip_req = 224 ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
225 i915_add_request(dev, NULL, request, &dev_priv->render_ring); 225 if (ret) {
226 if (overlay->last_flip_req == 0) 226 kfree(request);
227 return -ENOMEM; 227 return ret;
228 228 }
229 overlay->last_flip_req = request->seqno;
229 overlay->flip_tail = tail; 230 overlay->flip_tail = tail;
230 ret = i915_do_wait_request(dev, 231 ret = i915_do_wait_request(dev,
231 overlay->last_flip_req, true, 232 overlay->last_flip_req, true,
232 &dev_priv->render_ring); 233 LP_RING(dev_priv));
233 if (ret) 234 if (ret)
234 return ret; 235 return ret;
235 236
@@ -289,6 +290,7 @@ i830_deactivate_pipe_a(struct drm_device *dev)
289static int intel_overlay_on(struct intel_overlay *overlay) 290static int intel_overlay_on(struct intel_overlay *overlay)
290{ 291{
291 struct drm_device *dev = overlay->dev; 292 struct drm_device *dev = overlay->dev;
293 struct drm_i915_private *dev_priv = dev->dev_private;
292 struct drm_i915_gem_request *request; 294 struct drm_i915_gem_request *request;
293 int pipe_a_quirk = 0; 295 int pipe_a_quirk = 0;
294 int ret; 296 int ret;
@@ -308,7 +310,12 @@ static int intel_overlay_on(struct intel_overlay *overlay)
308 goto out; 310 goto out;
309 } 311 }
310 312
311 BEGIN_LP_RING(4); 313 ret = BEGIN_LP_RING(4);
314 if (ret) {
315 kfree(request);
316 goto out;
317 }
318
312 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON); 319 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON);
313 OUT_RING(overlay->flip_addr | OFC_UPDATE); 320 OUT_RING(overlay->flip_addr | OFC_UPDATE);
314 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); 321 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
@@ -332,6 +339,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
332 struct drm_i915_gem_request *request; 339 struct drm_i915_gem_request *request;
333 u32 flip_addr = overlay->flip_addr; 340 u32 flip_addr = overlay->flip_addr;
334 u32 tmp; 341 u32 tmp;
342 int ret;
335 343
336 BUG_ON(!overlay->active); 344 BUG_ON(!overlay->active);
337 345
@@ -347,36 +355,44 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
347 if (tmp & (1 << 17)) 355 if (tmp & (1 << 17))
348 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); 356 DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
349 357
350 BEGIN_LP_RING(2); 358 ret = BEGIN_LP_RING(2);
359 if (ret) {
360 kfree(request);
361 return ret;
362 }
351 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); 363 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
352 OUT_RING(flip_addr); 364 OUT_RING(flip_addr);
353 ADVANCE_LP_RING(); 365 ADVANCE_LP_RING();
354 366
355 overlay->last_flip_req = 367 ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv));
356 i915_add_request(dev, NULL, request, &dev_priv->render_ring); 368 if (ret) {
369 kfree(request);
370 return ret;
371 }
372
373 overlay->last_flip_req = request->seqno;
357 return 0; 374 return 0;
358} 375}
359 376
360static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) 377static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
361{ 378{
362 struct drm_gem_object *obj = &overlay->old_vid_bo->base; 379 struct drm_i915_gem_object *obj = overlay->old_vid_bo;
363 380
364 i915_gem_object_unpin(obj); 381 i915_gem_object_unpin(obj);
365 drm_gem_object_unreference(obj); 382 drm_gem_object_unreference(&obj->base);
366 383
367 overlay->old_vid_bo = NULL; 384 overlay->old_vid_bo = NULL;
368} 385}
369 386
370static void intel_overlay_off_tail(struct intel_overlay *overlay) 387static void intel_overlay_off_tail(struct intel_overlay *overlay)
371{ 388{
372 struct drm_gem_object *obj; 389 struct drm_i915_gem_object *obj = overlay->vid_bo;
373 390
374 /* never have the overlay hw on without showing a frame */ 391 /* never have the overlay hw on without showing a frame */
375 BUG_ON(!overlay->vid_bo); 392 BUG_ON(!overlay->vid_bo);
376 obj = &overlay->vid_bo->base;
377 393
378 i915_gem_object_unpin(obj); 394 i915_gem_object_unpin(obj);
379 drm_gem_object_unreference(obj); 395 drm_gem_object_unreference(&obj->base);
380 overlay->vid_bo = NULL; 396 overlay->vid_bo = NULL;
381 397
382 overlay->crtc->overlay = NULL; 398 overlay->crtc->overlay = NULL;
@@ -389,8 +405,10 @@ static int intel_overlay_off(struct intel_overlay *overlay,
389 bool interruptible) 405 bool interruptible)
390{ 406{
391 struct drm_device *dev = overlay->dev; 407 struct drm_device *dev = overlay->dev;
408 struct drm_i915_private *dev_priv = dev->dev_private;
392 u32 flip_addr = overlay->flip_addr; 409 u32 flip_addr = overlay->flip_addr;
393 struct drm_i915_gem_request *request; 410 struct drm_i915_gem_request *request;
411 int ret;
394 412
395 BUG_ON(!overlay->active); 413 BUG_ON(!overlay->active);
396 414
@@ -404,7 +422,11 @@ static int intel_overlay_off(struct intel_overlay *overlay,
404 * of the hw. Do it in both cases */ 422 * of the hw. Do it in both cases */
405 flip_addr |= OFC_UPDATE; 423 flip_addr |= OFC_UPDATE;
406 424
407 BEGIN_LP_RING(6); 425 ret = BEGIN_LP_RING(6);
426 if (ret) {
427 kfree(request);
428 return ret;
429 }
408 /* wait for overlay to go idle */ 430 /* wait for overlay to go idle */
409 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); 431 OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
410 OUT_RING(flip_addr); 432 OUT_RING(flip_addr);
@@ -432,7 +454,7 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay,
432 return 0; 454 return 0;
433 455
434 ret = i915_do_wait_request(dev, overlay->last_flip_req, 456 ret = i915_do_wait_request(dev, overlay->last_flip_req,
435 interruptible, &dev_priv->render_ring); 457 interruptible, LP_RING(dev_priv));
436 if (ret) 458 if (ret)
437 return ret; 459 return ret;
438 460
@@ -467,7 +489,12 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
467 if (request == NULL) 489 if (request == NULL)
468 return -ENOMEM; 490 return -ENOMEM;
469 491
470 BEGIN_LP_RING(2); 492 ret = BEGIN_LP_RING(2);
493 if (ret) {
494 kfree(request);
495 return ret;
496 }
497
471 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); 498 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
472 OUT_RING(MI_NOOP); 499 OUT_RING(MI_NOOP);
473 ADVANCE_LP_RING(); 500 ADVANCE_LP_RING();
@@ -736,13 +763,12 @@ static u32 overlay_cmd_reg(struct put_image_params *params)
736} 763}
737 764
738static int intel_overlay_do_put_image(struct intel_overlay *overlay, 765static int intel_overlay_do_put_image(struct intel_overlay *overlay,
739 struct drm_gem_object *new_bo, 766 struct drm_i915_gem_object *new_bo,
740 struct put_image_params *params) 767 struct put_image_params *params)
741{ 768{
742 int ret, tmp_width; 769 int ret, tmp_width;
743 struct overlay_registers *regs; 770 struct overlay_registers *regs;
744 bool scale_changed = false; 771 bool scale_changed = false;
745 struct drm_i915_gem_object *bo_priv = to_intel_bo(new_bo);
746 struct drm_device *dev = overlay->dev; 772 struct drm_device *dev = overlay->dev;
747 773
748 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 774 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -753,7 +779,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
753 if (ret != 0) 779 if (ret != 0)
754 return ret; 780 return ret;
755 781
756 ret = i915_gem_object_pin(new_bo, PAGE_SIZE); 782 ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true);
757 if (ret != 0) 783 if (ret != 0)
758 return ret; 784 return ret;
759 785
@@ -761,6 +787,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
761 if (ret != 0) 787 if (ret != 0)
762 goto out_unpin; 788 goto out_unpin;
763 789
790 ret = i915_gem_object_put_fence(new_bo);
791 if (ret)
792 goto out_unpin;
793
764 if (!overlay->active) { 794 if (!overlay->active) {
765 regs = intel_overlay_map_regs(overlay); 795 regs = intel_overlay_map_regs(overlay);
766 if (!regs) { 796 if (!regs) {
@@ -797,7 +827,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
797 regs->SWIDTHSW = calc_swidthsw(overlay->dev, 827 regs->SWIDTHSW = calc_swidthsw(overlay->dev,
798 params->offset_Y, tmp_width); 828 params->offset_Y, tmp_width);
799 regs->SHEIGHT = params->src_h; 829 regs->SHEIGHT = params->src_h;
800 regs->OBUF_0Y = bo_priv->gtt_offset + params-> offset_Y; 830 regs->OBUF_0Y = new_bo->gtt_offset + params-> offset_Y;
801 regs->OSTRIDE = params->stride_Y; 831 regs->OSTRIDE = params->stride_Y;
802 832
803 if (params->format & I915_OVERLAY_YUV_PLANAR) { 833 if (params->format & I915_OVERLAY_YUV_PLANAR) {
@@ -811,8 +841,8 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
811 params->src_w/uv_hscale); 841 params->src_w/uv_hscale);
812 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16; 842 regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16;
813 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16; 843 regs->SHEIGHT |= (params->src_h/uv_vscale) << 16;
814 regs->OBUF_0U = bo_priv->gtt_offset + params->offset_U; 844 regs->OBUF_0U = new_bo->gtt_offset + params->offset_U;
815 regs->OBUF_0V = bo_priv->gtt_offset + params->offset_V; 845 regs->OBUF_0V = new_bo->gtt_offset + params->offset_V;
816 regs->OSTRIDE |= params->stride_UV << 16; 846 regs->OSTRIDE |= params->stride_UV << 16;
817 } 847 }
818 848
@@ -829,7 +859,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
829 goto out_unpin; 859 goto out_unpin;
830 860
831 overlay->old_vid_bo = overlay->vid_bo; 861 overlay->old_vid_bo = overlay->vid_bo;
832 overlay->vid_bo = to_intel_bo(new_bo); 862 overlay->vid_bo = new_bo;
833 863
834 return 0; 864 return 0;
835 865
@@ -942,7 +972,7 @@ static int check_overlay_scaling(struct put_image_params *rec)
942 972
943static int check_overlay_src(struct drm_device *dev, 973static int check_overlay_src(struct drm_device *dev,
944 struct drm_intel_overlay_put_image *rec, 974 struct drm_intel_overlay_put_image *rec,
945 struct drm_gem_object *new_bo) 975 struct drm_i915_gem_object *new_bo)
946{ 976{
947 int uv_hscale = uv_hsubsampling(rec->flags); 977 int uv_hscale = uv_hsubsampling(rec->flags);
948 int uv_vscale = uv_vsubsampling(rec->flags); 978 int uv_vscale = uv_vsubsampling(rec->flags);
@@ -1027,7 +1057,7 @@ static int check_overlay_src(struct drm_device *dev,
1027 return -EINVAL; 1057 return -EINVAL;
1028 1058
1029 tmp = rec->stride_Y*rec->src_height; 1059 tmp = rec->stride_Y*rec->src_height;
1030 if (rec->offset_Y + tmp > new_bo->size) 1060 if (rec->offset_Y + tmp > new_bo->base.size)
1031 return -EINVAL; 1061 return -EINVAL;
1032 break; 1062 break;
1033 1063
@@ -1038,12 +1068,12 @@ static int check_overlay_src(struct drm_device *dev,
1038 return -EINVAL; 1068 return -EINVAL;
1039 1069
1040 tmp = rec->stride_Y * rec->src_height; 1070 tmp = rec->stride_Y * rec->src_height;
1041 if (rec->offset_Y + tmp > new_bo->size) 1071 if (rec->offset_Y + tmp > new_bo->base.size)
1042 return -EINVAL; 1072 return -EINVAL;
1043 1073
1044 tmp = rec->stride_UV * (rec->src_height / uv_vscale); 1074 tmp = rec->stride_UV * (rec->src_height / uv_vscale);
1045 if (rec->offset_U + tmp > new_bo->size || 1075 if (rec->offset_U + tmp > new_bo->base.size ||
1046 rec->offset_V + tmp > new_bo->size) 1076 rec->offset_V + tmp > new_bo->base.size)
1047 return -EINVAL; 1077 return -EINVAL;
1048 break; 1078 break;
1049 } 1079 }
@@ -1086,7 +1116,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1086 struct intel_overlay *overlay; 1116 struct intel_overlay *overlay;
1087 struct drm_mode_object *drmmode_obj; 1117 struct drm_mode_object *drmmode_obj;
1088 struct intel_crtc *crtc; 1118 struct intel_crtc *crtc;
1089 struct drm_gem_object *new_bo; 1119 struct drm_i915_gem_object *new_bo;
1090 struct put_image_params *params; 1120 struct put_image_params *params;
1091 int ret; 1121 int ret;
1092 1122
@@ -1125,8 +1155,8 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1125 } 1155 }
1126 crtc = to_intel_crtc(obj_to_crtc(drmmode_obj)); 1156 crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
1127 1157
1128 new_bo = drm_gem_object_lookup(dev, file_priv, 1158 new_bo = to_intel_bo(drm_gem_object_lookup(dev, file_priv,
1129 put_image_rec->bo_handle); 1159 put_image_rec->bo_handle));
1130 if (!new_bo) { 1160 if (!new_bo) {
1131 ret = -ENOENT; 1161 ret = -ENOENT;
1132 goto out_free; 1162 goto out_free;
@@ -1135,6 +1165,12 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1135 mutex_lock(&dev->mode_config.mutex); 1165 mutex_lock(&dev->mode_config.mutex);
1136 mutex_lock(&dev->struct_mutex); 1166 mutex_lock(&dev->struct_mutex);
1137 1167
1168 if (new_bo->tiling_mode) {
1169 DRM_ERROR("buffer used for overlay image can not be tiled\n");
1170 ret = -EINVAL;
1171 goto out_unlock;
1172 }
1173
1138 ret = intel_overlay_recover_from_interrupt(overlay, true); 1174 ret = intel_overlay_recover_from_interrupt(overlay, true);
1139 if (ret != 0) 1175 if (ret != 0)
1140 goto out_unlock; 1176 goto out_unlock;
@@ -1217,7 +1253,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
1217out_unlock: 1253out_unlock:
1218 mutex_unlock(&dev->struct_mutex); 1254 mutex_unlock(&dev->struct_mutex);
1219 mutex_unlock(&dev->mode_config.mutex); 1255 mutex_unlock(&dev->mode_config.mutex);
1220 drm_gem_object_unreference_unlocked(new_bo); 1256 drm_gem_object_unreference_unlocked(&new_bo->base);
1221out_free: 1257out_free:
1222 kfree(params); 1258 kfree(params);
1223 1259
@@ -1370,7 +1406,7 @@ void intel_setup_overlay(struct drm_device *dev)
1370{ 1406{
1371 drm_i915_private_t *dev_priv = dev->dev_private; 1407 drm_i915_private_t *dev_priv = dev->dev_private;
1372 struct intel_overlay *overlay; 1408 struct intel_overlay *overlay;
1373 struct drm_gem_object *reg_bo; 1409 struct drm_i915_gem_object *reg_bo;
1374 struct overlay_registers *regs; 1410 struct overlay_registers *regs;
1375 int ret; 1411 int ret;
1376 1412
@@ -1385,7 +1421,7 @@ void intel_setup_overlay(struct drm_device *dev)
1385 reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE); 1421 reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE);
1386 if (!reg_bo) 1422 if (!reg_bo)
1387 goto out_free; 1423 goto out_free;
1388 overlay->reg_bo = to_intel_bo(reg_bo); 1424 overlay->reg_bo = reg_bo;
1389 1425
1390 if (OVERLAY_NEEDS_PHYSICAL(dev)) { 1426 if (OVERLAY_NEEDS_PHYSICAL(dev)) {
1391 ret = i915_gem_attach_phys_object(dev, reg_bo, 1427 ret = i915_gem_attach_phys_object(dev, reg_bo,
@@ -1395,14 +1431,14 @@ void intel_setup_overlay(struct drm_device *dev)
1395 DRM_ERROR("failed to attach phys overlay regs\n"); 1431 DRM_ERROR("failed to attach phys overlay regs\n");
1396 goto out_free_bo; 1432 goto out_free_bo;
1397 } 1433 }
1398 overlay->flip_addr = overlay->reg_bo->phys_obj->handle->busaddr; 1434 overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
1399 } else { 1435 } else {
1400 ret = i915_gem_object_pin(reg_bo, PAGE_SIZE); 1436 ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true);
1401 if (ret) { 1437 if (ret) {
1402 DRM_ERROR("failed to pin overlay register bo\n"); 1438 DRM_ERROR("failed to pin overlay register bo\n");
1403 goto out_free_bo; 1439 goto out_free_bo;
1404 } 1440 }
1405 overlay->flip_addr = overlay->reg_bo->gtt_offset; 1441 overlay->flip_addr = reg_bo->gtt_offset;
1406 1442
1407 ret = i915_gem_object_set_to_gtt_domain(reg_bo, true); 1443 ret = i915_gem_object_set_to_gtt_domain(reg_bo, true);
1408 if (ret) { 1444 if (ret) {
@@ -1434,7 +1470,7 @@ void intel_setup_overlay(struct drm_device *dev)
1434out_unpin_bo: 1470out_unpin_bo:
1435 i915_gem_object_unpin(reg_bo); 1471 i915_gem_object_unpin(reg_bo);
1436out_free_bo: 1472out_free_bo:
1437 drm_gem_object_unreference(reg_bo); 1473 drm_gem_object_unreference(&reg_bo->base);
1438out_free: 1474out_free:
1439 kfree(overlay); 1475 kfree(overlay);
1440 return; 1476 return;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 92ff8f385278..7350ec2515c6 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -125,15 +125,55 @@ static int is_backlight_combination_mode(struct drm_device *dev)
125 return 0; 125 return 0;
126} 126}
127 127
128static u32 i915_read_blc_pwm_ctl(struct drm_i915_private *dev_priv)
129{
130 u32 val;
131
132 /* Restore the CTL value if it lost, e.g. GPU reset */
133
134 if (HAS_PCH_SPLIT(dev_priv->dev)) {
135 val = I915_READ(BLC_PWM_PCH_CTL2);
136 if (dev_priv->saveBLC_PWM_CTL2 == 0) {
137 dev_priv->saveBLC_PWM_CTL2 = val;
138 } else if (val == 0) {
139 I915_WRITE(BLC_PWM_PCH_CTL2,
140 dev_priv->saveBLC_PWM_CTL);
141 val = dev_priv->saveBLC_PWM_CTL;
142 }
143 } else {
144 val = I915_READ(BLC_PWM_CTL);
145 if (dev_priv->saveBLC_PWM_CTL == 0) {
146 dev_priv->saveBLC_PWM_CTL = val;
147 dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
148 } else if (val == 0) {
149 I915_WRITE(BLC_PWM_CTL,
150 dev_priv->saveBLC_PWM_CTL);
151 I915_WRITE(BLC_PWM_CTL2,
152 dev_priv->saveBLC_PWM_CTL2);
153 val = dev_priv->saveBLC_PWM_CTL;
154 }
155 }
156
157 return val;
158}
159
128u32 intel_panel_get_max_backlight(struct drm_device *dev) 160u32 intel_panel_get_max_backlight(struct drm_device *dev)
129{ 161{
130 struct drm_i915_private *dev_priv = dev->dev_private; 162 struct drm_i915_private *dev_priv = dev->dev_private;
131 u32 max; 163 u32 max;
132 164
165 max = i915_read_blc_pwm_ctl(dev_priv);
166 if (max == 0) {
167 /* XXX add code here to query mode clock or hardware clock
168 * and program max PWM appropriately.
169 */
170 printk_once(KERN_WARNING "fixme: max PWM is zero.\n");
171 return 1;
172 }
173
133 if (HAS_PCH_SPLIT(dev)) { 174 if (HAS_PCH_SPLIT(dev)) {
134 max = I915_READ(BLC_PWM_PCH_CTL2) >> 16; 175 max >>= 16;
135 } else { 176 } else {
136 max = I915_READ(BLC_PWM_CTL);
137 if (IS_PINEVIEW(dev)) { 177 if (IS_PINEVIEW(dev)) {
138 max >>= 17; 178 max >>= 17;
139 } else { 179 } else {
@@ -146,14 +186,6 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev)
146 max *= 0xff; 186 max *= 0xff;
147 } 187 }
148 188
149 if (max == 0) {
150 /* XXX add code here to query mode clock or hardware clock
151 * and program max PWM appropriately.
152 */
153 DRM_ERROR("fixme: max PWM is zero.\n");
154 max = 1;
155 }
156
157 DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max); 189 DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
158 return max; 190 return max;
159} 191}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 31cd7e33e820..56bc95c056dd 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -49,11 +49,11 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
49} 49}
50 50
51static void 51static void
52render_ring_flush(struct drm_device *dev, 52render_ring_flush(struct intel_ring_buffer *ring,
53 struct intel_ring_buffer *ring,
54 u32 invalidate_domains, 53 u32 invalidate_domains,
55 u32 flush_domains) 54 u32 flush_domains)
56{ 55{
56 struct drm_device *dev = ring->dev;
57 drm_i915_private_t *dev_priv = dev->dev_private; 57 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd; 58 u32 cmd;
59 59
@@ -109,49 +109,50 @@ render_ring_flush(struct drm_device *dev,
109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110 cmd |= MI_EXE_FLUSH; 110 cmd |= MI_EXE_FLUSH;
111 111
112 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
113 (IS_G4X(dev) || IS_GEN5(dev)))
114 cmd |= MI_INVALIDATE_ISP;
115
112#if WATCH_EXEC 116#if WATCH_EXEC
113 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd); 117 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114#endif 118#endif
115 intel_ring_begin(dev, ring, 2); 119 if (intel_ring_begin(ring, 2) == 0) {
116 intel_ring_emit(dev, ring, cmd); 120 intel_ring_emit(ring, cmd);
117 intel_ring_emit(dev, ring, MI_NOOP); 121 intel_ring_emit(ring, MI_NOOP);
118 intel_ring_advance(dev, ring); 122 intel_ring_advance(ring);
123 }
119 } 124 }
120} 125}
121 126
122static void ring_write_tail(struct drm_device *dev, 127static void ring_write_tail(struct intel_ring_buffer *ring,
123 struct intel_ring_buffer *ring,
124 u32 value) 128 u32 value)
125{ 129{
126 drm_i915_private_t *dev_priv = dev->dev_private; 130 drm_i915_private_t *dev_priv = ring->dev->dev_private;
127 I915_WRITE_TAIL(ring, value); 131 I915_WRITE_TAIL(ring, value);
128} 132}
129 133
130u32 intel_ring_get_active_head(struct drm_device *dev, 134u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 struct intel_ring_buffer *ring)
132{ 135{
133 drm_i915_private_t *dev_priv = dev->dev_private; 136 drm_i915_private_t *dev_priv = ring->dev->dev_private;
134 u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ? 137 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
135 RING_ACTHD(ring->mmio_base) : ACTHD; 138 RING_ACTHD(ring->mmio_base) : ACTHD;
136 139
137 return I915_READ(acthd_reg); 140 return I915_READ(acthd_reg);
138} 141}
139 142
140static int init_ring_common(struct drm_device *dev, 143static int init_ring_common(struct intel_ring_buffer *ring)
141 struct intel_ring_buffer *ring)
142{ 144{
145 drm_i915_private_t *dev_priv = ring->dev->dev_private;
146 struct drm_i915_gem_object *obj = ring->obj;
143 u32 head; 147 u32 head;
144 drm_i915_private_t *dev_priv = dev->dev_private;
145 struct drm_i915_gem_object *obj_priv;
146 obj_priv = to_intel_bo(ring->gem_object);
147 148
148 /* Stop the ring if it's running. */ 149 /* Stop the ring if it's running. */
149 I915_WRITE_CTL(ring, 0); 150 I915_WRITE_CTL(ring, 0);
150 I915_WRITE_HEAD(ring, 0); 151 I915_WRITE_HEAD(ring, 0);
151 ring->write_tail(dev, ring, 0); 152 ring->write_tail(ring, 0);
152 153
153 /* Initialize the ring. */ 154 /* Initialize the ring. */
154 I915_WRITE_START(ring, obj_priv->gtt_offset); 155 I915_WRITE_START(ring, obj->gtt_offset);
155 head = I915_READ_HEAD(ring) & HEAD_ADDR; 156 head = I915_READ_HEAD(ring) & HEAD_ADDR;
156 157
157 /* G45 ring initialization fails to reset head to zero */ 158 /* G45 ring initialization fails to reset head to zero */
@@ -178,12 +179,13 @@ static int init_ring_common(struct drm_device *dev,
178 } 179 }
179 180
180 I915_WRITE_CTL(ring, 181 I915_WRITE_CTL(ring,
181 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES) 182 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
182 | RING_REPORT_64K | RING_VALID); 183 | RING_REPORT_64K | RING_VALID);
183 184
184 head = I915_READ_HEAD(ring) & HEAD_ADDR;
185 /* If the head is still not zero, the ring is dead */ 185 /* If the head is still not zero, the ring is dead */
186 if (head != 0) { 186 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
187 I915_READ_START(ring) != obj->gtt_offset ||
188 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
187 DRM_ERROR("%s initialization failed " 189 DRM_ERROR("%s initialization failed "
188 "ctl %08x head %08x tail %08x start %08x\n", 190 "ctl %08x head %08x tail %08x start %08x\n",
189 ring->name, 191 ring->name,
@@ -194,8 +196,8 @@ static int init_ring_common(struct drm_device *dev,
194 return -EIO; 196 return -EIO;
195 } 197 }
196 198
197 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 199 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
198 i915_kernel_lost_context(dev); 200 i915_kernel_lost_context(ring->dev);
199 else { 201 else {
200 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; 202 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
201 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; 203 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
@@ -203,335 +205,500 @@ static int init_ring_common(struct drm_device *dev,
203 if (ring->space < 0) 205 if (ring->space < 0)
204 ring->space += ring->size; 206 ring->space += ring->size;
205 } 207 }
208
206 return 0; 209 return 0;
207} 210}
208 211
209static int init_render_ring(struct drm_device *dev, 212/*
210 struct intel_ring_buffer *ring) 213 * 965+ support PIPE_CONTROL commands, which provide finer grained control
214 * over cache flushing.
215 */
216struct pipe_control {
217 struct drm_i915_gem_object *obj;
218 volatile u32 *cpu_page;
219 u32 gtt_offset;
220};
221
222static int
223init_pipe_control(struct intel_ring_buffer *ring)
211{ 224{
212 drm_i915_private_t *dev_priv = dev->dev_private; 225 struct pipe_control *pc;
213 int ret = init_ring_common(dev, ring); 226 struct drm_i915_gem_object *obj;
214 int mode; 227 int ret;
228
229 if (ring->private)
230 return 0;
231
232 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
233 if (!pc)
234 return -ENOMEM;
235
236 obj = i915_gem_alloc_object(ring->dev, 4096);
237 if (obj == NULL) {
238 DRM_ERROR("Failed to allocate seqno page\n");
239 ret = -ENOMEM;
240 goto err;
241 }
242 obj->agp_type = AGP_USER_CACHED_MEMORY;
243
244 ret = i915_gem_object_pin(obj, 4096, true);
245 if (ret)
246 goto err_unref;
247
248 pc->gtt_offset = obj->gtt_offset;
249 pc->cpu_page = kmap(obj->pages[0]);
250 if (pc->cpu_page == NULL)
251 goto err_unpin;
252
253 pc->obj = obj;
254 ring->private = pc;
255 return 0;
256
257err_unpin:
258 i915_gem_object_unpin(obj);
259err_unref:
260 drm_gem_object_unreference(&obj->base);
261err:
262 kfree(pc);
263 return ret;
264}
265
266static void
267cleanup_pipe_control(struct intel_ring_buffer *ring)
268{
269 struct pipe_control *pc = ring->private;
270 struct drm_i915_gem_object *obj;
271
272 if (!ring->private)
273 return;
274
275 obj = pc->obj;
276 kunmap(obj->pages[0]);
277 i915_gem_object_unpin(obj);
278 drm_gem_object_unreference(&obj->base);
279
280 kfree(pc);
281 ring->private = NULL;
282}
283
284static int init_render_ring(struct intel_ring_buffer *ring)
285{
286 struct drm_device *dev = ring->dev;
287 struct drm_i915_private *dev_priv = dev->dev_private;
288 int ret = init_ring_common(ring);
215 289
216 if (INTEL_INFO(dev)->gen > 3) { 290 if (INTEL_INFO(dev)->gen > 3) {
217 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; 291 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
218 if (IS_GEN6(dev)) 292 if (IS_GEN6(dev))
219 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE; 293 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
220 I915_WRITE(MI_MODE, mode); 294 I915_WRITE(MI_MODE, mode);
221 } 295 }
296
297 if (INTEL_INFO(dev)->gen >= 6) {
298 } else if (IS_GEN5(dev)) {
299 ret = init_pipe_control(ring);
300 if (ret)
301 return ret;
302 }
303
222 return ret; 304 return ret;
223} 305}
224 306
225#define PIPE_CONTROL_FLUSH(addr) \ 307static void render_ring_cleanup(struct intel_ring_buffer *ring)
308{
309 if (!ring->private)
310 return;
311
312 cleanup_pipe_control(ring);
313}
314
315static void
316update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
317{
318 struct drm_device *dev = ring->dev;
319 struct drm_i915_private *dev_priv = dev->dev_private;
320 int id;
321
322 /*
323 * cs -> 1 = vcs, 0 = bcs
324 * vcs -> 1 = bcs, 0 = cs,
325 * bcs -> 1 = cs, 0 = vcs.
326 */
327 id = ring - dev_priv->ring;
328 id += 2 - i;
329 id %= 3;
330
331 intel_ring_emit(ring,
332 MI_SEMAPHORE_MBOX |
333 MI_SEMAPHORE_REGISTER |
334 MI_SEMAPHORE_UPDATE);
335 intel_ring_emit(ring, seqno);
336 intel_ring_emit(ring,
337 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
338}
339
340static int
341gen6_add_request(struct intel_ring_buffer *ring,
342 u32 *result)
343{
344 u32 seqno;
345 int ret;
346
347 ret = intel_ring_begin(ring, 10);
348 if (ret)
349 return ret;
350
351 seqno = i915_gem_get_seqno(ring->dev);
352 update_semaphore(ring, 0, seqno);
353 update_semaphore(ring, 1, seqno);
354
355 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
356 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
357 intel_ring_emit(ring, seqno);
358 intel_ring_emit(ring, MI_USER_INTERRUPT);
359 intel_ring_advance(ring);
360
361 *result = seqno;
362 return 0;
363}
364
365int
366intel_ring_sync(struct intel_ring_buffer *ring,
367 struct intel_ring_buffer *to,
368 u32 seqno)
369{
370 int ret;
371
372 ret = intel_ring_begin(ring, 4);
373 if (ret)
374 return ret;
375
376 intel_ring_emit(ring,
377 MI_SEMAPHORE_MBOX |
378 MI_SEMAPHORE_REGISTER |
379 intel_ring_sync_index(ring, to) << 17 |
380 MI_SEMAPHORE_COMPARE);
381 intel_ring_emit(ring, seqno);
382 intel_ring_emit(ring, 0);
383 intel_ring_emit(ring, MI_NOOP);
384 intel_ring_advance(ring);
385
386 return 0;
387}
388
389#define PIPE_CONTROL_FLUSH(ring__, addr__) \
226do { \ 390do { \
227 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ 391 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
228 PIPE_CONTROL_DEPTH_STALL | 2); \ 392 PIPE_CONTROL_DEPTH_STALL | 2); \
229 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ 393 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
230 OUT_RING(0); \ 394 intel_ring_emit(ring__, 0); \
231 OUT_RING(0); \ 395 intel_ring_emit(ring__, 0); \
232} while (0) 396} while (0)
233 397
234/** 398static int
235 * Creates a new sequence number, emitting a write of it to the status page 399pc_render_add_request(struct intel_ring_buffer *ring,
236 * plus an interrupt, which will trigger i915_user_interrupt_handler. 400 u32 *result)
237 *
238 * Must be called with struct_lock held.
239 *
240 * Returned sequence numbers are nonzero on success.
241 */
242static u32
243render_ring_add_request(struct drm_device *dev,
244 struct intel_ring_buffer *ring,
245 u32 flush_domains)
246{ 401{
247 drm_i915_private_t *dev_priv = dev->dev_private; 402 struct drm_device *dev = ring->dev;
248 u32 seqno; 403 u32 seqno = i915_gem_get_seqno(dev);
404 struct pipe_control *pc = ring->private;
405 u32 scratch_addr = pc->gtt_offset + 128;
406 int ret;
249 407
250 seqno = i915_gem_get_seqno(dev); 408 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
251 409 * incoherent with writes to memory, i.e. completely fubar,
252 if (IS_GEN6(dev)) { 410 * so we need to use PIPE_NOTIFY instead.
253 BEGIN_LP_RING(6); 411 *
254 OUT_RING(GFX_OP_PIPE_CONTROL | 3); 412 * However, we also need to workaround the qword write
255 OUT_RING(PIPE_CONTROL_QW_WRITE | 413 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
256 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH | 414 * memory before requesting an interrupt.
257 PIPE_CONTROL_NOTIFY); 415 */
258 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); 416 ret = intel_ring_begin(ring, 32);
259 OUT_RING(seqno); 417 if (ret)
260 OUT_RING(0); 418 return ret;
261 OUT_RING(0); 419
262 ADVANCE_LP_RING(); 420 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
263 } else if (HAS_PIPE_CONTROL(dev)) { 421 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
264 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128; 422 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
423 intel_ring_emit(ring, seqno);
424 intel_ring_emit(ring, 0);
425 PIPE_CONTROL_FLUSH(ring, scratch_addr);
426 scratch_addr += 128; /* write to separate cachelines */
427 PIPE_CONTROL_FLUSH(ring, scratch_addr);
428 scratch_addr += 128;
429 PIPE_CONTROL_FLUSH(ring, scratch_addr);
430 scratch_addr += 128;
431 PIPE_CONTROL_FLUSH(ring, scratch_addr);
432 scratch_addr += 128;
433 PIPE_CONTROL_FLUSH(ring, scratch_addr);
434 scratch_addr += 128;
435 PIPE_CONTROL_FLUSH(ring, scratch_addr);
436 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
437 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
438 PIPE_CONTROL_NOTIFY);
439 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
440 intel_ring_emit(ring, seqno);
441 intel_ring_emit(ring, 0);
442 intel_ring_advance(ring);
443
444 *result = seqno;
445 return 0;
446}
265 447
266 /* 448static int
267 * Workaround qword write incoherence by flushing the 449render_ring_add_request(struct intel_ring_buffer *ring,
268 * PIPE_NOTIFY buffers out to memory before requesting 450 u32 *result)
269 * an interrupt. 451{
270 */ 452 struct drm_device *dev = ring->dev;
271 BEGIN_LP_RING(32); 453 u32 seqno = i915_gem_get_seqno(dev);
272 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | 454 int ret;
273 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
274 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
275 OUT_RING(seqno);
276 OUT_RING(0);
277 PIPE_CONTROL_FLUSH(scratch_addr);
278 scratch_addr += 128; /* write to separate cachelines */
279 PIPE_CONTROL_FLUSH(scratch_addr);
280 scratch_addr += 128;
281 PIPE_CONTROL_FLUSH(scratch_addr);
282 scratch_addr += 128;
283 PIPE_CONTROL_FLUSH(scratch_addr);
284 scratch_addr += 128;
285 PIPE_CONTROL_FLUSH(scratch_addr);
286 scratch_addr += 128;
287 PIPE_CONTROL_FLUSH(scratch_addr);
288 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
289 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
290 PIPE_CONTROL_NOTIFY);
291 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
292 OUT_RING(seqno);
293 OUT_RING(0);
294 ADVANCE_LP_RING();
295 } else {
296 BEGIN_LP_RING(4);
297 OUT_RING(MI_STORE_DWORD_INDEX);
298 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
299 OUT_RING(seqno);
300 455
301 OUT_RING(MI_USER_INTERRUPT); 456 ret = intel_ring_begin(ring, 4);
302 ADVANCE_LP_RING(); 457 if (ret)
303 } 458 return ret;
304 return seqno; 459
460 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
461 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
462 intel_ring_emit(ring, seqno);
463 intel_ring_emit(ring, MI_USER_INTERRUPT);
464 intel_ring_advance(ring);
465
466 *result = seqno;
467 return 0;
305} 468}
306 469
307static u32 470static u32
308render_ring_get_seqno(struct drm_device *dev, 471ring_get_seqno(struct intel_ring_buffer *ring)
309 struct intel_ring_buffer *ring)
310{ 472{
311 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 473 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
312 if (HAS_PIPE_CONTROL(dev))
313 return ((volatile u32 *)(dev_priv->seqno_page))[0];
314 else
315 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
316} 474}
317 475
318static void 476static u32
319render_ring_get_user_irq(struct drm_device *dev, 477pc_render_get_seqno(struct intel_ring_buffer *ring)
320 struct intel_ring_buffer *ring)
321{ 478{
322 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 479 struct pipe_control *pc = ring->private;
323 unsigned long irqflags; 480 return pc->cpu_page[0];
481}
482
483static bool
484render_ring_get_irq(struct intel_ring_buffer *ring)
485{
486 struct drm_device *dev = ring->dev;
487
488 if (!dev->irq_enabled)
489 return false;
490
491 if (atomic_inc_return(&ring->irq_refcount) == 1) {
492 drm_i915_private_t *dev_priv = dev->dev_private;
493 unsigned long irqflags;
324 494
325 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 495 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
326 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
327 if (HAS_PCH_SPLIT(dev)) 496 if (HAS_PCH_SPLIT(dev))
328 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 497 ironlake_enable_graphics_irq(dev_priv,
498 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
329 else 499 else
330 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 500 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
501 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
331 } 502 }
332 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 503
504 return true;
333} 505}
334 506
335static void 507static void
336render_ring_put_user_irq(struct drm_device *dev, 508render_ring_put_irq(struct intel_ring_buffer *ring)
337 struct intel_ring_buffer *ring)
338{ 509{
339 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 510 struct drm_device *dev = ring->dev;
340 unsigned long irqflags; 511
512 if (atomic_dec_and_test(&ring->irq_refcount)) {
513 drm_i915_private_t *dev_priv = dev->dev_private;
514 unsigned long irqflags;
341 515
342 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 516 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
343 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
344 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
345 if (HAS_PCH_SPLIT(dev)) 517 if (HAS_PCH_SPLIT(dev))
346 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 518 ironlake_disable_graphics_irq(dev_priv,
519 GT_USER_INTERRUPT |
520 GT_PIPE_NOTIFY);
347 else 521 else
348 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 522 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
523 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
349 } 524 }
350 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
351} 525}
352 526
353void intel_ring_setup_status_page(struct drm_device *dev, 527void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
354 struct intel_ring_buffer *ring)
355{ 528{
356 drm_i915_private_t *dev_priv = dev->dev_private; 529 drm_i915_private_t *dev_priv = ring->dev->dev_private;
357 if (IS_GEN6(dev)) { 530 u32 mmio = IS_GEN6(ring->dev) ?
358 I915_WRITE(RING_HWS_PGA_GEN6(ring->mmio_base), 531 RING_HWS_PGA_GEN6(ring->mmio_base) :
359 ring->status_page.gfx_addr); 532 RING_HWS_PGA(ring->mmio_base);
360 I915_READ(RING_HWS_PGA_GEN6(ring->mmio_base)); /* posting read */ 533 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
361 } else { 534 POSTING_READ(mmio);
362 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
363 ring->status_page.gfx_addr);
364 I915_READ(RING_HWS_PGA(ring->mmio_base)); /* posting read */
365 }
366
367} 535}
368 536
369static void 537static void
370bsd_ring_flush(struct drm_device *dev, 538bsd_ring_flush(struct intel_ring_buffer *ring,
371 struct intel_ring_buffer *ring, 539 u32 invalidate_domains,
372 u32 invalidate_domains, 540 u32 flush_domains)
373 u32 flush_domains)
374{ 541{
375 intel_ring_begin(dev, ring, 2); 542 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
376 intel_ring_emit(dev, ring, MI_FLUSH); 543 return;
377 intel_ring_emit(dev, ring, MI_NOOP);
378 intel_ring_advance(dev, ring);
379}
380 544
381static int init_bsd_ring(struct drm_device *dev, 545 if (intel_ring_begin(ring, 2) == 0) {
382 struct intel_ring_buffer *ring) 546 intel_ring_emit(ring, MI_FLUSH);
383{ 547 intel_ring_emit(ring, MI_NOOP);
384 return init_ring_common(dev, ring); 548 intel_ring_advance(ring);
549 }
385} 550}
386 551
387static u32 552static int
388ring_add_request(struct drm_device *dev, 553ring_add_request(struct intel_ring_buffer *ring,
389 struct intel_ring_buffer *ring, 554 u32 *result)
390 u32 flush_domains)
391{ 555{
392 u32 seqno; 556 u32 seqno;
557 int ret;
558
559 ret = intel_ring_begin(ring, 4);
560 if (ret)
561 return ret;
393 562
394 seqno = i915_gem_get_seqno(dev); 563 seqno = i915_gem_get_seqno(ring->dev);
395 564
396 intel_ring_begin(dev, ring, 4); 565 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
397 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); 566 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
398 intel_ring_emit(dev, ring, 567 intel_ring_emit(ring, seqno);
399 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 568 intel_ring_emit(ring, MI_USER_INTERRUPT);
400 intel_ring_emit(dev, ring, seqno); 569 intel_ring_advance(ring);
401 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
402 intel_ring_advance(dev, ring);
403 570
404 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 571 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
405 572 *result = seqno;
406 return seqno; 573 return 0;
407} 574}
408 575
409static void 576static bool
410bsd_ring_get_user_irq(struct drm_device *dev, 577ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
411 struct intel_ring_buffer *ring)
412{ 578{
413 /* do nothing */ 579 struct drm_device *dev = ring->dev;
580
581 if (!dev->irq_enabled)
582 return false;
583
584 if (atomic_inc_return(&ring->irq_refcount) == 1) {
585 drm_i915_private_t *dev_priv = dev->dev_private;
586 unsigned long irqflags;
587
588 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
589 ironlake_enable_graphics_irq(dev_priv, flag);
590 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
591 }
592
593 return true;
414} 594}
595
415static void 596static void
416bsd_ring_put_user_irq(struct drm_device *dev, 597ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
417 struct intel_ring_buffer *ring)
418{ 598{
419 /* do nothing */ 599 struct drm_device *dev = ring->dev;
600
601 if (atomic_dec_and_test(&ring->irq_refcount)) {
602 drm_i915_private_t *dev_priv = dev->dev_private;
603 unsigned long irqflags;
604
605 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
606 ironlake_disable_graphics_irq(dev_priv, flag);
607 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
608 }
420} 609}
421 610
422static u32 611static bool
423ring_status_page_get_seqno(struct drm_device *dev, 612bsd_ring_get_irq(struct intel_ring_buffer *ring)
424 struct intel_ring_buffer *ring)
425{ 613{
426 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 614 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
615}
616static void
617bsd_ring_put_irq(struct intel_ring_buffer *ring)
618{
619 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
427} 620}
428 621
429static int 622static int
430ring_dispatch_gem_execbuffer(struct drm_device *dev, 623ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
431 struct intel_ring_buffer *ring,
432 struct drm_i915_gem_execbuffer2 *exec,
433 struct drm_clip_rect *cliprects,
434 uint64_t exec_offset)
435{ 624{
436 uint32_t exec_start; 625 int ret;
437 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 626
438 intel_ring_begin(dev, ring, 2); 627 ret = intel_ring_begin(ring, 2);
439 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START | 628 if (ret)
440 (2 << 6) | MI_BATCH_NON_SECURE_I965); 629 return ret;
441 intel_ring_emit(dev, ring, exec_start); 630
442 intel_ring_advance(dev, ring); 631 intel_ring_emit(ring,
632 MI_BATCH_BUFFER_START | (2 << 6) |
633 MI_BATCH_NON_SECURE_I965);
634 intel_ring_emit(ring, offset);
635 intel_ring_advance(ring);
636
443 return 0; 637 return 0;
444} 638}
445 639
446static int 640static int
447render_ring_dispatch_gem_execbuffer(struct drm_device *dev, 641render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
448 struct intel_ring_buffer *ring, 642 u32 offset, u32 len)
449 struct drm_i915_gem_execbuffer2 *exec,
450 struct drm_clip_rect *cliprects,
451 uint64_t exec_offset)
452{ 643{
644 struct drm_device *dev = ring->dev;
453 drm_i915_private_t *dev_priv = dev->dev_private; 645 drm_i915_private_t *dev_priv = dev->dev_private;
454 int nbox = exec->num_cliprects; 646 int ret;
455 int i = 0, count;
456 uint32_t exec_start, exec_len;
457 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
458 exec_len = (uint32_t) exec->batch_len;
459 647
460 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1); 648 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
461 649
462 count = nbox ? nbox : 1; 650 if (IS_I830(dev) || IS_845G(dev)) {
651 ret = intel_ring_begin(ring, 4);
652 if (ret)
653 return ret;
463 654
464 for (i = 0; i < count; i++) { 655 intel_ring_emit(ring, MI_BATCH_BUFFER);
465 if (i < nbox) { 656 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
466 int ret = i915_emit_box(dev, cliprects, i, 657 intel_ring_emit(ring, offset + len - 8);
467 exec->DR1, exec->DR4); 658 intel_ring_emit(ring, 0);
468 if (ret) 659 } else {
469 return ret; 660 ret = intel_ring_begin(ring, 2);
470 } 661 if (ret)
662 return ret;
471 663
472 if (IS_I830(dev) || IS_845G(dev)) { 664 if (INTEL_INFO(dev)->gen >= 4) {
473 intel_ring_begin(dev, ring, 4); 665 intel_ring_emit(ring,
474 intel_ring_emit(dev, ring, MI_BATCH_BUFFER); 666 MI_BATCH_BUFFER_START | (2 << 6) |
475 intel_ring_emit(dev, ring, 667 MI_BATCH_NON_SECURE_I965);
476 exec_start | MI_BATCH_NON_SECURE); 668 intel_ring_emit(ring, offset);
477 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
478 intel_ring_emit(dev, ring, 0);
479 } else { 669 } else {
480 intel_ring_begin(dev, ring, 2); 670 intel_ring_emit(ring,
481 if (INTEL_INFO(dev)->gen >= 4) { 671 MI_BATCH_BUFFER_START | (2 << 6));
482 intel_ring_emit(dev, ring, 672 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
483 MI_BATCH_BUFFER_START | (2 << 6)
484 | MI_BATCH_NON_SECURE_I965);
485 intel_ring_emit(dev, ring, exec_start);
486 } else {
487 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
488 | (2 << 6));
489 intel_ring_emit(dev, ring, exec_start |
490 MI_BATCH_NON_SECURE);
491 }
492 } 673 }
493 intel_ring_advance(dev, ring);
494 } 674 }
495 675 intel_ring_advance(ring);
496 if (IS_G4X(dev) || IS_GEN5(dev)) {
497 intel_ring_begin(dev, ring, 2);
498 intel_ring_emit(dev, ring, MI_FLUSH |
499 MI_NO_WRITE_FLUSH |
500 MI_INVALIDATE_ISP );
501 intel_ring_emit(dev, ring, MI_NOOP);
502 intel_ring_advance(dev, ring);
503 }
504 /* XXX breadcrumb */
505 676
506 return 0; 677 return 0;
507} 678}
508 679
509static void cleanup_status_page(struct drm_device *dev, 680static void cleanup_status_page(struct intel_ring_buffer *ring)
510 struct intel_ring_buffer *ring)
511{ 681{
512 drm_i915_private_t *dev_priv = dev->dev_private; 682 drm_i915_private_t *dev_priv = ring->dev->dev_private;
513 struct drm_gem_object *obj; 683 struct drm_i915_gem_object *obj;
514 struct drm_i915_gem_object *obj_priv;
515 684
516 obj = ring->status_page.obj; 685 obj = ring->status_page.obj;
517 if (obj == NULL) 686 if (obj == NULL)
518 return; 687 return;
519 obj_priv = to_intel_bo(obj);
520 688
521 kunmap(obj_priv->pages[0]); 689 kunmap(obj->pages[0]);
522 i915_gem_object_unpin(obj); 690 i915_gem_object_unpin(obj);
523 drm_gem_object_unreference(obj); 691 drm_gem_object_unreference(&obj->base);
524 ring->status_page.obj = NULL; 692 ring->status_page.obj = NULL;
525 693
526 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 694 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
527} 695}
528 696
529static int init_status_page(struct drm_device *dev, 697static int init_status_page(struct intel_ring_buffer *ring)
530 struct intel_ring_buffer *ring)
531{ 698{
699 struct drm_device *dev = ring->dev;
532 drm_i915_private_t *dev_priv = dev->dev_private; 700 drm_i915_private_t *dev_priv = dev->dev_private;
533 struct drm_gem_object *obj; 701 struct drm_i915_gem_object *obj;
534 struct drm_i915_gem_object *obj_priv;
535 int ret; 702 int ret;
536 703
537 obj = i915_gem_alloc_object(dev, 4096); 704 obj = i915_gem_alloc_object(dev, 4096);
@@ -540,16 +707,15 @@ static int init_status_page(struct drm_device *dev,
540 ret = -ENOMEM; 707 ret = -ENOMEM;
541 goto err; 708 goto err;
542 } 709 }
543 obj_priv = to_intel_bo(obj); 710 obj->agp_type = AGP_USER_CACHED_MEMORY;
544 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
545 711
546 ret = i915_gem_object_pin(obj, 4096); 712 ret = i915_gem_object_pin(obj, 4096, true);
547 if (ret != 0) { 713 if (ret != 0) {
548 goto err_unref; 714 goto err_unref;
549 } 715 }
550 716
551 ring->status_page.gfx_addr = obj_priv->gtt_offset; 717 ring->status_page.gfx_addr = obj->gtt_offset;
552 ring->status_page.page_addr = kmap(obj_priv->pages[0]); 718 ring->status_page.page_addr = kmap(obj->pages[0]);
553 if (ring->status_page.page_addr == NULL) { 719 if (ring->status_page.page_addr == NULL) {
554 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 720 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
555 goto err_unpin; 721 goto err_unpin;
@@ -557,7 +723,7 @@ static int init_status_page(struct drm_device *dev,
557 ring->status_page.obj = obj; 723 ring->status_page.obj = obj;
558 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 724 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
559 725
560 intel_ring_setup_status_page(dev, ring); 726 intel_ring_setup_status_page(ring);
561 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", 727 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
562 ring->name, ring->status_page.gfx_addr); 728 ring->name, ring->status_page.gfx_addr);
563 729
@@ -566,7 +732,7 @@ static int init_status_page(struct drm_device *dev,
566err_unpin: 732err_unpin:
567 i915_gem_object_unpin(obj); 733 i915_gem_object_unpin(obj);
568err_unref: 734err_unref:
569 drm_gem_object_unreference(obj); 735 drm_gem_object_unreference(&obj->base);
570err: 736err:
571 return ret; 737 return ret;
572} 738}
@@ -574,9 +740,7 @@ err:
574int intel_init_ring_buffer(struct drm_device *dev, 740int intel_init_ring_buffer(struct drm_device *dev,
575 struct intel_ring_buffer *ring) 741 struct intel_ring_buffer *ring)
576{ 742{
577 struct drm_i915_private *dev_priv = dev->dev_private; 743 struct drm_i915_gem_object *obj;
578 struct drm_i915_gem_object *obj_priv;
579 struct drm_gem_object *obj;
580 int ret; 744 int ret;
581 745
582 ring->dev = dev; 746 ring->dev = dev;
@@ -585,7 +749,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
585 INIT_LIST_HEAD(&ring->gpu_write_list); 749 INIT_LIST_HEAD(&ring->gpu_write_list);
586 750
587 if (I915_NEED_GFX_HWS(dev)) { 751 if (I915_NEED_GFX_HWS(dev)) {
588 ret = init_status_page(dev, ring); 752 ret = init_status_page(ring);
589 if (ret) 753 if (ret)
590 return ret; 754 return ret;
591 } 755 }
@@ -597,15 +761,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
597 goto err_hws; 761 goto err_hws;
598 } 762 }
599 763
600 ring->gem_object = obj; 764 ring->obj = obj;
601 765
602 ret = i915_gem_object_pin(obj, PAGE_SIZE); 766 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
603 if (ret) 767 if (ret)
604 goto err_unref; 768 goto err_unref;
605 769
606 obj_priv = to_intel_bo(obj);
607 ring->map.size = ring->size; 770 ring->map.size = ring->size;
608 ring->map.offset = dev->agp->base + obj_priv->gtt_offset; 771 ring->map.offset = dev->agp->base + obj->gtt_offset;
609 ring->map.type = 0; 772 ring->map.type = 0;
610 ring->map.flags = 0; 773 ring->map.flags = 0;
611 ring->map.mtrr = 0; 774 ring->map.mtrr = 0;
@@ -618,60 +781,57 @@ int intel_init_ring_buffer(struct drm_device *dev,
618 } 781 }
619 782
620 ring->virtual_start = ring->map.handle; 783 ring->virtual_start = ring->map.handle;
621 ret = ring->init(dev, ring); 784 ret = ring->init(ring);
622 if (ret) 785 if (ret)
623 goto err_unmap; 786 goto err_unmap;
624 787
625 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 788 return 0;
626 i915_kernel_lost_context(dev);
627 else {
628 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
629 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
630 ring->space = ring->head - (ring->tail + 8);
631 if (ring->space < 0)
632 ring->space += ring->size;
633 }
634 return ret;
635 789
636err_unmap: 790err_unmap:
637 drm_core_ioremapfree(&ring->map, dev); 791 drm_core_ioremapfree(&ring->map, dev);
638err_unpin: 792err_unpin:
639 i915_gem_object_unpin(obj); 793 i915_gem_object_unpin(obj);
640err_unref: 794err_unref:
641 drm_gem_object_unreference(obj); 795 drm_gem_object_unreference(&obj->base);
642 ring->gem_object = NULL; 796 ring->obj = NULL;
643err_hws: 797err_hws:
644 cleanup_status_page(dev, ring); 798 cleanup_status_page(ring);
645 return ret; 799 return ret;
646} 800}
647 801
648void intel_cleanup_ring_buffer(struct drm_device *dev, 802void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
649 struct intel_ring_buffer *ring)
650{ 803{
651 if (ring->gem_object == NULL) 804 struct drm_i915_private *dev_priv;
805 int ret;
806
807 if (ring->obj == NULL)
652 return; 808 return;
653 809
654 drm_core_ioremapfree(&ring->map, dev); 810 /* Disable the ring buffer. The ring must be idle at this point */
811 dev_priv = ring->dev->dev_private;
812 ret = intel_wait_ring_buffer(ring, ring->size - 8);
813 I915_WRITE_CTL(ring, 0);
655 814
656 i915_gem_object_unpin(ring->gem_object); 815 drm_core_ioremapfree(&ring->map, ring->dev);
657 drm_gem_object_unreference(ring->gem_object); 816
658 ring->gem_object = NULL; 817 i915_gem_object_unpin(ring->obj);
818 drm_gem_object_unreference(&ring->obj->base);
819 ring->obj = NULL;
659 820
660 if (ring->cleanup) 821 if (ring->cleanup)
661 ring->cleanup(ring); 822 ring->cleanup(ring);
662 823
663 cleanup_status_page(dev, ring); 824 cleanup_status_page(ring);
664} 825}
665 826
666static int intel_wrap_ring_buffer(struct drm_device *dev, 827static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
667 struct intel_ring_buffer *ring)
668{ 828{
669 unsigned int *virt; 829 unsigned int *virt;
670 int rem; 830 int rem;
671 rem = ring->size - ring->tail; 831 rem = ring->size - ring->tail;
672 832
673 if (ring->space < rem) { 833 if (ring->space < rem) {
674 int ret = intel_wait_ring_buffer(dev, ring, rem); 834 int ret = intel_wait_ring_buffer(ring, rem);
675 if (ret) 835 if (ret)
676 return ret; 836 return ret;
677 } 837 }
@@ -689,11 +849,11 @@ static int intel_wrap_ring_buffer(struct drm_device *dev,
689 return 0; 849 return 0;
690} 850}
691 851
692int intel_wait_ring_buffer(struct drm_device *dev, 852int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
693 struct intel_ring_buffer *ring, int n)
694{ 853{
854 struct drm_device *dev = ring->dev;
855 struct drm_i915_private *dev_priv = dev->dev_private;
695 unsigned long end; 856 unsigned long end;
696 drm_i915_private_t *dev_priv = dev->dev_private;
697 u32 head; 857 u32 head;
698 858
699 trace_i915_ring_wait_begin (dev); 859 trace_i915_ring_wait_begin (dev);
@@ -711,7 +871,7 @@ int intel_wait_ring_buffer(struct drm_device *dev,
711 if (ring->space < 0) 871 if (ring->space < 0)
712 ring->space += ring->size; 872 ring->space += ring->size;
713 if (ring->space >= n) { 873 if (ring->space >= n) {
714 trace_i915_ring_wait_end (dev); 874 trace_i915_ring_wait_end(dev);
715 return 0; 875 return 0;
716 } 876 }
717 877
@@ -722,29 +882,39 @@ int intel_wait_ring_buffer(struct drm_device *dev,
722 } 882 }
723 883
724 msleep(1); 884 msleep(1);
885 if (atomic_read(&dev_priv->mm.wedged))
886 return -EAGAIN;
725 } while (!time_after(jiffies, end)); 887 } while (!time_after(jiffies, end));
726 trace_i915_ring_wait_end (dev); 888 trace_i915_ring_wait_end (dev);
727 return -EBUSY; 889 return -EBUSY;
728} 890}
729 891
730void intel_ring_begin(struct drm_device *dev, 892int intel_ring_begin(struct intel_ring_buffer *ring,
731 struct intel_ring_buffer *ring, 893 int num_dwords)
732 int num_dwords)
733{ 894{
734 int n = 4*num_dwords; 895 int n = 4*num_dwords;
735 if (unlikely(ring->tail + n > ring->size)) 896 int ret;
736 intel_wrap_ring_buffer(dev, ring); 897
737 if (unlikely(ring->space < n)) 898 if (unlikely(ring->tail + n > ring->size)) {
738 intel_wait_ring_buffer(dev, ring, n); 899 ret = intel_wrap_ring_buffer(ring);
900 if (unlikely(ret))
901 return ret;
902 }
903
904 if (unlikely(ring->space < n)) {
905 ret = intel_wait_ring_buffer(ring, n);
906 if (unlikely(ret))
907 return ret;
908 }
739 909
740 ring->space -= n; 910 ring->space -= n;
911 return 0;
741} 912}
742 913
743void intel_ring_advance(struct drm_device *dev, 914void intel_ring_advance(struct intel_ring_buffer *ring)
744 struct intel_ring_buffer *ring)
745{ 915{
746 ring->tail &= ring->size - 1; 916 ring->tail &= ring->size - 1;
747 ring->write_tail(dev, ring, ring->tail); 917 ring->write_tail(ring, ring->tail);
748} 918}
749 919
750static const struct intel_ring_buffer render_ring = { 920static const struct intel_ring_buffer render_ring = {
@@ -756,10 +926,11 @@ static const struct intel_ring_buffer render_ring = {
756 .write_tail = ring_write_tail, 926 .write_tail = ring_write_tail,
757 .flush = render_ring_flush, 927 .flush = render_ring_flush,
758 .add_request = render_ring_add_request, 928 .add_request = render_ring_add_request,
759 .get_seqno = render_ring_get_seqno, 929 .get_seqno = ring_get_seqno,
760 .user_irq_get = render_ring_get_user_irq, 930 .irq_get = render_ring_get_irq,
761 .user_irq_put = render_ring_put_user_irq, 931 .irq_put = render_ring_put_irq,
762 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer, 932 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
933 .cleanup = render_ring_cleanup,
763}; 934};
764 935
765/* ring buffer for bit-stream decoder */ 936/* ring buffer for bit-stream decoder */
@@ -769,22 +940,21 @@ static const struct intel_ring_buffer bsd_ring = {
769 .id = RING_BSD, 940 .id = RING_BSD,
770 .mmio_base = BSD_RING_BASE, 941 .mmio_base = BSD_RING_BASE,
771 .size = 32 * PAGE_SIZE, 942 .size = 32 * PAGE_SIZE,
772 .init = init_bsd_ring, 943 .init = init_ring_common,
773 .write_tail = ring_write_tail, 944 .write_tail = ring_write_tail,
774 .flush = bsd_ring_flush, 945 .flush = bsd_ring_flush,
775 .add_request = ring_add_request, 946 .add_request = ring_add_request,
776 .get_seqno = ring_status_page_get_seqno, 947 .get_seqno = ring_get_seqno,
777 .user_irq_get = bsd_ring_get_user_irq, 948 .irq_get = bsd_ring_get_irq,
778 .user_irq_put = bsd_ring_put_user_irq, 949 .irq_put = bsd_ring_put_irq,
779 .dispatch_gem_execbuffer = ring_dispatch_gem_execbuffer, 950 .dispatch_execbuffer = ring_dispatch_execbuffer,
780}; 951};
781 952
782 953
783static void gen6_bsd_ring_write_tail(struct drm_device *dev, 954static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
784 struct intel_ring_buffer *ring,
785 u32 value) 955 u32 value)
786{ 956{
787 drm_i915_private_t *dev_priv = dev->dev_private; 957 drm_i915_private_t *dev_priv = ring->dev->dev_private;
788 958
789 /* Every tail move must follow the sequence below */ 959 /* Every tail move must follow the sequence below */
790 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 960 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
@@ -803,69 +973,80 @@ static void gen6_bsd_ring_write_tail(struct drm_device *dev,
803 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); 973 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
804} 974}
805 975
806static void gen6_ring_flush(struct drm_device *dev, 976static void gen6_ring_flush(struct intel_ring_buffer *ring,
807 struct intel_ring_buffer *ring,
808 u32 invalidate_domains, 977 u32 invalidate_domains,
809 u32 flush_domains) 978 u32 flush_domains)
810{ 979{
811 intel_ring_begin(dev, ring, 4); 980 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
812 intel_ring_emit(dev, ring, MI_FLUSH_DW); 981 return;
813 intel_ring_emit(dev, ring, 0); 982
814 intel_ring_emit(dev, ring, 0); 983 if (intel_ring_begin(ring, 4) == 0) {
815 intel_ring_emit(dev, ring, 0); 984 intel_ring_emit(ring, MI_FLUSH_DW);
816 intel_ring_advance(dev, ring); 985 intel_ring_emit(ring, 0);
986 intel_ring_emit(ring, 0);
987 intel_ring_emit(ring, 0);
988 intel_ring_advance(ring);
989 }
817} 990}
818 991
819static int 992static int
820gen6_ring_dispatch_gem_execbuffer(struct drm_device *dev, 993gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
821 struct intel_ring_buffer *ring, 994 u32 offset, u32 len)
822 struct drm_i915_gem_execbuffer2 *exec,
823 struct drm_clip_rect *cliprects,
824 uint64_t exec_offset)
825{ 995{
826 uint32_t exec_start; 996 int ret;
827 997
828 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 998 ret = intel_ring_begin(ring, 2);
999 if (ret)
1000 return ret;
829 1001
830 intel_ring_begin(dev, ring, 2); 1002 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
831 intel_ring_emit(dev, ring,
832 MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
833 /* bit0-7 is the length on GEN6+ */ 1003 /* bit0-7 is the length on GEN6+ */
834 intel_ring_emit(dev, ring, exec_start); 1004 intel_ring_emit(ring, offset);
835 intel_ring_advance(dev, ring); 1005 intel_ring_advance(ring);
836 1006
837 return 0; 1007 return 0;
838} 1008}
839 1009
1010static bool
1011gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1012{
1013 return ring_get_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
1014}
1015
1016static void
1017gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1018{
1019 ring_put_irq(ring, GT_GEN6_BSD_USER_INTERRUPT);
1020}
1021
840/* ring buffer for Video Codec for Gen6+ */ 1022/* ring buffer for Video Codec for Gen6+ */
841static const struct intel_ring_buffer gen6_bsd_ring = { 1023static const struct intel_ring_buffer gen6_bsd_ring = {
842 .name = "gen6 bsd ring", 1024 .name = "gen6 bsd ring",
843 .id = RING_BSD, 1025 .id = RING_BSD,
844 .mmio_base = GEN6_BSD_RING_BASE, 1026 .mmio_base = GEN6_BSD_RING_BASE,
845 .size = 32 * PAGE_SIZE, 1027 .size = 32 * PAGE_SIZE,
846 .init = init_bsd_ring, 1028 .init = init_ring_common,
847 .write_tail = gen6_bsd_ring_write_tail, 1029 .write_tail = gen6_bsd_ring_write_tail,
848 .flush = gen6_ring_flush, 1030 .flush = gen6_ring_flush,
849 .add_request = ring_add_request, 1031 .add_request = gen6_add_request,
850 .get_seqno = ring_status_page_get_seqno, 1032 .get_seqno = ring_get_seqno,
851 .user_irq_get = bsd_ring_get_user_irq, 1033 .irq_get = gen6_bsd_ring_get_irq,
852 .user_irq_put = bsd_ring_put_user_irq, 1034 .irq_put = gen6_bsd_ring_put_irq,
853 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 1035 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
854}; 1036};
855 1037
856/* Blitter support (SandyBridge+) */ 1038/* Blitter support (SandyBridge+) */
857 1039
858static void 1040static bool
859blt_ring_get_user_irq(struct drm_device *dev, 1041blt_ring_get_irq(struct intel_ring_buffer *ring)
860 struct intel_ring_buffer *ring)
861{ 1042{
862 /* do nothing */ 1043 return ring_get_irq(ring, GT_BLT_USER_INTERRUPT);
863} 1044}
1045
864static void 1046static void
865blt_ring_put_user_irq(struct drm_device *dev, 1047blt_ring_put_irq(struct intel_ring_buffer *ring)
866 struct intel_ring_buffer *ring)
867{ 1048{
868 /* do nothing */ 1049 ring_put_irq(ring, GT_BLT_USER_INTERRUPT);
869} 1050}
870 1051
871 1052
@@ -883,32 +1064,31 @@ to_blt_workaround(struct intel_ring_buffer *ring)
883 return ring->private; 1064 return ring->private;
884} 1065}
885 1066
886static int blt_ring_init(struct drm_device *dev, 1067static int blt_ring_init(struct intel_ring_buffer *ring)
887 struct intel_ring_buffer *ring)
888{ 1068{
889 if (NEED_BLT_WORKAROUND(dev)) { 1069 if (NEED_BLT_WORKAROUND(ring->dev)) {
890 struct drm_i915_gem_object *obj; 1070 struct drm_i915_gem_object *obj;
891 u32 __iomem *ptr; 1071 u32 *ptr;
892 int ret; 1072 int ret;
893 1073
894 obj = to_intel_bo(i915_gem_alloc_object(dev, 4096)); 1074 obj = i915_gem_alloc_object(ring->dev, 4096);
895 if (obj == NULL) 1075 if (obj == NULL)
896 return -ENOMEM; 1076 return -ENOMEM;
897 1077
898 ret = i915_gem_object_pin(&obj->base, 4096); 1078 ret = i915_gem_object_pin(obj, 4096, true);
899 if (ret) { 1079 if (ret) {
900 drm_gem_object_unreference(&obj->base); 1080 drm_gem_object_unreference(&obj->base);
901 return ret; 1081 return ret;
902 } 1082 }
903 1083
904 ptr = kmap(obj->pages[0]); 1084 ptr = kmap(obj->pages[0]);
905 iowrite32(MI_BATCH_BUFFER_END, ptr); 1085 *ptr++ = MI_BATCH_BUFFER_END;
906 iowrite32(MI_NOOP, ptr+1); 1086 *ptr++ = MI_NOOP;
907 kunmap(obj->pages[0]); 1087 kunmap(obj->pages[0]);
908 1088
909 ret = i915_gem_object_set_to_gtt_domain(&obj->base, false); 1089 ret = i915_gem_object_set_to_gtt_domain(obj, false);
910 if (ret) { 1090 if (ret) {
911 i915_gem_object_unpin(&obj->base); 1091 i915_gem_object_unpin(obj);
912 drm_gem_object_unreference(&obj->base); 1092 drm_gem_object_unreference(&obj->base);
913 return ret; 1093 return ret;
914 } 1094 }
@@ -916,51 +1096,39 @@ static int blt_ring_init(struct drm_device *dev,
916 ring->private = obj; 1096 ring->private = obj;
917 } 1097 }
918 1098
919 return init_ring_common(dev, ring); 1099 return init_ring_common(ring);
920} 1100}
921 1101
922static void blt_ring_begin(struct drm_device *dev, 1102static int blt_ring_begin(struct intel_ring_buffer *ring,
923 struct intel_ring_buffer *ring,
924 int num_dwords) 1103 int num_dwords)
925{ 1104{
926 if (ring->private) { 1105 if (ring->private) {
927 intel_ring_begin(dev, ring, num_dwords+2); 1106 int ret = intel_ring_begin(ring, num_dwords+2);
928 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START); 1107 if (ret)
929 intel_ring_emit(dev, ring, to_blt_workaround(ring)->gtt_offset); 1108 return ret;
1109
1110 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1111 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1112
1113 return 0;
930 } else 1114 } else
931 intel_ring_begin(dev, ring, 4); 1115 return intel_ring_begin(ring, 4);
932} 1116}
933 1117
934static void blt_ring_flush(struct drm_device *dev, 1118static void blt_ring_flush(struct intel_ring_buffer *ring,
935 struct intel_ring_buffer *ring,
936 u32 invalidate_domains, 1119 u32 invalidate_domains,
937 u32 flush_domains) 1120 u32 flush_domains)
938{ 1121{
939 blt_ring_begin(dev, ring, 4); 1122 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
940 intel_ring_emit(dev, ring, MI_FLUSH_DW); 1123 return;
941 intel_ring_emit(dev, ring, 0);
942 intel_ring_emit(dev, ring, 0);
943 intel_ring_emit(dev, ring, 0);
944 intel_ring_advance(dev, ring);
945}
946
947static u32
948blt_ring_add_request(struct drm_device *dev,
949 struct intel_ring_buffer *ring,
950 u32 flush_domains)
951{
952 u32 seqno = i915_gem_get_seqno(dev);
953
954 blt_ring_begin(dev, ring, 4);
955 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
956 intel_ring_emit(dev, ring,
957 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
958 intel_ring_emit(dev, ring, seqno);
959 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
960 intel_ring_advance(dev, ring);
961 1124
962 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 1125 if (blt_ring_begin(ring, 4) == 0) {
963 return seqno; 1126 intel_ring_emit(ring, MI_FLUSH_DW);
1127 intel_ring_emit(ring, 0);
1128 intel_ring_emit(ring, 0);
1129 intel_ring_emit(ring, 0);
1130 intel_ring_advance(ring);
1131 }
964} 1132}
965 1133
966static void blt_ring_cleanup(struct intel_ring_buffer *ring) 1134static void blt_ring_cleanup(struct intel_ring_buffer *ring)
@@ -981,47 +1149,54 @@ static const struct intel_ring_buffer gen6_blt_ring = {
981 .init = blt_ring_init, 1149 .init = blt_ring_init,
982 .write_tail = ring_write_tail, 1150 .write_tail = ring_write_tail,
983 .flush = blt_ring_flush, 1151 .flush = blt_ring_flush,
984 .add_request = blt_ring_add_request, 1152 .add_request = gen6_add_request,
985 .get_seqno = ring_status_page_get_seqno, 1153 .get_seqno = ring_get_seqno,
986 .user_irq_get = blt_ring_get_user_irq, 1154 .irq_get = blt_ring_get_irq,
987 .user_irq_put = blt_ring_put_user_irq, 1155 .irq_put = blt_ring_put_irq,
988 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 1156 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
989 .cleanup = blt_ring_cleanup, 1157 .cleanup = blt_ring_cleanup,
990}; 1158};
991 1159
992int intel_init_render_ring_buffer(struct drm_device *dev) 1160int intel_init_render_ring_buffer(struct drm_device *dev)
993{ 1161{
994 drm_i915_private_t *dev_priv = dev->dev_private; 1162 drm_i915_private_t *dev_priv = dev->dev_private;
995 1163 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
996 dev_priv->render_ring = render_ring; 1164
1165 *ring = render_ring;
1166 if (INTEL_INFO(dev)->gen >= 6) {
1167 ring->add_request = gen6_add_request;
1168 } else if (IS_GEN5(dev)) {
1169 ring->add_request = pc_render_add_request;
1170 ring->get_seqno = pc_render_get_seqno;
1171 }
997 1172
998 if (!I915_NEED_GFX_HWS(dev)) { 1173 if (!I915_NEED_GFX_HWS(dev)) {
999 dev_priv->render_ring.status_page.page_addr 1174 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1000 = dev_priv->status_page_dmah->vaddr; 1175 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1001 memset(dev_priv->render_ring.status_page.page_addr,
1002 0, PAGE_SIZE);
1003 } 1176 }
1004 1177
1005 return intel_init_ring_buffer(dev, &dev_priv->render_ring); 1178 return intel_init_ring_buffer(dev, ring);
1006} 1179}
1007 1180
1008int intel_init_bsd_ring_buffer(struct drm_device *dev) 1181int intel_init_bsd_ring_buffer(struct drm_device *dev)
1009{ 1182{
1010 drm_i915_private_t *dev_priv = dev->dev_private; 1183 drm_i915_private_t *dev_priv = dev->dev_private;
1184 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1011 1185
1012 if (IS_GEN6(dev)) 1186 if (IS_GEN6(dev))
1013 dev_priv->bsd_ring = gen6_bsd_ring; 1187 *ring = gen6_bsd_ring;
1014 else 1188 else
1015 dev_priv->bsd_ring = bsd_ring; 1189 *ring = bsd_ring;
1016 1190
1017 return intel_init_ring_buffer(dev, &dev_priv->bsd_ring); 1191 return intel_init_ring_buffer(dev, ring);
1018} 1192}
1019 1193
1020int intel_init_blt_ring_buffer(struct drm_device *dev) 1194int intel_init_blt_ring_buffer(struct drm_device *dev)
1021{ 1195{
1022 drm_i915_private_t *dev_priv = dev->dev_private; 1196 drm_i915_private_t *dev_priv = dev->dev_private;
1197 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1023 1198
1024 dev_priv->blt_ring = gen6_blt_ring; 1199 *ring = gen6_blt_ring;
1025 1200
1026 return intel_init_ring_buffer(dev, &dev_priv->blt_ring); 1201 return intel_init_ring_buffer(dev, ring);
1027} 1202}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index d2cd0f1efeed..8e2e357ad6ee 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -1,22 +1,37 @@
1#ifndef _INTEL_RINGBUFFER_H_ 1#ifndef _INTEL_RINGBUFFER_H_
2#define _INTEL_RINGBUFFER_H_ 2#define _INTEL_RINGBUFFER_H_
3 3
4enum {
5 RCS = 0x0,
6 VCS,
7 BCS,
8 I915_NUM_RINGS,
9};
10
4struct intel_hw_status_page { 11struct intel_hw_status_page {
5 void *page_addr; 12 u32 __iomem *page_addr;
6 unsigned int gfx_addr; 13 unsigned int gfx_addr;
7 struct drm_gem_object *obj; 14 struct drm_i915_gem_object *obj;
8}; 15};
9 16
10#define I915_READ_TAIL(ring) I915_READ(RING_TAIL(ring->mmio_base)) 17#define I915_RING_READ(reg) i915_safe_read(dev_priv, reg)
18
19#define I915_READ_TAIL(ring) I915_RING_READ(RING_TAIL(ring->mmio_base))
11#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val) 20#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val)
12#define I915_READ_START(ring) I915_READ(RING_START(ring->mmio_base)) 21
22#define I915_READ_START(ring) I915_RING_READ(RING_START(ring->mmio_base))
13#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val) 23#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val)
14#define I915_READ_HEAD(ring) I915_READ(RING_HEAD(ring->mmio_base)) 24
25#define I915_READ_HEAD(ring) I915_RING_READ(RING_HEAD(ring->mmio_base))
15#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val) 26#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val)
16#define I915_READ_CTL(ring) I915_READ(RING_CTL(ring->mmio_base)) 27
28#define I915_READ_CTL(ring) I915_RING_READ(RING_CTL(ring->mmio_base))
17#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val) 29#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val)
18 30
19struct drm_i915_gem_execbuffer2; 31#define I915_READ_NOPID(ring) I915_RING_READ(RING_NOPID(ring->mmio_base))
32#define I915_READ_SYNC_0(ring) I915_RING_READ(RING_SYNC_0(ring->mmio_base))
33#define I915_READ_SYNC_1(ring) I915_RING_READ(RING_SYNC_1(ring->mmio_base))
34
20struct intel_ring_buffer { 35struct intel_ring_buffer {
21 const char *name; 36 const char *name;
22 enum intel_ring_id { 37 enum intel_ring_id {
@@ -25,45 +40,36 @@ struct intel_ring_buffer {
25 RING_BLT = 0x4, 40 RING_BLT = 0x4,
26 } id; 41 } id;
27 u32 mmio_base; 42 u32 mmio_base;
28 unsigned long size;
29 void *virtual_start; 43 void *virtual_start;
30 struct drm_device *dev; 44 struct drm_device *dev;
31 struct drm_gem_object *gem_object; 45 struct drm_i915_gem_object *obj;
32 46
33 u32 actual_head; 47 u32 actual_head;
34 u32 head; 48 u32 head;
35 u32 tail; 49 u32 tail;
36 int space; 50 int space;
51 int size;
37 struct intel_hw_status_page status_page; 52 struct intel_hw_status_page status_page;
38 53
39 u32 irq_gem_seqno; /* last seq seem at irq time */ 54 u32 irq_seqno; /* last seq seem at irq time */
40 u32 waiting_gem_seqno; 55 u32 waiting_seqno;
41 int user_irq_refcount; 56 u32 sync_seqno[I915_NUM_RINGS-1];
42 void (*user_irq_get)(struct drm_device *dev, 57 atomic_t irq_refcount;
43 struct intel_ring_buffer *ring); 58 bool __must_check (*irq_get)(struct intel_ring_buffer *ring);
44 void (*user_irq_put)(struct drm_device *dev, 59 void (*irq_put)(struct intel_ring_buffer *ring);
45 struct intel_ring_buffer *ring);
46 60
47 int (*init)(struct drm_device *dev, 61 int (*init)(struct intel_ring_buffer *ring);
48 struct intel_ring_buffer *ring);
49 62
50 void (*write_tail)(struct drm_device *dev, 63 void (*write_tail)(struct intel_ring_buffer *ring,
51 struct intel_ring_buffer *ring,
52 u32 value); 64 u32 value);
53 void (*flush)(struct drm_device *dev, 65 void (*flush)(struct intel_ring_buffer *ring,
54 struct intel_ring_buffer *ring, 66 u32 invalidate_domains,
55 u32 invalidate_domains, 67 u32 flush_domains);
56 u32 flush_domains); 68 int (*add_request)(struct intel_ring_buffer *ring,
57 u32 (*add_request)(struct drm_device *dev, 69 u32 *seqno);
58 struct intel_ring_buffer *ring, 70 u32 (*get_seqno)(struct intel_ring_buffer *ring);
59 u32 flush_domains); 71 int (*dispatch_execbuffer)(struct intel_ring_buffer *ring,
60 u32 (*get_seqno)(struct drm_device *dev, 72 u32 offset, u32 length);
61 struct intel_ring_buffer *ring);
62 int (*dispatch_gem_execbuffer)(struct drm_device *dev,
63 struct intel_ring_buffer *ring,
64 struct drm_i915_gem_execbuffer2 *exec,
65 struct drm_clip_rect *cliprects,
66 uint64_t exec_offset);
67 void (*cleanup)(struct intel_ring_buffer *ring); 73 void (*cleanup)(struct intel_ring_buffer *ring);
68 74
69 /** 75 /**
@@ -96,7 +102,7 @@ struct intel_ring_buffer {
96 /** 102 /**
97 * Do we have some not yet emitted requests outstanding? 103 * Do we have some not yet emitted requests outstanding?
98 */ 104 */
99 bool outstanding_lazy_request; 105 u32 outstanding_lazy_request;
100 106
101 wait_queue_head_t irq_queue; 107 wait_queue_head_t irq_queue;
102 drm_local_map_t map; 108 drm_local_map_t map;
@@ -105,44 +111,54 @@ struct intel_ring_buffer {
105}; 111};
106 112
107static inline u32 113static inline u32
114intel_ring_sync_index(struct intel_ring_buffer *ring,
115 struct intel_ring_buffer *other)
116{
117 int idx;
118
119 /*
120 * cs -> 0 = vcs, 1 = bcs
121 * vcs -> 0 = bcs, 1 = cs,
122 * bcs -> 0 = cs, 1 = vcs.
123 */
124
125 idx = (other - ring) - 1;
126 if (idx < 0)
127 idx += I915_NUM_RINGS;
128
129 return idx;
130}
131
132static inline u32
108intel_read_status_page(struct intel_ring_buffer *ring, 133intel_read_status_page(struct intel_ring_buffer *ring,
109 int reg) 134 int reg)
110{ 135{
111 u32 *regs = ring->status_page.page_addr; 136 return ioread32(ring->status_page.page_addr + reg);
112 return regs[reg];
113} 137}
114 138
115int intel_init_ring_buffer(struct drm_device *dev, 139void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring);
116 struct intel_ring_buffer *ring); 140int __must_check intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n);
117void intel_cleanup_ring_buffer(struct drm_device *dev, 141int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n);
118 struct intel_ring_buffer *ring); 142
119int intel_wait_ring_buffer(struct drm_device *dev, 143static inline void intel_ring_emit(struct intel_ring_buffer *ring,
120 struct intel_ring_buffer *ring, int n); 144 u32 data)
121void intel_ring_begin(struct drm_device *dev,
122 struct intel_ring_buffer *ring, int n);
123
124static inline void intel_ring_emit(struct drm_device *dev,
125 struct intel_ring_buffer *ring,
126 unsigned int data)
127{ 145{
128 unsigned int *virt = ring->virtual_start + ring->tail; 146 iowrite32(data, ring->virtual_start + ring->tail);
129 *virt = data;
130 ring->tail += 4; 147 ring->tail += 4;
131} 148}
132 149
133void intel_ring_advance(struct drm_device *dev, 150void intel_ring_advance(struct intel_ring_buffer *ring);
134 struct intel_ring_buffer *ring);
135 151
136u32 intel_ring_get_seqno(struct drm_device *dev, 152u32 intel_ring_get_seqno(struct intel_ring_buffer *ring);
137 struct intel_ring_buffer *ring); 153int intel_ring_sync(struct intel_ring_buffer *ring,
154 struct intel_ring_buffer *to,
155 u32 seqno);
138 156
139int intel_init_render_ring_buffer(struct drm_device *dev); 157int intel_init_render_ring_buffer(struct drm_device *dev);
140int intel_init_bsd_ring_buffer(struct drm_device *dev); 158int intel_init_bsd_ring_buffer(struct drm_device *dev);
141int intel_init_blt_ring_buffer(struct drm_device *dev); 159int intel_init_blt_ring_buffer(struct drm_device *dev);
142 160
143u32 intel_ring_get_active_head(struct drm_device *dev, 161u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
144 struct intel_ring_buffer *ring); 162void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
145void intel_ring_setup_status_page(struct drm_device *dev,
146 struct intel_ring_buffer *ring);
147 163
148#endif /* _INTEL_RINGBUFFER_H_ */ 164#endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 6bc42fa2a6ec..9d0af36a13ec 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1045,7 +1045,9 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
1045 1045
1046 /* Set the SDVO control regs. */ 1046 /* Set the SDVO control regs. */
1047 if (INTEL_INFO(dev)->gen >= 4) { 1047 if (INTEL_INFO(dev)->gen >= 4) {
1048 sdvox = SDVO_BORDER_ENABLE; 1048 sdvox = 0;
1049 if (INTEL_INFO(dev)->gen < 5)
1050 sdvox |= SDVO_BORDER_ENABLE;
1049 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) 1051 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
1050 sdvox |= SDVO_VSYNC_ACTIVE_HIGH; 1052 sdvox |= SDVO_VSYNC_ACTIVE_HIGH;
1051 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) 1053 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1075,7 +1077,8 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
1075 sdvox |= (pixel_multiplier - 1) << SDVO_PORT_MULTIPLY_SHIFT; 1077 sdvox |= (pixel_multiplier - 1) << SDVO_PORT_MULTIPLY_SHIFT;
1076 } 1078 }
1077 1079
1078 if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL) 1080 if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL &&
1081 INTEL_INFO(dev)->gen < 5)
1079 sdvox |= SDVO_STALL_SELECT; 1082 sdvox |= SDVO_STALL_SELECT;
1080 intel_sdvo_write_sdvox(intel_sdvo, sdvox); 1083 intel_sdvo_write_sdvox(intel_sdvo, sdvox);
1081} 1084}
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 2f7681989316..93206e4eaa6f 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1245,10 +1245,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
1245 int type; 1245 int type;
1246 1246
1247 /* Disable TV interrupts around load detect or we'll recurse */ 1247 /* Disable TV interrupts around load detect or we'll recurse */
1248 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1248 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1249 i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | 1249 i915_disable_pipestat(dev_priv, 0,
1250 PIPE_HOTPLUG_INTERRUPT_ENABLE |
1250 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); 1251 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
1251 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1252 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1252 1253
1253 save_tv_dac = tv_dac = I915_READ(TV_DAC); 1254 save_tv_dac = tv_dac = I915_READ(TV_DAC);
1254 save_tv_ctl = tv_ctl = I915_READ(TV_CTL); 1255 save_tv_ctl = tv_ctl = I915_READ(TV_CTL);
@@ -1301,10 +1302,11 @@ intel_tv_detect_type (struct intel_tv *intel_tv)
1301 I915_WRITE(TV_CTL, save_tv_ctl); 1302 I915_WRITE(TV_CTL, save_tv_ctl);
1302 1303
1303 /* Restore interrupt config */ 1304 /* Restore interrupt config */
1304 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 1305 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1305 i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | 1306 i915_enable_pipestat(dev_priv, 0,
1307 PIPE_HOTPLUG_INTERRUPT_ENABLE |
1306 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); 1308 PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
1307 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 1309 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1308 1310
1309 return type; 1311 return type;
1310} 1312}
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 72730e9ca06c..21d6c29c2d21 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -10,7 +10,7 @@ config DRM_NOUVEAU
10 select FB 10 select FB
11 select FRAMEBUFFER_CONSOLE if !EMBEDDED 11 select FRAMEBUFFER_CONSOLE if !EMBEDDED
12 select FB_BACKLIGHT if DRM_NOUVEAU_BACKLIGHT 12 select FB_BACKLIGHT if DRM_NOUVEAU_BACKLIGHT
13 select ACPI_VIDEO if ACPI 13 select ACPI_VIDEO if ACPI && X86 && BACKLIGHT_CLASS_DEVICE && VIDEO_OUTPUT_CONTROL && INPUT
14 help 14 help
15 Choose this option for open-source nVidia support. 15 Choose this option for open-source nVidia support.
16 16
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index 23fa82d667d6..e12c97fd8db8 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -5,27 +5,32 @@
5ccflags-y := -Iinclude/drm 5ccflags-y := -Iinclude/drm
6nouveau-y := nouveau_drv.o nouveau_state.o nouveau_channel.o nouveau_mem.o \ 6nouveau-y := nouveau_drv.o nouveau_state.o nouveau_channel.o nouveau_mem.o \
7 nouveau_object.o nouveau_irq.o nouveau_notifier.o \ 7 nouveau_object.o nouveau_irq.o nouveau_notifier.o \
8 nouveau_sgdma.o nouveau_dma.o \ 8 nouveau_sgdma.o nouveau_dma.o nouveau_util.o \
9 nouveau_bo.o nouveau_fence.o nouveau_gem.o nouveau_ttm.o \ 9 nouveau_bo.o nouveau_fence.o nouveau_gem.o nouveau_ttm.o \
10 nouveau_hw.o nouveau_calc.o nouveau_bios.o nouveau_i2c.o \ 10 nouveau_hw.o nouveau_calc.o nouveau_bios.o nouveau_i2c.o \
11 nouveau_display.o nouveau_connector.o nouveau_fbcon.o \ 11 nouveau_display.o nouveau_connector.o nouveau_fbcon.o \
12 nouveau_dp.o nouveau_ramht.o \ 12 nouveau_dp.o nouveau_ramht.o \
13 nouveau_pm.o nouveau_volt.o nouveau_perf.o nouveau_temp.o \ 13 nouveau_pm.o nouveau_volt.o nouveau_perf.o nouveau_temp.o \
14 nouveau_mm.o nouveau_vm.o \
14 nv04_timer.o \ 15 nv04_timer.o \
15 nv04_mc.o nv40_mc.o nv50_mc.o \ 16 nv04_mc.o nv40_mc.o nv50_mc.o \
16 nv04_fb.o nv10_fb.o nv30_fb.o nv40_fb.o nv50_fb.o nvc0_fb.o \ 17 nv04_fb.o nv10_fb.o nv30_fb.o nv40_fb.o nv50_fb.o nvc0_fb.o \
17 nv04_fifo.o nv10_fifo.o nv40_fifo.o nv50_fifo.o nvc0_fifo.o \ 18 nv04_fifo.o nv10_fifo.o nv40_fifo.o nv50_fifo.o nvc0_fifo.o \
18 nv04_graph.o nv10_graph.o nv20_graph.o \ 19 nv04_graph.o nv10_graph.o nv20_graph.o \
19 nv40_graph.o nv50_graph.o nvc0_graph.o \ 20 nv40_graph.o nv50_graph.o nvc0_graph.o \
20 nv40_grctx.o nv50_grctx.o \ 21 nv40_grctx.o nv50_grctx.o nvc0_grctx.o \
22 nv84_crypt.o \
21 nv04_instmem.o nv50_instmem.o nvc0_instmem.o \ 23 nv04_instmem.o nv50_instmem.o nvc0_instmem.o \
22 nv50_crtc.o nv50_dac.o nv50_sor.o \ 24 nv50_evo.o nv50_crtc.o nv50_dac.o nv50_sor.o \
23 nv50_cursor.o nv50_display.o nv50_fbcon.o \ 25 nv50_cursor.o nv50_display.o \
24 nv04_dac.o nv04_dfp.o nv04_tv.o nv17_tv.o nv17_tv_modes.o \ 26 nv04_dac.o nv04_dfp.o nv04_tv.o nv17_tv.o nv17_tv_modes.o \
25 nv04_crtc.o nv04_display.o nv04_cursor.o nv04_fbcon.o \ 27 nv04_crtc.o nv04_display.o nv04_cursor.o \
28 nv04_fbcon.o nv50_fbcon.o nvc0_fbcon.o \
26 nv10_gpio.o nv50_gpio.o \ 29 nv10_gpio.o nv50_gpio.o \
27 nv50_calc.o \ 30 nv50_calc.o \
28 nv04_pm.o nv50_pm.o nva3_pm.o 31 nv04_pm.o nv50_pm.o nva3_pm.o \
32 nv50_vram.o nvc0_vram.o \
33 nv50_vm.o nvc0_vm.o
29 34
30nouveau-$(CONFIG_DRM_NOUVEAU_DEBUG) += nouveau_debugfs.o 35nouveau-$(CONFIG_DRM_NOUVEAU_DEBUG) += nouveau_debugfs.o
31nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o 36nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 119152606e4c..a54238058dc5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -130,10 +130,15 @@ static int nouveau_dsm_init(void)
130 130
131static int nouveau_dsm_get_client_id(struct pci_dev *pdev) 131static int nouveau_dsm_get_client_id(struct pci_dev *pdev)
132{ 132{
133 if (nouveau_dsm_priv.dhandle == DEVICE_ACPI_HANDLE(&pdev->dev)) 133 /* easy option one - intel vendor ID means Integrated */
134 if (pdev->vendor == PCI_VENDOR_ID_INTEL)
134 return VGA_SWITCHEROO_IGD; 135 return VGA_SWITCHEROO_IGD;
135 else 136
136 return VGA_SWITCHEROO_DIS; 137 /* is this device on Bus 0? - this may need improving */
138 if (pdev->bus->number == 0)
139 return VGA_SWITCHEROO_IGD;
140
141 return VGA_SWITCHEROO_DIS;
137} 142}
138 143
139static struct vga_switcheroo_handler nouveau_dsm_handler = { 144static struct vga_switcheroo_handler nouveau_dsm_handler = {
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index b2293576f278..d3046559bf05 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -6053,52 +6053,17 @@ static struct dcb_entry *new_dcb_entry(struct dcb_table *dcb)
6053 return entry; 6053 return entry;
6054} 6054}
6055 6055
6056static void fabricate_vga_output(struct dcb_table *dcb, int i2c, int heads) 6056static void fabricate_dcb_output(struct dcb_table *dcb, int type, int i2c,
6057 int heads, int or)
6057{ 6058{
6058 struct dcb_entry *entry = new_dcb_entry(dcb); 6059 struct dcb_entry *entry = new_dcb_entry(dcb);
6059 6060
6060 entry->type = 0; 6061 entry->type = type;
6061 entry->i2c_index = i2c; 6062 entry->i2c_index = i2c;
6062 entry->heads = heads; 6063 entry->heads = heads;
6063 entry->location = DCB_LOC_ON_CHIP; 6064 if (type != OUTPUT_ANALOG)
6064 entry->or = 1; 6065 entry->location = !DCB_LOC_ON_CHIP; /* ie OFF CHIP */
6065} 6066 entry->or = or;
6066
6067static void fabricate_dvi_i_output(struct dcb_table *dcb, bool twoHeads)
6068{
6069 struct dcb_entry *entry = new_dcb_entry(dcb);
6070
6071 entry->type = 2;
6072 entry->i2c_index = LEGACY_I2C_PANEL;
6073 entry->heads = twoHeads ? 3 : 1;
6074 entry->location = !DCB_LOC_ON_CHIP; /* ie OFF CHIP */
6075 entry->or = 1; /* means |0x10 gets set on CRE_LCD__INDEX */
6076 entry->duallink_possible = false; /* SiI164 and co. are single link */
6077
6078#if 0
6079 /*
6080 * For dvi-a either crtc probably works, but my card appears to only
6081 * support dvi-d. "nvidia" still attempts to program it for dvi-a,
6082 * doing the full fp output setup (program 0x6808.. fp dimension regs,
6083 * setting 0x680848 to 0x10000111 to enable, maybe setting 0x680880);
6084 * the monitor picks up the mode res ok and lights up, but no pixel
6085 * data appears, so the board manufacturer probably connected up the
6086 * sync lines, but missed the video traces / components
6087 *
6088 * with this introduction, dvi-a left as an exercise for the reader.
6089 */
6090 fabricate_vga_output(dcb, LEGACY_I2C_PANEL, entry->heads);
6091#endif
6092}
6093
6094static void fabricate_tv_output(struct dcb_table *dcb, bool twoHeads)
6095{
6096 struct dcb_entry *entry = new_dcb_entry(dcb);
6097
6098 entry->type = 1;
6099 entry->i2c_index = LEGACY_I2C_TV;
6100 entry->heads = twoHeads ? 3 : 1;
6101 entry->location = !DCB_LOC_ON_CHIP; /* ie OFF CHIP */
6102} 6067}
6103 6068
6104static bool 6069static bool
@@ -6365,8 +6330,36 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
6365 return true; 6330 return true;
6366} 6331}
6367 6332
6333static void
6334fabricate_dcb_encoder_table(struct drm_device *dev, struct nvbios *bios)
6335{
6336 struct dcb_table *dcb = &bios->dcb;
6337 int all_heads = (nv_two_heads(dev) ? 3 : 1);
6338
6339#ifdef __powerpc__
6340 /* Apple iMac G4 NV17 */
6341 if (of_machine_is_compatible("PowerMac4,5")) {
6342 fabricate_dcb_output(dcb, OUTPUT_TMDS, 0, all_heads, 1);
6343 fabricate_dcb_output(dcb, OUTPUT_ANALOG, 1, all_heads, 2);
6344 return;
6345 }
6346#endif
6347
6348 /* Make up some sane defaults */
6349 fabricate_dcb_output(dcb, OUTPUT_ANALOG, LEGACY_I2C_CRT, 1, 1);
6350
6351 if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
6352 fabricate_dcb_output(dcb, OUTPUT_TV, LEGACY_I2C_TV,
6353 all_heads, 0);
6354
6355 else if (bios->tmds.output0_script_ptr ||
6356 bios->tmds.output1_script_ptr)
6357 fabricate_dcb_output(dcb, OUTPUT_TMDS, LEGACY_I2C_PANEL,
6358 all_heads, 1);
6359}
6360
6368static int 6361static int
6369parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads) 6362parse_dcb_table(struct drm_device *dev, struct nvbios *bios)
6370{ 6363{
6371 struct drm_nouveau_private *dev_priv = dev->dev_private; 6364 struct drm_nouveau_private *dev_priv = dev->dev_private;
6372 struct dcb_table *dcb = &bios->dcb; 6365 struct dcb_table *dcb = &bios->dcb;
@@ -6386,12 +6379,7 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads)
6386 6379
6387 /* this situation likely means a really old card, pre DCB */ 6380 /* this situation likely means a really old card, pre DCB */
6388 if (dcbptr == 0x0) { 6381 if (dcbptr == 0x0) {
6389 NV_INFO(dev, "Assuming a CRT output exists\n"); 6382 fabricate_dcb_encoder_table(dev, bios);
6390 fabricate_vga_output(dcb, LEGACY_I2C_CRT, 1);
6391
6392 if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
6393 fabricate_tv_output(dcb, twoHeads);
6394
6395 return 0; 6383 return 0;
6396 } 6384 }
6397 6385
@@ -6451,21 +6439,7 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios, bool twoHeads)
6451 */ 6439 */
6452 NV_TRACEWARN(dev, "No useful information in BIOS output table; " 6440 NV_TRACEWARN(dev, "No useful information in BIOS output table; "
6453 "adding all possible outputs\n"); 6441 "adding all possible outputs\n");
6454 fabricate_vga_output(dcb, LEGACY_I2C_CRT, 1); 6442 fabricate_dcb_encoder_table(dev, bios);
6455
6456 /*
6457 * Attempt to detect TV before DVI because the test
6458 * for the former is more accurate and it rules the
6459 * latter out.
6460 */
6461 if (nv04_tv_identify(dev,
6462 bios->legacy.i2c_indices.tv) >= 0)
6463 fabricate_tv_output(dcb, twoHeads);
6464
6465 else if (bios->tmds.output0_script_ptr ||
6466 bios->tmds.output1_script_ptr)
6467 fabricate_dvi_i_output(dcb, twoHeads);
6468
6469 return 0; 6443 return 0;
6470 } 6444 }
6471 6445
@@ -6859,7 +6833,7 @@ nouveau_bios_init(struct drm_device *dev)
6859 if (ret) 6833 if (ret)
6860 return ret; 6834 return ret;
6861 6835
6862 ret = parse_dcb_table(dev, bios, nv_two_heads(dev)); 6836 ret = parse_dcb_table(dev, bios);
6863 if (ret) 6837 if (ret)
6864 return ret; 6838 return ret;
6865 6839
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index c41e1c200ef5..a7fae26f4654 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -32,6 +32,8 @@
32#include "nouveau_drm.h" 32#include "nouveau_drm.h"
33#include "nouveau_drv.h" 33#include "nouveau_drv.h"
34#include "nouveau_dma.h" 34#include "nouveau_dma.h"
35#include "nouveau_mm.h"
36#include "nouveau_vm.h"
35 37
36#include <linux/log2.h> 38#include <linux/log2.h>
37#include <linux/slab.h> 39#include <linux/slab.h>
@@ -46,82 +48,51 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
46 if (unlikely(nvbo->gem)) 48 if (unlikely(nvbo->gem))
47 DRM_ERROR("bo %p still attached to GEM object\n", bo); 49 DRM_ERROR("bo %p still attached to GEM object\n", bo);
48 50
49 if (nvbo->tile) 51 nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
50 nv10_mem_expire_tiling(dev, nvbo->tile, NULL); 52 nouveau_vm_put(&nvbo->vma);
51
52 kfree(nvbo); 53 kfree(nvbo);
53} 54}
54 55
55static void 56static void
56nouveau_bo_fixup_align(struct drm_device *dev, 57nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, int *size,
57 uint32_t tile_mode, uint32_t tile_flags, 58 int *page_shift)
58 int *align, int *size)
59{ 59{
60 struct drm_nouveau_private *dev_priv = dev->dev_private; 60 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
61
62 /*
63 * Some of the tile_flags have a periodic structure of N*4096 bytes,
64 * align to to that as well as the page size. Align the size to the
65 * appropriate boundaries. This does imply that sizes are rounded up
66 * 3-7 pages, so be aware of this and do not waste memory by allocating
67 * many small buffers.
68 */
69 if (dev_priv->card_type == NV_50) {
70 uint32_t block_size = dev_priv->vram_size >> 15;
71 int i;
72
73 switch (tile_flags) {
74 case 0x1800:
75 case 0x2800:
76 case 0x4800:
77 case 0x7a00:
78 if (is_power_of_2(block_size)) {
79 for (i = 1; i < 10; i++) {
80 *align = 12 * i * block_size;
81 if (!(*align % 65536))
82 break;
83 }
84 } else {
85 for (i = 1; i < 10; i++) {
86 *align = 8 * i * block_size;
87 if (!(*align % 65536))
88 break;
89 }
90 }
91 *size = roundup(*size, *align);
92 break;
93 default:
94 break;
95 }
96 61
97 } else { 62 if (dev_priv->card_type < NV_50) {
98 if (tile_mode) { 63 if (nvbo->tile_mode) {
99 if (dev_priv->chipset >= 0x40) { 64 if (dev_priv->chipset >= 0x40) {
100 *align = 65536; 65 *align = 65536;
101 *size = roundup(*size, 64 * tile_mode); 66 *size = roundup(*size, 64 * nvbo->tile_mode);
102 67
103 } else if (dev_priv->chipset >= 0x30) { 68 } else if (dev_priv->chipset >= 0x30) {
104 *align = 32768; 69 *align = 32768;
105 *size = roundup(*size, 64 * tile_mode); 70 *size = roundup(*size, 64 * nvbo->tile_mode);
106 71
107 } else if (dev_priv->chipset >= 0x20) { 72 } else if (dev_priv->chipset >= 0x20) {
108 *align = 16384; 73 *align = 16384;
109 *size = roundup(*size, 64 * tile_mode); 74 *size = roundup(*size, 64 * nvbo->tile_mode);
110 75
111 } else if (dev_priv->chipset >= 0x10) { 76 } else if (dev_priv->chipset >= 0x10) {
112 *align = 16384; 77 *align = 16384;
113 *size = roundup(*size, 32 * tile_mode); 78 *size = roundup(*size, 32 * nvbo->tile_mode);
114 } 79 }
115 } 80 }
81 } else {
82 if (likely(dev_priv->chan_vm)) {
83 if (*size > 256 * 1024)
84 *page_shift = dev_priv->chan_vm->lpg_shift;
85 else
86 *page_shift = dev_priv->chan_vm->spg_shift;
87 } else {
88 *page_shift = 12;
89 }
90
91 *size = roundup(*size, (1 << *page_shift));
92 *align = max((1 << *page_shift), *align);
116 } 93 }
117 94
118 /* ALIGN works only on powers of two. */
119 *size = roundup(*size, PAGE_SIZE); 95 *size = roundup(*size, PAGE_SIZE);
120
121 if (dev_priv->card_type == NV_50) {
122 *size = roundup(*size, 65536);
123 *align = max(65536, *align);
124 }
125} 96}
126 97
127int 98int
@@ -132,7 +103,7 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
132{ 103{
133 struct drm_nouveau_private *dev_priv = dev->dev_private; 104 struct drm_nouveau_private *dev_priv = dev->dev_private;
134 struct nouveau_bo *nvbo; 105 struct nouveau_bo *nvbo;
135 int ret = 0; 106 int ret = 0, page_shift = 0;
136 107
137 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 108 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
138 if (!nvbo) 109 if (!nvbo)
@@ -145,10 +116,18 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
145 nvbo->tile_flags = tile_flags; 116 nvbo->tile_flags = tile_flags;
146 nvbo->bo.bdev = &dev_priv->ttm.bdev; 117 nvbo->bo.bdev = &dev_priv->ttm.bdev;
147 118
148 nouveau_bo_fixup_align(dev, tile_mode, nouveau_bo_tile_layout(nvbo), 119 nouveau_bo_fixup_align(nvbo, &align, &size, &page_shift);
149 &align, &size);
150 align >>= PAGE_SHIFT; 120 align >>= PAGE_SHIFT;
151 121
122 if (!nvbo->no_vm && dev_priv->chan_vm) {
123 ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
124 NV_MEM_ACCESS_RW, &nvbo->vma);
125 if (ret) {
126 kfree(nvbo);
127 return ret;
128 }
129 }
130
152 nouveau_bo_placement_set(nvbo, flags, 0); 131 nouveau_bo_placement_set(nvbo, flags, 0);
153 132
154 nvbo->channel = chan; 133 nvbo->channel = chan;
@@ -161,6 +140,11 @@ nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
161 } 140 }
162 nvbo->channel = NULL; 141 nvbo->channel = NULL;
163 142
143 if (nvbo->vma.node) {
144 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
145 nvbo->bo.offset = nvbo->vma.offset;
146 }
147
164 *pnvbo = nvbo; 148 *pnvbo = nvbo;
165 return 0; 149 return 0;
166} 150}
@@ -244,7 +228,7 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
244 228
245 nouveau_bo_placement_set(nvbo, memtype, 0); 229 nouveau_bo_placement_set(nvbo, memtype, 0);
246 230
247 ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false); 231 ret = nouveau_bo_validate(nvbo, false, false, false);
248 if (ret == 0) { 232 if (ret == 0) {
249 switch (bo->mem.mem_type) { 233 switch (bo->mem.mem_type) {
250 case TTM_PL_VRAM: 234 case TTM_PL_VRAM:
@@ -280,7 +264,7 @@ nouveau_bo_unpin(struct nouveau_bo *nvbo)
280 264
281 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); 265 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
282 266
283 ret = ttm_bo_validate(bo, &nvbo->placement, false, false, false); 267 ret = nouveau_bo_validate(nvbo, false, false, false);
284 if (ret == 0) { 268 if (ret == 0) {
285 switch (bo->mem.mem_type) { 269 switch (bo->mem.mem_type) {
286 case TTM_PL_VRAM: 270 case TTM_PL_VRAM:
@@ -319,6 +303,25 @@ nouveau_bo_unmap(struct nouveau_bo *nvbo)
319 ttm_bo_kunmap(&nvbo->kmap); 303 ttm_bo_kunmap(&nvbo->kmap);
320} 304}
321 305
306int
307nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
308 bool no_wait_reserve, bool no_wait_gpu)
309{
310 int ret;
311
312 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible,
313 no_wait_reserve, no_wait_gpu);
314 if (ret)
315 return ret;
316
317 if (nvbo->vma.node) {
318 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
319 nvbo->bo.offset = nvbo->vma.offset;
320 }
321
322 return 0;
323}
324
322u16 325u16
323nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) 326nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
324{ 327{
@@ -410,37 +413,40 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
410 man->default_caching = TTM_PL_FLAG_CACHED; 413 man->default_caching = TTM_PL_FLAG_CACHED;
411 break; 414 break;
412 case TTM_PL_VRAM: 415 case TTM_PL_VRAM:
413 man->func = &ttm_bo_manager_func; 416 if (dev_priv->card_type >= NV_50) {
417 man->func = &nouveau_vram_manager;
418 man->io_reserve_fastpath = false;
419 man->use_io_reserve_lru = true;
420 } else {
421 man->func = &ttm_bo_manager_func;
422 }
414 man->flags = TTM_MEMTYPE_FLAG_FIXED | 423 man->flags = TTM_MEMTYPE_FLAG_FIXED |
415 TTM_MEMTYPE_FLAG_MAPPABLE; 424 TTM_MEMTYPE_FLAG_MAPPABLE;
416 man->available_caching = TTM_PL_FLAG_UNCACHED | 425 man->available_caching = TTM_PL_FLAG_UNCACHED |
417 TTM_PL_FLAG_WC; 426 TTM_PL_FLAG_WC;
418 man->default_caching = TTM_PL_FLAG_WC; 427 man->default_caching = TTM_PL_FLAG_WC;
419 if (dev_priv->card_type == NV_50)
420 man->gpu_offset = 0x40000000;
421 else
422 man->gpu_offset = 0;
423 break; 428 break;
424 case TTM_PL_TT: 429 case TTM_PL_TT:
425 man->func = &ttm_bo_manager_func; 430 man->func = &ttm_bo_manager_func;
426 switch (dev_priv->gart_info.type) { 431 switch (dev_priv->gart_info.type) {
427 case NOUVEAU_GART_AGP: 432 case NOUVEAU_GART_AGP:
428 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 433 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
429 man->available_caching = TTM_PL_FLAG_UNCACHED; 434 man->available_caching = TTM_PL_FLAG_UNCACHED |
430 man->default_caching = TTM_PL_FLAG_UNCACHED; 435 TTM_PL_FLAG_WC;
436 man->default_caching = TTM_PL_FLAG_WC;
431 break; 437 break;
432 case NOUVEAU_GART_SGDMA: 438 case NOUVEAU_GART_SGDMA:
433 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | 439 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
434 TTM_MEMTYPE_FLAG_CMA; 440 TTM_MEMTYPE_FLAG_CMA;
435 man->available_caching = TTM_PL_MASK_CACHING; 441 man->available_caching = TTM_PL_MASK_CACHING;
436 man->default_caching = TTM_PL_FLAG_CACHED; 442 man->default_caching = TTM_PL_FLAG_CACHED;
443 man->gpu_offset = dev_priv->gart_info.aper_base;
437 break; 444 break;
438 default: 445 default:
439 NV_ERROR(dev, "Unknown GART type: %d\n", 446 NV_ERROR(dev, "Unknown GART type: %d\n",
440 dev_priv->gart_info.type); 447 dev_priv->gart_info.type);
441 return -EINVAL; 448 return -EINVAL;
442 } 449 }
443 man->gpu_offset = dev_priv->vm_gart_base;
444 break; 450 break;
445 default: 451 default:
446 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); 452 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
@@ -485,16 +491,9 @@ nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
485 if (ret) 491 if (ret)
486 return ret; 492 return ret;
487 493
488 if (nvbo->channel) {
489 ret = nouveau_fence_sync(fence, nvbo->channel);
490 if (ret)
491 goto out;
492 }
493
494 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict, 494 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
495 no_wait_reserve, no_wait_gpu, new_mem); 495 no_wait_reserve, no_wait_gpu, new_mem);
496out: 496 nouveau_fence_unref(&fence);
497 nouveau_fence_unref((void *)&fence);
498 return ret; 497 return ret;
499} 498}
500 499
@@ -516,6 +515,58 @@ nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
516} 515}
517 516
518static int 517static int
518nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
519 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
520{
521 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
522 struct nouveau_bo *nvbo = nouveau_bo(bo);
523 u64 src_offset = old_mem->start << PAGE_SHIFT;
524 u64 dst_offset = new_mem->start << PAGE_SHIFT;
525 u32 page_count = new_mem->num_pages;
526 int ret;
527
528 if (!nvbo->no_vm) {
529 if (old_mem->mem_type == TTM_PL_VRAM)
530 src_offset = nvbo->vma.offset;
531 else
532 src_offset += dev_priv->gart_info.aper_base;
533
534 if (new_mem->mem_type == TTM_PL_VRAM)
535 dst_offset = nvbo->vma.offset;
536 else
537 dst_offset += dev_priv->gart_info.aper_base;
538 }
539
540 page_count = new_mem->num_pages;
541 while (page_count) {
542 int line_count = (page_count > 2047) ? 2047 : page_count;
543
544 ret = RING_SPACE(chan, 12);
545 if (ret)
546 return ret;
547
548 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2);
549 OUT_RING (chan, upper_32_bits(dst_offset));
550 OUT_RING (chan, lower_32_bits(dst_offset));
551 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6);
552 OUT_RING (chan, upper_32_bits(src_offset));
553 OUT_RING (chan, lower_32_bits(src_offset));
554 OUT_RING (chan, PAGE_SIZE); /* src_pitch */
555 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */
556 OUT_RING (chan, PAGE_SIZE); /* line_length */
557 OUT_RING (chan, line_count);
558 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1);
559 OUT_RING (chan, 0x00100110);
560
561 page_count -= line_count;
562 src_offset += (PAGE_SIZE * line_count);
563 dst_offset += (PAGE_SIZE * line_count);
564 }
565
566 return 0;
567}
568
569static int
519nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 570nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
520 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 571 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
521{ 572{
@@ -529,14 +580,14 @@ nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
529 dst_offset = new_mem->start << PAGE_SHIFT; 580 dst_offset = new_mem->start << PAGE_SHIFT;
530 if (!nvbo->no_vm) { 581 if (!nvbo->no_vm) {
531 if (old_mem->mem_type == TTM_PL_VRAM) 582 if (old_mem->mem_type == TTM_PL_VRAM)
532 src_offset += dev_priv->vm_vram_base; 583 src_offset = nvbo->vma.offset;
533 else 584 else
534 src_offset += dev_priv->vm_gart_base; 585 src_offset += dev_priv->gart_info.aper_base;
535 586
536 if (new_mem->mem_type == TTM_PL_VRAM) 587 if (new_mem->mem_type == TTM_PL_VRAM)
537 dst_offset += dev_priv->vm_vram_base; 588 dst_offset = nvbo->vma.offset;
538 else 589 else
539 dst_offset += dev_priv->vm_gart_base; 590 dst_offset += dev_priv->gart_info.aper_base;
540 } 591 }
541 592
542 ret = RING_SPACE(chan, 3); 593 ret = RING_SPACE(chan, 3);
@@ -683,17 +734,27 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
683 int ret; 734 int ret;
684 735
685 chan = nvbo->channel; 736 chan = nvbo->channel;
686 if (!chan || nvbo->no_vm) 737 if (!chan || nvbo->no_vm) {
687 chan = dev_priv->channel; 738 chan = dev_priv->channel;
739 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
740 }
688 741
689 if (dev_priv->card_type < NV_50) 742 if (dev_priv->card_type < NV_50)
690 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 743 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
691 else 744 else
745 if (dev_priv->card_type < NV_C0)
692 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 746 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
693 if (ret) 747 else
694 return ret; 748 ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
749 if (ret == 0) {
750 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
751 no_wait_reserve,
752 no_wait_gpu, new_mem);
753 }
695 754
696 return nouveau_bo_move_accel_cleanup(chan, nvbo, evict, no_wait_reserve, no_wait_gpu, new_mem); 755 if (chan == dev_priv->channel)
756 mutex_unlock(&chan->mutex);
757 return ret;
697} 758}
698 759
699static int 760static int
@@ -771,7 +832,6 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
771 struct drm_device *dev = dev_priv->dev; 832 struct drm_device *dev = dev_priv->dev;
772 struct nouveau_bo *nvbo = nouveau_bo(bo); 833 struct nouveau_bo *nvbo = nouveau_bo(bo);
773 uint64_t offset; 834 uint64_t offset;
774 int ret;
775 835
776 if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) { 836 if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) {
777 /* Nothing to do. */ 837 /* Nothing to do. */
@@ -781,18 +841,12 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
781 841
782 offset = new_mem->start << PAGE_SHIFT; 842 offset = new_mem->start << PAGE_SHIFT;
783 843
784 if (dev_priv->card_type == NV_50) { 844 if (dev_priv->chan_vm) {
785 ret = nv50_mem_vm_bind_linear(dev, 845 nouveau_vm_map(&nvbo->vma, new_mem->mm_node);
786 offset + dev_priv->vm_vram_base,
787 new_mem->size,
788 nouveau_bo_tile_layout(nvbo),
789 offset);
790 if (ret)
791 return ret;
792
793 } else if (dev_priv->card_type >= NV_10) { 846 } else if (dev_priv->card_type >= NV_10) {
794 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, 847 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
795 nvbo->tile_mode); 848 nvbo->tile_mode,
849 nvbo->tile_flags);
796 } 850 }
797 851
798 return 0; 852 return 0;
@@ -808,9 +862,7 @@ nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
808 862
809 if (dev_priv->card_type >= NV_10 && 863 if (dev_priv->card_type >= NV_10 &&
810 dev_priv->card_type < NV_50) { 864 dev_priv->card_type < NV_50) {
811 if (*old_tile) 865 nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
812 nv10_mem_expire_tiling(dev, *old_tile, bo->sync_obj);
813
814 *old_tile = new_tile; 866 *old_tile = new_tile;
815 } 867 }
816} 868}
@@ -879,6 +931,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
879 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 931 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
880 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 932 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
881 struct drm_device *dev = dev_priv->dev; 933 struct drm_device *dev = dev_priv->dev;
934 int ret;
882 935
883 mem->bus.addr = NULL; 936 mem->bus.addr = NULL;
884 mem->bus.offset = 0; 937 mem->bus.offset = 0;
@@ -901,9 +954,40 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
901#endif 954#endif
902 break; 955 break;
903 case TTM_PL_VRAM: 956 case TTM_PL_VRAM:
904 mem->bus.offset = mem->start << PAGE_SHIFT; 957 {
958 struct nouveau_vram *vram = mem->mm_node;
959 u8 page_shift;
960
961 if (!dev_priv->bar1_vm) {
962 mem->bus.offset = mem->start << PAGE_SHIFT;
963 mem->bus.base = pci_resource_start(dev->pdev, 1);
964 mem->bus.is_iomem = true;
965 break;
966 }
967
968 if (dev_priv->card_type == NV_C0)
969 page_shift = vram->page_shift;
970 else
971 page_shift = 12;
972
973 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
974 page_shift, NV_MEM_ACCESS_RW,
975 &vram->bar_vma);
976 if (ret)
977 return ret;
978
979 nouveau_vm_map(&vram->bar_vma, vram);
980 if (ret) {
981 nouveau_vm_put(&vram->bar_vma);
982 return ret;
983 }
984
985 mem->bus.offset = vram->bar_vma.offset;
986 if (dev_priv->card_type == NV_50) /*XXX*/
987 mem->bus.offset -= 0x0020000000ULL;
905 mem->bus.base = pci_resource_start(dev->pdev, 1); 988 mem->bus.base = pci_resource_start(dev->pdev, 1);
906 mem->bus.is_iomem = true; 989 mem->bus.is_iomem = true;
990 }
907 break; 991 break;
908 default: 992 default:
909 return -EINVAL; 993 return -EINVAL;
@@ -914,6 +998,17 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
914static void 998static void
915nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 999nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
916{ 1000{
1001 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
1002 struct nouveau_vram *vram = mem->mm_node;
1003
1004 if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
1005 return;
1006
1007 if (!vram->bar_vma.node)
1008 return;
1009
1010 nouveau_vm_unmap(&vram->bar_vma);
1011 nouveau_vm_put(&vram->bar_vma);
917} 1012}
918 1013
919static int 1014static int
@@ -939,7 +1034,23 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
939 nvbo->placement.fpfn = 0; 1034 nvbo->placement.fpfn = 0;
940 nvbo->placement.lpfn = dev_priv->fb_mappable_pages; 1035 nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
941 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0); 1036 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0);
942 return ttm_bo_validate(bo, &nvbo->placement, false, true, false); 1037 return nouveau_bo_validate(nvbo, false, true, false);
1038}
1039
1040void
1041nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
1042{
1043 struct nouveau_fence *old_fence;
1044
1045 if (likely(fence))
1046 nouveau_fence_ref(fence);
1047
1048 spin_lock(&nvbo->bo.bdev->fence_lock);
1049 old_fence = nvbo->bo.sync_obj;
1050 nvbo->bo.sync_obj = fence;
1051 spin_unlock(&nvbo->bo.bdev->fence_lock);
1052
1053 nouveau_fence_unref(&old_fence);
943} 1054}
944 1055
945struct ttm_bo_driver nouveau_bo_driver = { 1056struct ttm_bo_driver nouveau_bo_driver = {
@@ -949,11 +1060,11 @@ struct ttm_bo_driver nouveau_bo_driver = {
949 .evict_flags = nouveau_bo_evict_flags, 1060 .evict_flags = nouveau_bo_evict_flags,
950 .move = nouveau_bo_move, 1061 .move = nouveau_bo_move,
951 .verify_access = nouveau_bo_verify_access, 1062 .verify_access = nouveau_bo_verify_access,
952 .sync_obj_signaled = nouveau_fence_signalled, 1063 .sync_obj_signaled = __nouveau_fence_signalled,
953 .sync_obj_wait = nouveau_fence_wait, 1064 .sync_obj_wait = __nouveau_fence_wait,
954 .sync_obj_flush = nouveau_fence_flush, 1065 .sync_obj_flush = __nouveau_fence_flush,
955 .sync_obj_unref = nouveau_fence_unref, 1066 .sync_obj_unref = __nouveau_fence_unref,
956 .sync_obj_ref = nouveau_fence_ref, 1067 .sync_obj_ref = __nouveau_fence_ref,
957 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1068 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
958 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1069 .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
959 .io_mem_free = &nouveau_ttm_io_mem_free, 1070 .io_mem_free = &nouveau_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c
index 373950e34814..3960d66d7aba 100644
--- a/drivers/gpu/drm/nouveau/nouveau_channel.c
+++ b/drivers/gpu/drm/nouveau/nouveau_channel.c
@@ -38,23 +38,28 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
38 int ret; 38 int ret;
39 39
40 if (dev_priv->card_type >= NV_50) { 40 if (dev_priv->card_type >= NV_50) {
41 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0, 41 if (dev_priv->card_type < NV_C0) {
42 dev_priv->vm_end, NV_DMA_ACCESS_RO, 42 ret = nouveau_gpuobj_dma_new(chan,
43 NV_DMA_TARGET_AGP, &pushbuf); 43 NV_CLASS_DMA_IN_MEMORY, 0,
44 (1ULL << 40),
45 NV_MEM_ACCESS_RO,
46 NV_MEM_TARGET_VM,
47 &pushbuf);
48 }
44 chan->pushbuf_base = pb->bo.offset; 49 chan->pushbuf_base = pb->bo.offset;
45 } else 50 } else
46 if (pb->bo.mem.mem_type == TTM_PL_TT) { 51 if (pb->bo.mem.mem_type == TTM_PL_TT) {
47 ret = nouveau_gpuobj_gart_dma_new(chan, 0, 52 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
48 dev_priv->gart_info.aper_size, 53 dev_priv->gart_info.aper_size,
49 NV_DMA_ACCESS_RO, &pushbuf, 54 NV_MEM_ACCESS_RO,
50 NULL); 55 NV_MEM_TARGET_GART, &pushbuf);
51 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; 56 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
52 } else 57 } else
53 if (dev_priv->card_type != NV_04) { 58 if (dev_priv->card_type != NV_04) {
54 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0, 59 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
55 dev_priv->fb_available_size, 60 dev_priv->fb_available_size,
56 NV_DMA_ACCESS_RO, 61 NV_MEM_ACCESS_RO,
57 NV_DMA_TARGET_VIDMEM, &pushbuf); 62 NV_MEM_TARGET_VRAM, &pushbuf);
58 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; 63 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
59 } else { 64 } else {
60 /* NV04 cmdbuf hack, from original ddx.. not sure of it's 65 /* NV04 cmdbuf hack, from original ddx.. not sure of it's
@@ -62,17 +67,16 @@ nouveau_channel_pushbuf_ctxdma_init(struct nouveau_channel *chan)
62 * VRAM. 67 * VRAM.
63 */ 68 */
64 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 69 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
65 pci_resource_start(dev->pdev, 70 pci_resource_start(dev->pdev, 1),
66 1),
67 dev_priv->fb_available_size, 71 dev_priv->fb_available_size,
68 NV_DMA_ACCESS_RO, 72 NV_MEM_ACCESS_RO,
69 NV_DMA_TARGET_PCI, &pushbuf); 73 NV_MEM_TARGET_PCI, &pushbuf);
70 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT; 74 chan->pushbuf_base = pb->bo.mem.start << PAGE_SHIFT;
71 } 75 }
72 76
73 nouveau_gpuobj_ref(pushbuf, &chan->pushbuf); 77 nouveau_gpuobj_ref(pushbuf, &chan->pushbuf);
74 nouveau_gpuobj_ref(NULL, &pushbuf); 78 nouveau_gpuobj_ref(NULL, &pushbuf);
75 return 0; 79 return ret;
76} 80}
77 81
78static struct nouveau_bo * 82static struct nouveau_bo *
@@ -100,6 +104,13 @@ nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
100 return NULL; 104 return NULL;
101 } 105 }
102 106
107 ret = nouveau_bo_map(pushbuf);
108 if (ret) {
109 nouveau_bo_unpin(pushbuf);
110 nouveau_bo_ref(NULL, &pushbuf);
111 return NULL;
112 }
113
103 return pushbuf; 114 return pushbuf;
104} 115}
105 116
@@ -107,74 +118,59 @@ nouveau_channel_user_pushbuf_alloc(struct drm_device *dev)
107int 118int
108nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret, 119nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
109 struct drm_file *file_priv, 120 struct drm_file *file_priv,
110 uint32_t vram_handle, uint32_t tt_handle) 121 uint32_t vram_handle, uint32_t gart_handle)
111{ 122{
112 struct drm_nouveau_private *dev_priv = dev->dev_private; 123 struct drm_nouveau_private *dev_priv = dev->dev_private;
113 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
114 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo; 124 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
115 struct nouveau_channel *chan; 125 struct nouveau_channel *chan;
116 int channel, user; 126 unsigned long flags;
117 int ret; 127 int ret;
118 128
119 /* 129 /* allocate and lock channel structure */
120 * Alright, here is the full story 130 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
121 * Nvidia cards have multiple hw fifo contexts (praise them for that, 131 if (!chan)
122 * no complicated crash-prone context switches) 132 return -ENOMEM;
123 * We allocate a new context for each app and let it write to it 133 chan->dev = dev;
124 * directly (woo, full userspace command submission !) 134 chan->file_priv = file_priv;
125 * When there are no more contexts, you lost 135 chan->vram_handle = vram_handle;
126 */ 136 chan->gart_handle = gart_handle;
127 for (channel = 0; channel < pfifo->channels; channel++) { 137
128 if (dev_priv->fifos[channel] == NULL) 138 kref_init(&chan->ref);
139 atomic_set(&chan->users, 1);
140 mutex_init(&chan->mutex);
141 mutex_lock(&chan->mutex);
142
143 /* allocate hw channel id */
144 spin_lock_irqsave(&dev_priv->channels.lock, flags);
145 for (chan->id = 0; chan->id < pfifo->channels; chan->id++) {
146 if (!dev_priv->channels.ptr[chan->id]) {
147 nouveau_channel_ref(chan, &dev_priv->channels.ptr[chan->id]);
129 break; 148 break;
149 }
130 } 150 }
151 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
131 152
132 /* no more fifos. you lost. */ 153 if (chan->id == pfifo->channels) {
133 if (channel == pfifo->channels) 154 mutex_unlock(&chan->mutex);
134 return -EINVAL; 155 kfree(chan);
156 return -ENODEV;
157 }
135 158
136 dev_priv->fifos[channel] = kzalloc(sizeof(struct nouveau_channel), 159 NV_DEBUG(dev, "initialising channel %d\n", chan->id);
137 GFP_KERNEL);
138 if (!dev_priv->fifos[channel])
139 return -ENOMEM;
140 chan = dev_priv->fifos[channel];
141 INIT_LIST_HEAD(&chan->nvsw.vbl_wait); 160 INIT_LIST_HEAD(&chan->nvsw.vbl_wait);
161 INIT_LIST_HEAD(&chan->nvsw.flip);
142 INIT_LIST_HEAD(&chan->fence.pending); 162 INIT_LIST_HEAD(&chan->fence.pending);
143 chan->dev = dev;
144 chan->id = channel;
145 chan->file_priv = file_priv;
146 chan->vram_handle = vram_handle;
147 chan->gart_handle = tt_handle;
148
149 NV_INFO(dev, "Allocating FIFO number %d\n", channel);
150 163
151 /* Allocate DMA push buffer */ 164 /* Allocate DMA push buffer */
152 chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev); 165 chan->pushbuf_bo = nouveau_channel_user_pushbuf_alloc(dev);
153 if (!chan->pushbuf_bo) { 166 if (!chan->pushbuf_bo) {
154 ret = -ENOMEM; 167 ret = -ENOMEM;
155 NV_ERROR(dev, "pushbuf %d\n", ret); 168 NV_ERROR(dev, "pushbuf %d\n", ret);
156 nouveau_channel_free(chan); 169 nouveau_channel_put(&chan);
157 return ret; 170 return ret;
158 } 171 }
159 172
160 nouveau_dma_pre_init(chan); 173 nouveau_dma_pre_init(chan);
161
162 /* Locate channel's user control regs */
163 if (dev_priv->card_type < NV_40)
164 user = NV03_USER(channel);
165 else
166 if (dev_priv->card_type < NV_50)
167 user = NV40_USER(channel);
168 else
169 user = NV50_USER(channel);
170
171 chan->user = ioremap(pci_resource_start(dev->pdev, 0) + user,
172 PAGE_SIZE);
173 if (!chan->user) {
174 NV_ERROR(dev, "ioremap of regs failed.\n");
175 nouveau_channel_free(chan);
176 return -ENOMEM;
177 }
178 chan->user_put = 0x40; 174 chan->user_put = 0x40;
179 chan->user_get = 0x44; 175 chan->user_get = 0x44;
180 176
@@ -182,15 +178,15 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
182 ret = nouveau_notifier_init_channel(chan); 178 ret = nouveau_notifier_init_channel(chan);
183 if (ret) { 179 if (ret) {
184 NV_ERROR(dev, "ntfy %d\n", ret); 180 NV_ERROR(dev, "ntfy %d\n", ret);
185 nouveau_channel_free(chan); 181 nouveau_channel_put(&chan);
186 return ret; 182 return ret;
187 } 183 }
188 184
189 /* Setup channel's default objects */ 185 /* Setup channel's default objects */
190 ret = nouveau_gpuobj_channel_init(chan, vram_handle, tt_handle); 186 ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
191 if (ret) { 187 if (ret) {
192 NV_ERROR(dev, "gpuobj %d\n", ret); 188 NV_ERROR(dev, "gpuobj %d\n", ret);
193 nouveau_channel_free(chan); 189 nouveau_channel_put(&chan);
194 return ret; 190 return ret;
195 } 191 }
196 192
@@ -198,24 +194,17 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
198 ret = nouveau_channel_pushbuf_ctxdma_init(chan); 194 ret = nouveau_channel_pushbuf_ctxdma_init(chan);
199 if (ret) { 195 if (ret) {
200 NV_ERROR(dev, "pbctxdma %d\n", ret); 196 NV_ERROR(dev, "pbctxdma %d\n", ret);
201 nouveau_channel_free(chan); 197 nouveau_channel_put(&chan);
202 return ret; 198 return ret;
203 } 199 }
204 200
205 /* disable the fifo caches */ 201 /* disable the fifo caches */
206 pfifo->reassign(dev, false); 202 pfifo->reassign(dev, false);
207 203
208 /* Create a graphics context for new channel */
209 ret = pgraph->create_context(chan);
210 if (ret) {
211 nouveau_channel_free(chan);
212 return ret;
213 }
214
215 /* Construct inital RAMFC for new channel */ 204 /* Construct inital RAMFC for new channel */
216 ret = pfifo->create_context(chan); 205 ret = pfifo->create_context(chan);
217 if (ret) { 206 if (ret) {
218 nouveau_channel_free(chan); 207 nouveau_channel_put(&chan);
219 return ret; 208 return ret;
220 } 209 }
221 210
@@ -225,83 +214,111 @@ nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
225 if (!ret) 214 if (!ret)
226 ret = nouveau_fence_channel_init(chan); 215 ret = nouveau_fence_channel_init(chan);
227 if (ret) { 216 if (ret) {
228 nouveau_channel_free(chan); 217 nouveau_channel_put(&chan);
229 return ret; 218 return ret;
230 } 219 }
231 220
232 nouveau_debugfs_channel_init(chan); 221 nouveau_debugfs_channel_init(chan);
233 222
234 NV_INFO(dev, "%s: initialised FIFO %d\n", __func__, channel); 223 NV_DEBUG(dev, "channel %d initialised\n", chan->id);
235 *chan_ret = chan; 224 *chan_ret = chan;
236 return 0; 225 return 0;
237} 226}
238 227
239/* stops a fifo */ 228struct nouveau_channel *
229nouveau_channel_get_unlocked(struct nouveau_channel *ref)
230{
231 struct nouveau_channel *chan = NULL;
232
233 if (likely(ref && atomic_inc_not_zero(&ref->users)))
234 nouveau_channel_ref(ref, &chan);
235
236 return chan;
237}
238
239struct nouveau_channel *
240nouveau_channel_get(struct drm_device *dev, struct drm_file *file_priv, int id)
241{
242 struct drm_nouveau_private *dev_priv = dev->dev_private;
243 struct nouveau_channel *chan;
244 unsigned long flags;
245
246 if (unlikely(id < 0 || id >= NOUVEAU_MAX_CHANNEL_NR))
247 return ERR_PTR(-EINVAL);
248
249 spin_lock_irqsave(&dev_priv->channels.lock, flags);
250 chan = nouveau_channel_get_unlocked(dev_priv->channels.ptr[id]);
251 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
252
253 if (unlikely(!chan))
254 return ERR_PTR(-EINVAL);
255
256 if (unlikely(file_priv && chan->file_priv != file_priv)) {
257 nouveau_channel_put_unlocked(&chan);
258 return ERR_PTR(-EINVAL);
259 }
260
261 mutex_lock(&chan->mutex);
262 return chan;
263}
264
240void 265void
241nouveau_channel_free(struct nouveau_channel *chan) 266nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
242{ 267{
268 struct nouveau_channel *chan = *pchan;
243 struct drm_device *dev = chan->dev; 269 struct drm_device *dev = chan->dev;
244 struct drm_nouveau_private *dev_priv = dev->dev_private; 270 struct drm_nouveau_private *dev_priv = dev->dev_private;
245 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
246 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo; 271 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
272 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
273 struct nouveau_crypt_engine *pcrypt = &dev_priv->engine.crypt;
247 unsigned long flags; 274 unsigned long flags;
248 int ret;
249 275
250 NV_INFO(dev, "%s: freeing fifo %d\n", __func__, chan->id); 276 /* decrement the refcount, and we're done if there's still refs */
277 if (likely(!atomic_dec_and_test(&chan->users))) {
278 nouveau_channel_ref(NULL, pchan);
279 return;
280 }
251 281
282 /* noone wants the channel anymore */
283 NV_DEBUG(dev, "freeing channel %d\n", chan->id);
252 nouveau_debugfs_channel_fini(chan); 284 nouveau_debugfs_channel_fini(chan);
253 285
254 /* Give outstanding push buffers a chance to complete */ 286 /* give it chance to idle */
255 nouveau_fence_update(chan); 287 nouveau_channel_idle(chan);
256 if (chan->fence.sequence != chan->fence.sequence_ack) {
257 struct nouveau_fence *fence = NULL;
258 288
259 ret = nouveau_fence_new(chan, &fence, true); 289 /* ensure all outstanding fences are signaled. they should be if the
260 if (ret == 0) {
261 ret = nouveau_fence_wait(fence, NULL, false, false);
262 nouveau_fence_unref((void *)&fence);
263 }
264
265 if (ret)
266 NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
267 }
268
269 /* Ensure all outstanding fences are signaled. They should be if the
270 * above attempts at idling were OK, but if we failed this'll tell TTM 290 * above attempts at idling were OK, but if we failed this'll tell TTM
271 * we're done with the buffers. 291 * we're done with the buffers.
272 */ 292 */
273 nouveau_fence_channel_fini(chan); 293 nouveau_fence_channel_fini(chan);
274 294
275 /* This will prevent pfifo from switching channels. */ 295 /* boot it off the hardware */
276 pfifo->reassign(dev, false); 296 pfifo->reassign(dev, false);
277 297
278 /* We want to give pgraph a chance to idle and get rid of all potential 298 /* We want to give pgraph a chance to idle and get rid of all
279 * errors. We need to do this before the lock, otherwise the irq handler 299 * potential errors. We need to do this without the context
280 * is unable to process them. 300 * switch lock held, otherwise the irq handler is unable to
301 * process them.
281 */ 302 */
282 if (pgraph->channel(dev) == chan) 303 if (pgraph->channel(dev) == chan)
283 nouveau_wait_for_idle(dev); 304 nouveau_wait_for_idle(dev);
284 305
285 spin_lock_irqsave(&dev_priv->context_switch_lock, flags); 306 /* destroy the engine specific contexts */
286
287 pgraph->fifo_access(dev, false);
288 if (pgraph->channel(dev) == chan)
289 pgraph->unload_context(dev);
290 pgraph->destroy_context(chan);
291 pgraph->fifo_access(dev, true);
292
293 if (pfifo->channel_id(dev) == chan->id) {
294 pfifo->disable(dev);
295 pfifo->unload_context(dev);
296 pfifo->enable(dev);
297 }
298 pfifo->destroy_context(chan); 307 pfifo->destroy_context(chan);
308 pgraph->destroy_context(chan);
309 if (pcrypt->destroy_context)
310 pcrypt->destroy_context(chan);
299 311
300 pfifo->reassign(dev, true); 312 pfifo->reassign(dev, true);
301 313
302 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags); 314 /* aside from its resources, the channel should now be dead,
315 * remove it from the channel list
316 */
317 spin_lock_irqsave(&dev_priv->channels.lock, flags);
318 nouveau_channel_ref(NULL, &dev_priv->channels.ptr[chan->id]);
319 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
303 320
304 /* Release the channel's resources */ 321 /* destroy any resources the channel owned */
305 nouveau_gpuobj_ref(NULL, &chan->pushbuf); 322 nouveau_gpuobj_ref(NULL, &chan->pushbuf);
306 if (chan->pushbuf_bo) { 323 if (chan->pushbuf_bo) {
307 nouveau_bo_unmap(chan->pushbuf_bo); 324 nouveau_bo_unmap(chan->pushbuf_bo);
@@ -310,44 +327,80 @@ nouveau_channel_free(struct nouveau_channel *chan)
310 } 327 }
311 nouveau_gpuobj_channel_takedown(chan); 328 nouveau_gpuobj_channel_takedown(chan);
312 nouveau_notifier_takedown_channel(chan); 329 nouveau_notifier_takedown_channel(chan);
313 if (chan->user)
314 iounmap(chan->user);
315 330
316 dev_priv->fifos[chan->id] = NULL; 331 nouveau_channel_ref(NULL, pchan);
332}
333
334void
335nouveau_channel_put(struct nouveau_channel **pchan)
336{
337 mutex_unlock(&(*pchan)->mutex);
338 nouveau_channel_put_unlocked(pchan);
339}
340
341static void
342nouveau_channel_del(struct kref *ref)
343{
344 struct nouveau_channel *chan =
345 container_of(ref, struct nouveau_channel, ref);
346
317 kfree(chan); 347 kfree(chan);
318} 348}
319 349
350void
351nouveau_channel_ref(struct nouveau_channel *chan,
352 struct nouveau_channel **pchan)
353{
354 if (chan)
355 kref_get(&chan->ref);
356
357 if (*pchan)
358 kref_put(&(*pchan)->ref, nouveau_channel_del);
359
360 *pchan = chan;
361}
362
363void
364nouveau_channel_idle(struct nouveau_channel *chan)
365{
366 struct drm_device *dev = chan->dev;
367 struct nouveau_fence *fence = NULL;
368 int ret;
369
370 nouveau_fence_update(chan);
371
372 if (chan->fence.sequence != chan->fence.sequence_ack) {
373 ret = nouveau_fence_new(chan, &fence, true);
374 if (!ret) {
375 ret = nouveau_fence_wait(fence, false, false);
376 nouveau_fence_unref(&fence);
377 }
378
379 if (ret)
380 NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
381 }
382}
383
320/* cleans up all the fifos from file_priv */ 384/* cleans up all the fifos from file_priv */
321void 385void
322nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv) 386nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
323{ 387{
324 struct drm_nouveau_private *dev_priv = dev->dev_private; 388 struct drm_nouveau_private *dev_priv = dev->dev_private;
325 struct nouveau_engine *engine = &dev_priv->engine; 389 struct nouveau_engine *engine = &dev_priv->engine;
390 struct nouveau_channel *chan;
326 int i; 391 int i;
327 392
328 NV_DEBUG(dev, "clearing FIFO enables from file_priv\n"); 393 NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
329 for (i = 0; i < engine->fifo.channels; i++) { 394 for (i = 0; i < engine->fifo.channels; i++) {
330 struct nouveau_channel *chan = dev_priv->fifos[i]; 395 chan = nouveau_channel_get(dev, file_priv, i);
396 if (IS_ERR(chan))
397 continue;
331 398
332 if (chan && chan->file_priv == file_priv) 399 atomic_dec(&chan->users);
333 nouveau_channel_free(chan); 400 nouveau_channel_put(&chan);
334 } 401 }
335} 402}
336 403
337int
338nouveau_channel_owner(struct drm_device *dev, struct drm_file *file_priv,
339 int channel)
340{
341 struct drm_nouveau_private *dev_priv = dev->dev_private;
342 struct nouveau_engine *engine = &dev_priv->engine;
343
344 if (channel >= engine->fifo.channels)
345 return 0;
346 if (dev_priv->fifos[channel] == NULL)
347 return 0;
348
349 return (dev_priv->fifos[channel]->file_priv == file_priv);
350}
351 404
352/*********************************** 405/***********************************
353 * ioctls wrapping the functions 406 * ioctls wrapping the functions
@@ -383,36 +436,44 @@ nouveau_ioctl_fifo_alloc(struct drm_device *dev, void *data,
383 else 436 else
384 init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART; 437 init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
385 438
386 init->subchan[0].handle = NvM2MF; 439 if (dev_priv->card_type < NV_C0) {
387 if (dev_priv->card_type < NV_50) 440 init->subchan[0].handle = NvM2MF;
388 init->subchan[0].grclass = 0x0039; 441 if (dev_priv->card_type < NV_50)
389 else 442 init->subchan[0].grclass = 0x0039;
390 init->subchan[0].grclass = 0x5039; 443 else
391 init->subchan[1].handle = NvSw; 444 init->subchan[0].grclass = 0x5039;
392 init->subchan[1].grclass = NV_SW; 445 init->subchan[1].handle = NvSw;
393 init->nr_subchan = 2; 446 init->subchan[1].grclass = NV_SW;
447 init->nr_subchan = 2;
448 } else {
449 init->subchan[0].handle = 0x9039;
450 init->subchan[0].grclass = 0x9039;
451 init->nr_subchan = 1;
452 }
394 453
395 /* Named memory object area */ 454 /* Named memory object area */
396 ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem, 455 ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem,
397 &init->notifier_handle); 456 &init->notifier_handle);
398 if (ret) {
399 nouveau_channel_free(chan);
400 return ret;
401 }
402 457
403 return 0; 458 if (ret == 0)
459 atomic_inc(&chan->users); /* userspace reference */
460 nouveau_channel_put(&chan);
461 return ret;
404} 462}
405 463
406static int 464static int
407nouveau_ioctl_fifo_free(struct drm_device *dev, void *data, 465nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
408 struct drm_file *file_priv) 466 struct drm_file *file_priv)
409{ 467{
410 struct drm_nouveau_channel_free *cfree = data; 468 struct drm_nouveau_channel_free *req = data;
411 struct nouveau_channel *chan; 469 struct nouveau_channel *chan;
412 470
413 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(cfree->channel, file_priv, chan); 471 chan = nouveau_channel_get(dev, file_priv, req->channel);
472 if (IS_ERR(chan))
473 return PTR_ERR(chan);
414 474
415 nouveau_channel_free(chan); 475 atomic_dec(&chan->users);
476 nouveau_channel_put(&chan);
416 return 0; 477 return 0;
417} 478}
418 479
@@ -421,18 +482,18 @@ nouveau_ioctl_fifo_free(struct drm_device *dev, void *data,
421 ***********************************/ 482 ***********************************/
422 483
423struct drm_ioctl_desc nouveau_ioctls[] = { 484struct drm_ioctl_desc nouveau_ioctls[] = {
424 DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_AUTH), 485 DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH),
425 DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 486 DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
426 DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_AUTH), 487 DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_ioctl_fifo_alloc, DRM_UNLOCKED|DRM_AUTH),
427 DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_AUTH), 488 DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_ioctl_fifo_free, DRM_UNLOCKED|DRM_AUTH),
428 DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_AUTH), 489 DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH),
429 DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_AUTH), 490 DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_ioctl_notifier_alloc, DRM_UNLOCKED|DRM_AUTH),
430 DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_AUTH), 491 DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH),
431 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH), 492 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH),
432 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH), 493 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH),
433 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH), 494 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
434 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_AUTH), 495 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
435 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_AUTH), 496 DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH),
436}; 497};
437 498
438int nouveau_max_ioctl = DRM_ARRAY_SIZE(nouveau_ioctls); 499int nouveau_max_ioctl = DRM_ARRAY_SIZE(nouveau_ioctls);
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 52c356e9a3d1..a21e00076839 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -37,6 +37,8 @@
37#include "nouveau_connector.h" 37#include "nouveau_connector.h"
38#include "nouveau_hw.h" 38#include "nouveau_hw.h"
39 39
40static void nouveau_connector_hotplug(void *, int);
41
40static struct nouveau_encoder * 42static struct nouveau_encoder *
41find_encoder_by_type(struct drm_connector *connector, int type) 43find_encoder_by_type(struct drm_connector *connector, int type)
42{ 44{
@@ -94,22 +96,30 @@ nouveau_connector_bpp(struct drm_connector *connector)
94} 96}
95 97
96static void 98static void
97nouveau_connector_destroy(struct drm_connector *drm_connector) 99nouveau_connector_destroy(struct drm_connector *connector)
98{ 100{
99 struct nouveau_connector *nv_connector = 101 struct nouveau_connector *nv_connector = nouveau_connector(connector);
100 nouveau_connector(drm_connector); 102 struct drm_nouveau_private *dev_priv;
103 struct nouveau_gpio_engine *pgpio;
101 struct drm_device *dev; 104 struct drm_device *dev;
102 105
103 if (!nv_connector) 106 if (!nv_connector)
104 return; 107 return;
105 108
106 dev = nv_connector->base.dev; 109 dev = nv_connector->base.dev;
110 dev_priv = dev->dev_private;
107 NV_DEBUG_KMS(dev, "\n"); 111 NV_DEBUG_KMS(dev, "\n");
108 112
113 pgpio = &dev_priv->engine.gpio;
114 if (pgpio->irq_unregister) {
115 pgpio->irq_unregister(dev, nv_connector->dcb->gpio_tag,
116 nouveau_connector_hotplug, connector);
117 }
118
109 kfree(nv_connector->edid); 119 kfree(nv_connector->edid);
110 drm_sysfs_connector_remove(drm_connector); 120 drm_sysfs_connector_remove(connector);
111 drm_connector_cleanup(drm_connector); 121 drm_connector_cleanup(connector);
112 kfree(drm_connector); 122 kfree(connector);
113} 123}
114 124
115static struct nouveau_i2c_chan * 125static struct nouveau_i2c_chan *
@@ -760,6 +770,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
760{ 770{
761 const struct drm_connector_funcs *funcs = &nouveau_connector_funcs; 771 const struct drm_connector_funcs *funcs = &nouveau_connector_funcs;
762 struct drm_nouveau_private *dev_priv = dev->dev_private; 772 struct drm_nouveau_private *dev_priv = dev->dev_private;
773 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
763 struct nouveau_connector *nv_connector = NULL; 774 struct nouveau_connector *nv_connector = NULL;
764 struct dcb_connector_table_entry *dcb = NULL; 775 struct dcb_connector_table_entry *dcb = NULL;
765 struct drm_connector *connector; 776 struct drm_connector *connector;
@@ -876,6 +887,11 @@ nouveau_connector_create(struct drm_device *dev, int index)
876 break; 887 break;
877 } 888 }
878 889
890 if (pgpio->irq_register) {
891 pgpio->irq_register(dev, nv_connector->dcb->gpio_tag,
892 nouveau_connector_hotplug, connector);
893 }
894
879 drm_sysfs_connector_add(connector); 895 drm_sysfs_connector_add(connector);
880 dcb->drm = connector; 896 dcb->drm = connector;
881 return dcb->drm; 897 return dcb->drm;
@@ -886,3 +902,29 @@ fail:
886 return ERR_PTR(ret); 902 return ERR_PTR(ret);
887 903
888} 904}
905
906static void
907nouveau_connector_hotplug(void *data, int plugged)
908{
909 struct drm_connector *connector = data;
910 struct drm_device *dev = connector->dev;
911
912 NV_INFO(dev, "%splugged %s\n", plugged ? "" : "un",
913 drm_get_connector_name(connector));
914
915 if (connector->encoder && connector->encoder->crtc &&
916 connector->encoder->crtc->enabled) {
917 struct nouveau_encoder *nv_encoder = nouveau_encoder(connector->encoder);
918 struct drm_encoder_helper_funcs *helper =
919 connector->encoder->helper_private;
920
921 if (nv_encoder->dcb->type == OUTPUT_DP) {
922 if (plugged)
923 helper->dpms(connector->encoder, DRM_MODE_DPMS_ON);
924 else
925 helper->dpms(connector->encoder, DRM_MODE_DPMS_OFF);
926 }
927 }
928
929 drm_helper_hpd_irq_event(dev);
930}
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 2e11fd65b4dd..505c6bfb4d75 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -29,6 +29,9 @@
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_fb.h" 30#include "nouveau_fb.h"
31#include "nouveau_fbcon.h" 31#include "nouveau_fbcon.h"
32#include "nouveau_hw.h"
33#include "nouveau_crtc.h"
34#include "nouveau_dma.h"
32 35
33static void 36static void
34nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb) 37nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
@@ -104,3 +107,207 @@ const struct drm_mode_config_funcs nouveau_mode_config_funcs = {
104 .output_poll_changed = nouveau_fbcon_output_poll_changed, 107 .output_poll_changed = nouveau_fbcon_output_poll_changed,
105}; 108};
106 109
110int
111nouveau_vblank_enable(struct drm_device *dev, int crtc)
112{
113 struct drm_nouveau_private *dev_priv = dev->dev_private;
114
115 if (dev_priv->card_type >= NV_50)
116 nv_mask(dev, NV50_PDISPLAY_INTR_EN_1, 0,
117 NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc));
118 else
119 NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0,
120 NV_PCRTC_INTR_0_VBLANK);
121
122 return 0;
123}
124
125void
126nouveau_vblank_disable(struct drm_device *dev, int crtc)
127{
128 struct drm_nouveau_private *dev_priv = dev->dev_private;
129
130 if (dev_priv->card_type >= NV_50)
131 nv_mask(dev, NV50_PDISPLAY_INTR_EN_1,
132 NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc), 0);
133 else
134 NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0, 0);
135}
136
137static int
138nouveau_page_flip_reserve(struct nouveau_bo *old_bo,
139 struct nouveau_bo *new_bo)
140{
141 int ret;
142
143 ret = nouveau_bo_pin(new_bo, TTM_PL_FLAG_VRAM);
144 if (ret)
145 return ret;
146
147 ret = ttm_bo_reserve(&new_bo->bo, false, false, false, 0);
148 if (ret)
149 goto fail;
150
151 ret = ttm_bo_reserve(&old_bo->bo, false, false, false, 0);
152 if (ret)
153 goto fail_unreserve;
154
155 return 0;
156
157fail_unreserve:
158 ttm_bo_unreserve(&new_bo->bo);
159fail:
160 nouveau_bo_unpin(new_bo);
161 return ret;
162}
163
164static void
165nouveau_page_flip_unreserve(struct nouveau_bo *old_bo,
166 struct nouveau_bo *new_bo,
167 struct nouveau_fence *fence)
168{
169 nouveau_bo_fence(new_bo, fence);
170 ttm_bo_unreserve(&new_bo->bo);
171
172 nouveau_bo_fence(old_bo, fence);
173 ttm_bo_unreserve(&old_bo->bo);
174
175 nouveau_bo_unpin(old_bo);
176}
177
178static int
179nouveau_page_flip_emit(struct nouveau_channel *chan,
180 struct nouveau_bo *old_bo,
181 struct nouveau_bo *new_bo,
182 struct nouveau_page_flip_state *s,
183 struct nouveau_fence **pfence)
184{
185 struct drm_device *dev = chan->dev;
186 unsigned long flags;
187 int ret;
188
189 /* Queue it to the pending list */
190 spin_lock_irqsave(&dev->event_lock, flags);
191 list_add_tail(&s->head, &chan->nvsw.flip);
192 spin_unlock_irqrestore(&dev->event_lock, flags);
193
194 /* Synchronize with the old framebuffer */
195 ret = nouveau_fence_sync(old_bo->bo.sync_obj, chan);
196 if (ret)
197 goto fail;
198
199 /* Emit the pageflip */
200 ret = RING_SPACE(chan, 2);
201 if (ret)
202 goto fail;
203
204 BEGIN_RING(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
205 OUT_RING(chan, 0);
206 FIRE_RING(chan);
207
208 ret = nouveau_fence_new(chan, pfence, true);
209 if (ret)
210 goto fail;
211
212 return 0;
213fail:
214 spin_lock_irqsave(&dev->event_lock, flags);
215 list_del(&s->head);
216 spin_unlock_irqrestore(&dev->event_lock, flags);
217 return ret;
218}
219
220int
221nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
222 struct drm_pending_vblank_event *event)
223{
224 struct drm_device *dev = crtc->dev;
225 struct drm_nouveau_private *dev_priv = dev->dev_private;
226 struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->fb)->nvbo;
227 struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
228 struct nouveau_page_flip_state *s;
229 struct nouveau_channel *chan;
230 struct nouveau_fence *fence;
231 int ret;
232
233 if (dev_priv->engine.graph.accel_blocked)
234 return -ENODEV;
235
236 s = kzalloc(sizeof(*s), GFP_KERNEL);
237 if (!s)
238 return -ENOMEM;
239
240 /* Don't let the buffers go away while we flip */
241 ret = nouveau_page_flip_reserve(old_bo, new_bo);
242 if (ret)
243 goto fail_free;
244
245 /* Initialize a page flip struct */
246 *s = (struct nouveau_page_flip_state)
247 { { }, s->event, nouveau_crtc(crtc)->index,
248 fb->bits_per_pixel, fb->pitch, crtc->x, crtc->y,
249 new_bo->bo.offset };
250
251 /* Choose the channel the flip will be handled in */
252 chan = nouveau_fence_channel(new_bo->bo.sync_obj);
253 if (!chan)
254 chan = nouveau_channel_get_unlocked(dev_priv->channel);
255 mutex_lock(&chan->mutex);
256
257 /* Emit a page flip */
258 ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence);
259 nouveau_channel_put(&chan);
260 if (ret)
261 goto fail_unreserve;
262
263 /* Update the crtc struct and cleanup */
264 crtc->fb = fb;
265
266 nouveau_page_flip_unreserve(old_bo, new_bo, fence);
267 nouveau_fence_unref(&fence);
268 return 0;
269
270fail_unreserve:
271 nouveau_page_flip_unreserve(old_bo, new_bo, NULL);
272fail_free:
273 kfree(s);
274 return ret;
275}
276
277int
278nouveau_finish_page_flip(struct nouveau_channel *chan,
279 struct nouveau_page_flip_state *ps)
280{
281 struct drm_device *dev = chan->dev;
282 struct nouveau_page_flip_state *s;
283 unsigned long flags;
284
285 spin_lock_irqsave(&dev->event_lock, flags);
286
287 if (list_empty(&chan->nvsw.flip)) {
288 NV_ERROR(dev, "Unexpected pageflip in channel %d.\n", chan->id);
289 spin_unlock_irqrestore(&dev->event_lock, flags);
290 return -EINVAL;
291 }
292
293 s = list_first_entry(&chan->nvsw.flip,
294 struct nouveau_page_flip_state, head);
295 if (s->event) {
296 struct drm_pending_vblank_event *e = s->event;
297 struct timeval now;
298
299 do_gettimeofday(&now);
300 e->event.sequence = 0;
301 e->event.tv_sec = now.tv_sec;
302 e->event.tv_usec = now.tv_usec;
303 list_add_tail(&e->base.link, &e->base.file_priv->event_list);
304 wake_up_interruptible(&e->base.file_priv->event_wait);
305 }
306
307 list_del(&s->head);
308 *ps = *s;
309 kfree(s);
310
311 spin_unlock_irqrestore(&dev->event_lock, flags);
312 return 0;
313}
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 82581e600dcd..65699bfaaaea 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -36,7 +36,7 @@ nouveau_dma_pre_init(struct nouveau_channel *chan)
36 struct drm_nouveau_private *dev_priv = chan->dev->dev_private; 36 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
37 struct nouveau_bo *pushbuf = chan->pushbuf_bo; 37 struct nouveau_bo *pushbuf = chan->pushbuf_bo;
38 38
39 if (dev_priv->card_type == NV_50) { 39 if (dev_priv->card_type >= NV_50) {
40 const int ib_size = pushbuf->bo.mem.size / 2; 40 const int ib_size = pushbuf->bo.mem.size / 2;
41 41
42 chan->dma.ib_base = (pushbuf->bo.mem.size - ib_size) >> 2; 42 chan->dma.ib_base = (pushbuf->bo.mem.size - ib_size) >> 2;
@@ -59,17 +59,26 @@ nouveau_dma_init(struct nouveau_channel *chan)
59{ 59{
60 struct drm_device *dev = chan->dev; 60 struct drm_device *dev = chan->dev;
61 struct drm_nouveau_private *dev_priv = dev->dev_private; 61 struct drm_nouveau_private *dev_priv = dev->dev_private;
62 struct nouveau_gpuobj *obj = NULL;
63 int ret, i; 62 int ret, i;
64 63
65 /* Create NV_MEMORY_TO_MEMORY_FORMAT for buffer moves */ 64 if (dev_priv->card_type >= NV_C0) {
66 ret = nouveau_gpuobj_gr_new(chan, dev_priv->card_type < NV_50 ? 65 ret = nouveau_gpuobj_gr_new(chan, 0x9039, 0x9039);
67 0x0039 : 0x5039, &obj); 66 if (ret)
68 if (ret) 67 return ret;
69 return ret; 68
69 ret = RING_SPACE(chan, 2);
70 if (ret)
71 return ret;
72
73 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0000, 1);
74 OUT_RING (chan, 0x00009039);
75 FIRE_RING (chan);
76 return 0;
77 }
70 78
71 ret = nouveau_ramht_insert(chan, NvM2MF, obj); 79 /* Create NV_MEMORY_TO_MEMORY_FORMAT for buffer moves */
72 nouveau_gpuobj_ref(NULL, &obj); 80 ret = nouveau_gpuobj_gr_new(chan, NvM2MF, dev_priv->card_type < NV_50 ?
81 0x0039 : 0x5039);
73 if (ret) 82 if (ret)
74 return ret; 83 return ret;
75 84
@@ -78,11 +87,6 @@ nouveau_dma_init(struct nouveau_channel *chan)
78 if (ret) 87 if (ret)
79 return ret; 88 return ret;
80 89
81 /* Map push buffer */
82 ret = nouveau_bo_map(chan->pushbuf_bo);
83 if (ret)
84 return ret;
85
86 /* Insert NOPS for NOUVEAU_DMA_SKIPS */ 90 /* Insert NOPS for NOUVEAU_DMA_SKIPS */
87 ret = RING_SPACE(chan, NOUVEAU_DMA_SKIPS); 91 ret = RING_SPACE(chan, NOUVEAU_DMA_SKIPS);
88 if (ret) 92 if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h
index d578c21d3c8d..c36f1763feaa 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.h
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.h
@@ -77,7 +77,8 @@ enum {
77 /* G80+ display objects */ 77 /* G80+ display objects */
78 NvEvoVRAM = 0x01000000, 78 NvEvoVRAM = 0x01000000,
79 NvEvoFB16 = 0x01000001, 79 NvEvoFB16 = 0x01000001,
80 NvEvoFB32 = 0x01000002 80 NvEvoFB32 = 0x01000002,
81 NvEvoVRAM_LP = 0x01000003
81}; 82};
82 83
83#define NV_MEMORY_TO_MEMORY_FORMAT 0x00000039 84#define NV_MEMORY_TO_MEMORY_FORMAT 0x00000039
@@ -125,6 +126,12 @@ extern void
125OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords); 126OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords);
126 127
127static inline void 128static inline void
129BEGIN_NVC0(struct nouveau_channel *chan, int op, int subc, int mthd, int size)
130{
131 OUT_RING(chan, (op << 28) | (size << 16) | (subc << 13) | (mthd >> 2));
132}
133
134static inline void
128BEGIN_RING(struct nouveau_channel *chan, int subc, int mthd, int size) 135BEGIN_RING(struct nouveau_channel *chan, int subc, int mthd, int size)
129{ 136{
130 OUT_RING(chan, (subc << 13) | (size << 18) | mthd); 137 OUT_RING(chan, (subc << 13) | (size << 18) | mthd);
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 4562f309ae3d..38d599554bce 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -279,7 +279,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder)
279 struct bit_displayport_encoder_table *dpe; 279 struct bit_displayport_encoder_table *dpe;
280 int dpe_headerlen; 280 int dpe_headerlen;
281 uint8_t config[4], status[3]; 281 uint8_t config[4], status[3];
282 bool cr_done, cr_max_vs, eq_done; 282 bool cr_done, cr_max_vs, eq_done, hpd_state;
283 int ret = 0, i, tries, voltage; 283 int ret = 0, i, tries, voltage;
284 284
285 NV_DEBUG_KMS(dev, "link training!!\n"); 285 NV_DEBUG_KMS(dev, "link training!!\n");
@@ -297,7 +297,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder)
297 /* disable hotplug detect, this flips around on some panels during 297 /* disable hotplug detect, this flips around on some panels during
298 * link training. 298 * link training.
299 */ 299 */
300 pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, false); 300 hpd_state = pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, false);
301 301
302 if (dpe->script0) { 302 if (dpe->script0) {
303 NV_DEBUG_KMS(dev, "SOR-%d: running DP script 0\n", nv_encoder->or); 303 NV_DEBUG_KMS(dev, "SOR-%d: running DP script 0\n", nv_encoder->or);
@@ -439,7 +439,7 @@ stop:
439 } 439 }
440 440
441 /* re-enable hotplug detect */ 441 /* re-enable hotplug detect */
442 pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, true); 442 pgpio->irq_enable(dev, nv_connector->dcb->gpio_tag, hpd_state);
443 443
444 return eq_done; 444 return eq_done;
445} 445}
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
index 90875494a65a..13bb672a16f4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.c
@@ -115,6 +115,10 @@ MODULE_PARM_DESC(perflvl_wr, "Allow perflvl changes (warning: dangerous!)\n");
115int nouveau_perflvl_wr; 115int nouveau_perflvl_wr;
116module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400); 116module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400);
117 117
118MODULE_PARM_DESC(msi, "Enable MSI (default: off)\n");
119int nouveau_msi;
120module_param_named(msi, nouveau_msi, int, 0400);
121
118int nouveau_fbpercrtc; 122int nouveau_fbpercrtc;
119#if 0 123#if 0
120module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400); 124module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400);
@@ -167,6 +171,9 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
167 if (pm_state.event == PM_EVENT_PRETHAW) 171 if (pm_state.event == PM_EVENT_PRETHAW)
168 return 0; 172 return 0;
169 173
174 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
175 return 0;
176
170 NV_INFO(dev, "Disabling fbcon acceleration...\n"); 177 NV_INFO(dev, "Disabling fbcon acceleration...\n");
171 nouveau_fbcon_save_disable_accel(dev); 178 nouveau_fbcon_save_disable_accel(dev);
172 179
@@ -193,23 +200,10 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
193 200
194 NV_INFO(dev, "Idling channels...\n"); 201 NV_INFO(dev, "Idling channels...\n");
195 for (i = 0; i < pfifo->channels; i++) { 202 for (i = 0; i < pfifo->channels; i++) {
196 struct nouveau_fence *fence = NULL; 203 chan = dev_priv->channels.ptr[i];
197
198 chan = dev_priv->fifos[i];
199 if (!chan || (dev_priv->card_type >= NV_50 &&
200 chan == dev_priv->fifos[0]))
201 continue;
202
203 ret = nouveau_fence_new(chan, &fence, true);
204 if (ret == 0) {
205 ret = nouveau_fence_wait(fence, NULL, false, false);
206 nouveau_fence_unref((void *)&fence);
207 }
208 204
209 if (ret) { 205 if (chan && chan->pushbuf_bo)
210 NV_ERROR(dev, "Failed to idle channel %d for suspend\n", 206 nouveau_channel_idle(chan);
211 chan->id);
212 }
213 } 207 }
214 208
215 pgraph->fifo_access(dev, false); 209 pgraph->fifo_access(dev, false);
@@ -219,17 +213,17 @@ nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
219 pfifo->unload_context(dev); 213 pfifo->unload_context(dev);
220 pgraph->unload_context(dev); 214 pgraph->unload_context(dev);
221 215
222 NV_INFO(dev, "Suspending GPU objects...\n"); 216 ret = pinstmem->suspend(dev);
223 ret = nouveau_gpuobj_suspend(dev);
224 if (ret) { 217 if (ret) {
225 NV_ERROR(dev, "... failed: %d\n", ret); 218 NV_ERROR(dev, "... failed: %d\n", ret);
226 goto out_abort; 219 goto out_abort;
227 } 220 }
228 221
229 ret = pinstmem->suspend(dev); 222 NV_INFO(dev, "Suspending GPU objects...\n");
223 ret = nouveau_gpuobj_suspend(dev);
230 if (ret) { 224 if (ret) {
231 NV_ERROR(dev, "... failed: %d\n", ret); 225 NV_ERROR(dev, "... failed: %d\n", ret);
232 nouveau_gpuobj_suspend_cleanup(dev); 226 pinstmem->resume(dev);
233 goto out_abort; 227 goto out_abort;
234 } 228 }
235 229
@@ -263,6 +257,9 @@ nouveau_pci_resume(struct pci_dev *pdev)
263 struct drm_crtc *crtc; 257 struct drm_crtc *crtc;
264 int ret, i; 258 int ret, i;
265 259
260 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
261 return 0;
262
266 nouveau_fbcon_save_disable_accel(dev); 263 nouveau_fbcon_save_disable_accel(dev);
267 264
268 NV_INFO(dev, "We're back, enabling device...\n"); 265 NV_INFO(dev, "We're back, enabling device...\n");
@@ -294,17 +291,18 @@ nouveau_pci_resume(struct pci_dev *pdev)
294 } 291 }
295 } 292 }
296 293
294 NV_INFO(dev, "Restoring GPU objects...\n");
295 nouveau_gpuobj_resume(dev);
296
297 NV_INFO(dev, "Reinitialising engines...\n"); 297 NV_INFO(dev, "Reinitialising engines...\n");
298 engine->instmem.resume(dev); 298 engine->instmem.resume(dev);
299 engine->mc.init(dev); 299 engine->mc.init(dev);
300 engine->timer.init(dev); 300 engine->timer.init(dev);
301 engine->fb.init(dev); 301 engine->fb.init(dev);
302 engine->graph.init(dev); 302 engine->graph.init(dev);
303 engine->crypt.init(dev);
303 engine->fifo.init(dev); 304 engine->fifo.init(dev);
304 305
305 NV_INFO(dev, "Restoring GPU objects...\n");
306 nouveau_gpuobj_resume(dev);
307
308 nouveau_irq_postinstall(dev); 306 nouveau_irq_postinstall(dev);
309 307
310 /* Re-write SKIPS, they'll have been lost over the suspend */ 308 /* Re-write SKIPS, they'll have been lost over the suspend */
@@ -313,7 +311,7 @@ nouveau_pci_resume(struct pci_dev *pdev)
313 int j; 311 int j;
314 312
315 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 313 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
316 chan = dev_priv->fifos[i]; 314 chan = dev_priv->channels.ptr[i];
317 if (!chan || !chan->pushbuf_bo) 315 if (!chan || !chan->pushbuf_bo)
318 continue; 316 continue;
319 317
@@ -347,13 +345,11 @@ nouveau_pci_resume(struct pci_dev *pdev)
347 345
348 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 346 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
349 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); 347 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
348 u32 offset = nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT;
350 349
351 nv_crtc->cursor.set_offset(nv_crtc, 350 nv_crtc->cursor.set_offset(nv_crtc, offset);
352 nv_crtc->cursor.nvbo->bo.offset -
353 dev_priv->vm_vram_base);
354
355 nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x, 351 nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
356 nv_crtc->cursor_saved_y); 352 nv_crtc->cursor_saved_y);
357 } 353 }
358 354
359 /* Force CLUT to get re-loaded during modeset */ 355 /* Force CLUT to get re-loaded during modeset */
@@ -393,6 +389,9 @@ static struct drm_driver driver = {
393 .irq_postinstall = nouveau_irq_postinstall, 389 .irq_postinstall = nouveau_irq_postinstall,
394 .irq_uninstall = nouveau_irq_uninstall, 390 .irq_uninstall = nouveau_irq_uninstall,
395 .irq_handler = nouveau_irq_handler, 391 .irq_handler = nouveau_irq_handler,
392 .get_vblank_counter = drm_vblank_count,
393 .enable_vblank = nouveau_vblank_enable,
394 .disable_vblank = nouveau_vblank_disable,
396 .reclaim_buffers = drm_core_reclaim_buffers, 395 .reclaim_buffers = drm_core_reclaim_buffers,
397 .ioctls = nouveau_ioctls, 396 .ioctls = nouveau_ioctls,
398 .fops = { 397 .fops = {
@@ -403,6 +402,7 @@ static struct drm_driver driver = {
403 .mmap = nouveau_ttm_mmap, 402 .mmap = nouveau_ttm_mmap,
404 .poll = drm_poll, 403 .poll = drm_poll,
405 .fasync = drm_fasync, 404 .fasync = drm_fasync,
405 .read = drm_read,
406#if defined(CONFIG_COMPAT) 406#if defined(CONFIG_COMPAT)
407 .compat_ioctl = nouveau_compat_ioctl, 407 .compat_ioctl = nouveau_compat_ioctl,
408#endif 408#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index 1c7db64c03bf..46e32573b3a3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -54,22 +54,37 @@ struct nouveau_fpriv {
54#include "nouveau_drm.h" 54#include "nouveau_drm.h"
55#include "nouveau_reg.h" 55#include "nouveau_reg.h"
56#include "nouveau_bios.h" 56#include "nouveau_bios.h"
57#include "nouveau_util.h"
58
57struct nouveau_grctx; 59struct nouveau_grctx;
60struct nouveau_vram;
61#include "nouveau_vm.h"
58 62
59#define MAX_NUM_DCB_ENTRIES 16 63#define MAX_NUM_DCB_ENTRIES 16
60 64
61#define NOUVEAU_MAX_CHANNEL_NR 128 65#define NOUVEAU_MAX_CHANNEL_NR 128
62#define NOUVEAU_MAX_TILE_NR 15 66#define NOUVEAU_MAX_TILE_NR 15
63 67
64#define NV50_VM_MAX_VRAM (2*1024*1024*1024ULL) 68struct nouveau_vram {
65#define NV50_VM_BLOCK (512*1024*1024ULL) 69 struct drm_device *dev;
66#define NV50_VM_VRAM_NR (NV50_VM_MAX_VRAM / NV50_VM_BLOCK) 70
71 struct nouveau_vma bar_vma;
72 u8 page_shift;
73
74 struct list_head regions;
75 u32 memtype;
76 u64 offset;
77 u64 size;
78};
67 79
68struct nouveau_tile_reg { 80struct nouveau_tile_reg {
69 struct nouveau_fence *fence;
70 uint32_t addr;
71 uint32_t size;
72 bool used; 81 bool used;
82 uint32_t addr;
83 uint32_t limit;
84 uint32_t pitch;
85 uint32_t zcomp;
86 struct drm_mm_node *tag_mem;
87 struct nouveau_fence *fence;
73}; 88};
74 89
75struct nouveau_bo { 90struct nouveau_bo {
@@ -88,6 +103,7 @@ struct nouveau_bo {
88 103
89 struct nouveau_channel *channel; 104 struct nouveau_channel *channel;
90 105
106 struct nouveau_vma vma;
91 bool mappable; 107 bool mappable;
92 bool no_vm; 108 bool no_vm;
93 109
@@ -96,7 +112,6 @@ struct nouveau_bo {
96 struct nouveau_tile_reg *tile; 112 struct nouveau_tile_reg *tile;
97 113
98 struct drm_gem_object *gem; 114 struct drm_gem_object *gem;
99 struct drm_file *cpu_filp;
100 int pin_refcnt; 115 int pin_refcnt;
101}; 116};
102 117
@@ -133,20 +148,28 @@ enum nouveau_flags {
133 148
134#define NVOBJ_ENGINE_SW 0 149#define NVOBJ_ENGINE_SW 0
135#define NVOBJ_ENGINE_GR 1 150#define NVOBJ_ENGINE_GR 1
136#define NVOBJ_ENGINE_DISPLAY 2 151#define NVOBJ_ENGINE_PPP 2
152#define NVOBJ_ENGINE_COPY 3
153#define NVOBJ_ENGINE_VP 4
154#define NVOBJ_ENGINE_CRYPT 5
155#define NVOBJ_ENGINE_BSP 6
156#define NVOBJ_ENGINE_DISPLAY 0xcafe0001
137#define NVOBJ_ENGINE_INT 0xdeadbeef 157#define NVOBJ_ENGINE_INT 0xdeadbeef
138 158
159#define NVOBJ_FLAG_DONT_MAP (1 << 0)
139#define NVOBJ_FLAG_ZERO_ALLOC (1 << 1) 160#define NVOBJ_FLAG_ZERO_ALLOC (1 << 1)
140#define NVOBJ_FLAG_ZERO_FREE (1 << 2) 161#define NVOBJ_FLAG_ZERO_FREE (1 << 2)
162#define NVOBJ_FLAG_VM (1 << 3)
163
164#define NVOBJ_CINST_GLOBAL 0xdeadbeef
165
141struct nouveau_gpuobj { 166struct nouveau_gpuobj {
142 struct drm_device *dev; 167 struct drm_device *dev;
143 struct kref refcount; 168 struct kref refcount;
144 struct list_head list; 169 struct list_head list;
145 170
146 struct drm_mm_node *im_pramin; 171 void *node;
147 struct nouveau_bo *im_backing; 172 u32 *suspend;
148 uint32_t *im_backing_suspend;
149 int im_bound;
150 173
151 uint32_t flags; 174 uint32_t flags;
152 175
@@ -162,10 +185,29 @@ struct nouveau_gpuobj {
162 void *priv; 185 void *priv;
163}; 186};
164 187
188struct nouveau_page_flip_state {
189 struct list_head head;
190 struct drm_pending_vblank_event *event;
191 int crtc, bpp, pitch, x, y;
192 uint64_t offset;
193};
194
195enum nouveau_channel_mutex_class {
196 NOUVEAU_UCHANNEL_MUTEX,
197 NOUVEAU_KCHANNEL_MUTEX
198};
199
165struct nouveau_channel { 200struct nouveau_channel {
166 struct drm_device *dev; 201 struct drm_device *dev;
167 int id; 202 int id;
168 203
204 /* references to the channel data structure */
205 struct kref ref;
206 /* users of the hardware channel resources, the hardware
207 * context will be kicked off when it reaches zero. */
208 atomic_t users;
209 struct mutex mutex;
210
169 /* owner of this fifo */ 211 /* owner of this fifo */
170 struct drm_file *file_priv; 212 struct drm_file *file_priv;
171 /* mapping of the fifo itself */ 213 /* mapping of the fifo itself */
@@ -198,16 +240,17 @@ struct nouveau_channel {
198 /* PFIFO context */ 240 /* PFIFO context */
199 struct nouveau_gpuobj *ramfc; 241 struct nouveau_gpuobj *ramfc;
200 struct nouveau_gpuobj *cache; 242 struct nouveau_gpuobj *cache;
243 void *fifo_priv;
201 244
202 /* PGRAPH context */ 245 /* PGRAPH context */
203 /* XXX may be merge 2 pointers as private data ??? */ 246 /* XXX may be merge 2 pointers as private data ??? */
204 struct nouveau_gpuobj *ramin_grctx; 247 struct nouveau_gpuobj *ramin_grctx;
248 struct nouveau_gpuobj *crypt_ctx;
205 void *pgraph_ctx; 249 void *pgraph_ctx;
206 250
207 /* NV50 VM */ 251 /* NV50 VM */
252 struct nouveau_vm *vm;
208 struct nouveau_gpuobj *vm_pd; 253 struct nouveau_gpuobj *vm_pd;
209 struct nouveau_gpuobj *vm_gart_pt;
210 struct nouveau_gpuobj *vm_vram_pt[NV50_VM_VRAM_NR];
211 254
212 /* Objects */ 255 /* Objects */
213 struct nouveau_gpuobj *ramin; /* Private instmem */ 256 struct nouveau_gpuobj *ramin; /* Private instmem */
@@ -238,9 +281,11 @@ struct nouveau_channel {
238 281
239 struct { 282 struct {
240 struct nouveau_gpuobj *vblsem; 283 struct nouveau_gpuobj *vblsem;
284 uint32_t vblsem_head;
241 uint32_t vblsem_offset; 285 uint32_t vblsem_offset;
242 uint32_t vblsem_rval; 286 uint32_t vblsem_rval;
243 struct list_head vbl_wait; 287 struct list_head vbl_wait;
288 struct list_head flip;
244 } nvsw; 289 } nvsw;
245 290
246 struct { 291 struct {
@@ -258,11 +303,11 @@ struct nouveau_instmem_engine {
258 int (*suspend)(struct drm_device *dev); 303 int (*suspend)(struct drm_device *dev);
259 void (*resume)(struct drm_device *dev); 304 void (*resume)(struct drm_device *dev);
260 305
261 int (*populate)(struct drm_device *, struct nouveau_gpuobj *, 306 int (*get)(struct nouveau_gpuobj *, u32 size, u32 align);
262 uint32_t *size); 307 void (*put)(struct nouveau_gpuobj *);
263 void (*clear)(struct drm_device *, struct nouveau_gpuobj *); 308 int (*map)(struct nouveau_gpuobj *);
264 int (*bind)(struct drm_device *, struct nouveau_gpuobj *); 309 void (*unmap)(struct nouveau_gpuobj *);
265 int (*unbind)(struct drm_device *, struct nouveau_gpuobj *); 310
266 void (*flush)(struct drm_device *); 311 void (*flush)(struct drm_device *);
267}; 312};
268 313
@@ -279,15 +324,21 @@ struct nouveau_timer_engine {
279 324
280struct nouveau_fb_engine { 325struct nouveau_fb_engine {
281 int num_tiles; 326 int num_tiles;
327 struct drm_mm tag_heap;
328 void *priv;
282 329
283 int (*init)(struct drm_device *dev); 330 int (*init)(struct drm_device *dev);
284 void (*takedown)(struct drm_device *dev); 331 void (*takedown)(struct drm_device *dev);
285 332
286 void (*set_region_tiling)(struct drm_device *dev, int i, uint32_t addr, 333 void (*init_tile_region)(struct drm_device *dev, int i,
287 uint32_t size, uint32_t pitch); 334 uint32_t addr, uint32_t size,
335 uint32_t pitch, uint32_t flags);
336 void (*set_tile_region)(struct drm_device *dev, int i);
337 void (*free_tile_region)(struct drm_device *dev, int i);
288}; 338};
289 339
290struct nouveau_fifo_engine { 340struct nouveau_fifo_engine {
341 void *priv;
291 int channels; 342 int channels;
292 343
293 struct nouveau_gpuobj *playlist[2]; 344 struct nouveau_gpuobj *playlist[2];
@@ -310,22 +361,11 @@ struct nouveau_fifo_engine {
310 void (*tlb_flush)(struct drm_device *dev); 361 void (*tlb_flush)(struct drm_device *dev);
311}; 362};
312 363
313struct nouveau_pgraph_object_method {
314 int id;
315 int (*exec)(struct nouveau_channel *chan, int grclass, int mthd,
316 uint32_t data);
317};
318
319struct nouveau_pgraph_object_class {
320 int id;
321 bool software;
322 struct nouveau_pgraph_object_method *methods;
323};
324
325struct nouveau_pgraph_engine { 364struct nouveau_pgraph_engine {
326 struct nouveau_pgraph_object_class *grclass;
327 bool accel_blocked; 365 bool accel_blocked;
366 bool registered;
328 int grctx_size; 367 int grctx_size;
368 void *priv;
329 369
330 /* NV2x/NV3x context table (0x400780) */ 370 /* NV2x/NV3x context table (0x400780) */
331 struct nouveau_gpuobj *ctx_table; 371 struct nouveau_gpuobj *ctx_table;
@@ -342,8 +382,7 @@ struct nouveau_pgraph_engine {
342 int (*unload_context)(struct drm_device *); 382 int (*unload_context)(struct drm_device *);
343 void (*tlb_flush)(struct drm_device *dev); 383 void (*tlb_flush)(struct drm_device *dev);
344 384
345 void (*set_region_tiling)(struct drm_device *dev, int i, uint32_t addr, 385 void (*set_tile_region)(struct drm_device *dev, int i);
346 uint32_t size, uint32_t pitch);
347}; 386};
348 387
349struct nouveau_display_engine { 388struct nouveau_display_engine {
@@ -355,13 +394,19 @@ struct nouveau_display_engine {
355}; 394};
356 395
357struct nouveau_gpio_engine { 396struct nouveau_gpio_engine {
397 void *priv;
398
358 int (*init)(struct drm_device *); 399 int (*init)(struct drm_device *);
359 void (*takedown)(struct drm_device *); 400 void (*takedown)(struct drm_device *);
360 401
361 int (*get)(struct drm_device *, enum dcb_gpio_tag); 402 int (*get)(struct drm_device *, enum dcb_gpio_tag);
362 int (*set)(struct drm_device *, enum dcb_gpio_tag, int state); 403 int (*set)(struct drm_device *, enum dcb_gpio_tag, int state);
363 404
364 void (*irq_enable)(struct drm_device *, enum dcb_gpio_tag, bool on); 405 int (*irq_register)(struct drm_device *, enum dcb_gpio_tag,
406 void (*)(void *, int), void *);
407 void (*irq_unregister)(struct drm_device *, enum dcb_gpio_tag,
408 void (*)(void *, int), void *);
409 bool (*irq_enable)(struct drm_device *, enum dcb_gpio_tag, bool on);
365}; 410};
366 411
367struct nouveau_pm_voltage_level { 412struct nouveau_pm_voltage_level {
@@ -437,6 +482,7 @@ struct nouveau_pm_engine {
437 struct nouveau_pm_level *cur; 482 struct nouveau_pm_level *cur;
438 483
439 struct device *hwmon; 484 struct device *hwmon;
485 struct notifier_block acpi_nb;
440 486
441 int (*clock_get)(struct drm_device *, u32 id); 487 int (*clock_get)(struct drm_device *, u32 id);
442 void *(*clock_pre)(struct drm_device *, struct nouveau_pm_level *, 488 void *(*clock_pre)(struct drm_device *, struct nouveau_pm_level *,
@@ -449,6 +495,25 @@ struct nouveau_pm_engine {
449 int (*temp_get)(struct drm_device *); 495 int (*temp_get)(struct drm_device *);
450}; 496};
451 497
498struct nouveau_crypt_engine {
499 bool registered;
500
501 int (*init)(struct drm_device *);
502 void (*takedown)(struct drm_device *);
503 int (*create_context)(struct nouveau_channel *);
504 void (*destroy_context)(struct nouveau_channel *);
505 void (*tlb_flush)(struct drm_device *dev);
506};
507
508struct nouveau_vram_engine {
509 int (*init)(struct drm_device *);
510 int (*get)(struct drm_device *, u64, u32 align, u32 size_nc,
511 u32 type, struct nouveau_vram **);
512 void (*put)(struct drm_device *, struct nouveau_vram **);
513
514 bool (*flags_valid)(struct drm_device *, u32 tile_flags);
515};
516
452struct nouveau_engine { 517struct nouveau_engine {
453 struct nouveau_instmem_engine instmem; 518 struct nouveau_instmem_engine instmem;
454 struct nouveau_mc_engine mc; 519 struct nouveau_mc_engine mc;
@@ -459,6 +524,8 @@ struct nouveau_engine {
459 struct nouveau_display_engine display; 524 struct nouveau_display_engine display;
460 struct nouveau_gpio_engine gpio; 525 struct nouveau_gpio_engine gpio;
461 struct nouveau_pm_engine pm; 526 struct nouveau_pm_engine pm;
527 struct nouveau_crypt_engine crypt;
528 struct nouveau_vram_engine vram;
462}; 529};
463 530
464struct nouveau_pll_vals { 531struct nouveau_pll_vals {
@@ -577,18 +644,15 @@ struct drm_nouveau_private {
577 bool ramin_available; 644 bool ramin_available;
578 struct drm_mm ramin_heap; 645 struct drm_mm ramin_heap;
579 struct list_head gpuobj_list; 646 struct list_head gpuobj_list;
647 struct list_head classes;
580 648
581 struct nouveau_bo *vga_ram; 649 struct nouveau_bo *vga_ram;
582 650
651 /* interrupt handling */
652 void (*irq_handler[32])(struct drm_device *);
653 bool msi_enabled;
583 struct workqueue_struct *wq; 654 struct workqueue_struct *wq;
584 struct work_struct irq_work; 655 struct work_struct irq_work;
585 struct work_struct hpd_work;
586
587 struct {
588 spinlock_t lock;
589 uint32_t hpd0_bits;
590 uint32_t hpd1_bits;
591 } hpd_state;
592 656
593 struct list_head vbl_waiting; 657 struct list_head vbl_waiting;
594 658
@@ -605,8 +669,10 @@ struct drm_nouveau_private {
605 struct nouveau_bo *bo; 669 struct nouveau_bo *bo;
606 } fence; 670 } fence;
607 671
608 int fifo_alloc_count; 672 struct {
609 struct nouveau_channel *fifos[NOUVEAU_MAX_CHANNEL_NR]; 673 spinlock_t lock;
674 struct nouveau_channel *ptr[NOUVEAU_MAX_CHANNEL_NR];
675 } channels;
610 676
611 struct nouveau_engine engine; 677 struct nouveau_engine engine;
612 struct nouveau_channel *channel; 678 struct nouveau_channel *channel;
@@ -632,12 +698,14 @@ struct drm_nouveau_private {
632 uint64_t aper_free; 698 uint64_t aper_free;
633 699
634 struct nouveau_gpuobj *sg_ctxdma; 700 struct nouveau_gpuobj *sg_ctxdma;
635 struct page *sg_dummy_page; 701 struct nouveau_vma vma;
636 dma_addr_t sg_dummy_bus;
637 } gart_info; 702 } gart_info;
638 703
639 /* nv10-nv40 tiling regions */ 704 /* nv10-nv40 tiling regions */
640 struct nouveau_tile_reg tile[NOUVEAU_MAX_TILE_NR]; 705 struct {
706 struct nouveau_tile_reg reg[NOUVEAU_MAX_TILE_NR];
707 spinlock_t lock;
708 } tile;
641 709
642 /* VRAM/fb configuration */ 710 /* VRAM/fb configuration */
643 uint64_t vram_size; 711 uint64_t vram_size;
@@ -650,14 +718,12 @@ struct drm_nouveau_private {
650 uint64_t fb_aper_free; 718 uint64_t fb_aper_free;
651 int fb_mtrr; 719 int fb_mtrr;
652 720
721 /* BAR control (NV50-) */
722 struct nouveau_vm *bar1_vm;
723 struct nouveau_vm *bar3_vm;
724
653 /* G8x/G9x virtual address space */ 725 /* G8x/G9x virtual address space */
654 uint64_t vm_gart_base; 726 struct nouveau_vm *chan_vm;
655 uint64_t vm_gart_size;
656 uint64_t vm_vram_base;
657 uint64_t vm_vram_size;
658 uint64_t vm_end;
659 struct nouveau_gpuobj *vm_vram_pt[NV50_VM_VRAM_NR];
660 int vm_vram_pt_nr;
661 727
662 struct nvbios vbios; 728 struct nvbios vbios;
663 729
@@ -674,6 +740,7 @@ struct drm_nouveau_private {
674 struct backlight_device *backlight; 740 struct backlight_device *backlight;
675 741
676 struct nouveau_channel *evo; 742 struct nouveau_channel *evo;
743 u32 evo_alloc;
677 struct { 744 struct {
678 struct dcb_entry *dcb; 745 struct dcb_entry *dcb;
679 u16 script; 746 u16 script;
@@ -686,6 +753,8 @@ struct drm_nouveau_private {
686 753
687 struct nouveau_fbdev *nfbdev; 754 struct nouveau_fbdev *nfbdev;
688 struct apertures_struct *apertures; 755 struct apertures_struct *apertures;
756
757 bool powered_down;
689}; 758};
690 759
691static inline struct drm_nouveau_private * 760static inline struct drm_nouveau_private *
@@ -719,16 +788,6 @@ nouveau_bo_ref(struct nouveau_bo *ref, struct nouveau_bo **pnvbo)
719 return 0; 788 return 0;
720} 789}
721 790
722#define NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(id, cl, ch) do { \
723 struct drm_nouveau_private *nv = dev->dev_private; \
724 if (!nouveau_channel_owner(dev, (cl), (id))) { \
725 NV_ERROR(dev, "pid %d doesn't own channel %d\n", \
726 DRM_CURRENTPID, (id)); \
727 return -EPERM; \
728 } \
729 (ch) = nv->fifos[(id)]; \
730} while (0)
731
732/* nouveau_drv.c */ 791/* nouveau_drv.c */
733extern int nouveau_agpmode; 792extern int nouveau_agpmode;
734extern int nouveau_duallink; 793extern int nouveau_duallink;
@@ -748,6 +807,7 @@ extern int nouveau_force_post;
748extern int nouveau_override_conntype; 807extern int nouveau_override_conntype;
749extern char *nouveau_perflvl; 808extern char *nouveau_perflvl;
750extern int nouveau_perflvl_wr; 809extern int nouveau_perflvl_wr;
810extern int nouveau_msi;
751 811
752extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state); 812extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state);
753extern int nouveau_pci_resume(struct pci_dev *pdev); 813extern int nouveau_pci_resume(struct pci_dev *pdev);
@@ -762,8 +822,10 @@ extern int nouveau_ioctl_getparam(struct drm_device *, void *data,
762 struct drm_file *); 822 struct drm_file *);
763extern int nouveau_ioctl_setparam(struct drm_device *, void *data, 823extern int nouveau_ioctl_setparam(struct drm_device *, void *data,
764 struct drm_file *); 824 struct drm_file *);
765extern bool nouveau_wait_until(struct drm_device *, uint64_t timeout, 825extern bool nouveau_wait_eq(struct drm_device *, uint64_t timeout,
766 uint32_t reg, uint32_t mask, uint32_t val); 826 uint32_t reg, uint32_t mask, uint32_t val);
827extern bool nouveau_wait_ne(struct drm_device *, uint64_t timeout,
828 uint32_t reg, uint32_t mask, uint32_t val);
767extern bool nouveau_wait_for_idle(struct drm_device *); 829extern bool nouveau_wait_for_idle(struct drm_device *);
768extern int nouveau_card_init(struct drm_device *); 830extern int nouveau_card_init(struct drm_device *);
769 831
@@ -775,18 +837,18 @@ extern void nouveau_mem_gart_fini(struct drm_device *);
775extern int nouveau_mem_init_agp(struct drm_device *); 837extern int nouveau_mem_init_agp(struct drm_device *);
776extern int nouveau_mem_reset_agp(struct drm_device *); 838extern int nouveau_mem_reset_agp(struct drm_device *);
777extern void nouveau_mem_close(struct drm_device *); 839extern void nouveau_mem_close(struct drm_device *);
778extern struct nouveau_tile_reg *nv10_mem_set_tiling(struct drm_device *dev, 840extern int nouveau_mem_detect(struct drm_device *);
779 uint32_t addr, 841extern bool nouveau_mem_flags_valid(struct drm_device *, u32 tile_flags);
780 uint32_t size, 842extern struct nouveau_tile_reg *nv10_mem_set_tiling(
781 uint32_t pitch); 843 struct drm_device *dev, uint32_t addr, uint32_t size,
782extern void nv10_mem_expire_tiling(struct drm_device *dev, 844 uint32_t pitch, uint32_t flags);
783 struct nouveau_tile_reg *tile, 845extern void nv10_mem_put_tile_region(struct drm_device *dev,
784 struct nouveau_fence *fence); 846 struct nouveau_tile_reg *tile,
785extern int nv50_mem_vm_bind_linear(struct drm_device *, uint64_t virt, 847 struct nouveau_fence *fence);
786 uint32_t size, uint32_t flags, 848extern const struct ttm_mem_type_manager_func nouveau_vram_manager;
787 uint64_t phys); 849
788extern void nv50_mem_vm_unbind(struct drm_device *, uint64_t virt, 850/* nvc0_vram.c */
789 uint32_t size); 851extern const struct ttm_mem_type_manager_func nvc0_vram_manager;
790 852
791/* nouveau_notifier.c */ 853/* nouveau_notifier.c */
792extern int nouveau_notifier_init_channel(struct nouveau_channel *); 854extern int nouveau_notifier_init_channel(struct nouveau_channel *);
@@ -803,21 +865,44 @@ extern int nouveau_ioctl_notifier_free(struct drm_device *, void *data,
803extern struct drm_ioctl_desc nouveau_ioctls[]; 865extern struct drm_ioctl_desc nouveau_ioctls[];
804extern int nouveau_max_ioctl; 866extern int nouveau_max_ioctl;
805extern void nouveau_channel_cleanup(struct drm_device *, struct drm_file *); 867extern void nouveau_channel_cleanup(struct drm_device *, struct drm_file *);
806extern int nouveau_channel_owner(struct drm_device *, struct drm_file *,
807 int channel);
808extern int nouveau_channel_alloc(struct drm_device *dev, 868extern int nouveau_channel_alloc(struct drm_device *dev,
809 struct nouveau_channel **chan, 869 struct nouveau_channel **chan,
810 struct drm_file *file_priv, 870 struct drm_file *file_priv,
811 uint32_t fb_ctxdma, uint32_t tt_ctxdma); 871 uint32_t fb_ctxdma, uint32_t tt_ctxdma);
812extern void nouveau_channel_free(struct nouveau_channel *); 872extern struct nouveau_channel *
873nouveau_channel_get_unlocked(struct nouveau_channel *);
874extern struct nouveau_channel *
875nouveau_channel_get(struct drm_device *, struct drm_file *, int id);
876extern void nouveau_channel_put_unlocked(struct nouveau_channel **);
877extern void nouveau_channel_put(struct nouveau_channel **);
878extern void nouveau_channel_ref(struct nouveau_channel *chan,
879 struct nouveau_channel **pchan);
880extern void nouveau_channel_idle(struct nouveau_channel *chan);
813 881
814/* nouveau_object.c */ 882/* nouveau_object.c */
883#define NVOBJ_CLASS(d,c,e) do { \
884 int ret = nouveau_gpuobj_class_new((d), (c), NVOBJ_ENGINE_##e); \
885 if (ret) \
886 return ret; \
887} while(0)
888
889#define NVOBJ_MTHD(d,c,m,e) do { \
890 int ret = nouveau_gpuobj_mthd_new((d), (c), (m), (e)); \
891 if (ret) \
892 return ret; \
893} while(0)
894
815extern int nouveau_gpuobj_early_init(struct drm_device *); 895extern int nouveau_gpuobj_early_init(struct drm_device *);
816extern int nouveau_gpuobj_init(struct drm_device *); 896extern int nouveau_gpuobj_init(struct drm_device *);
817extern void nouveau_gpuobj_takedown(struct drm_device *); 897extern void nouveau_gpuobj_takedown(struct drm_device *);
818extern int nouveau_gpuobj_suspend(struct drm_device *dev); 898extern int nouveau_gpuobj_suspend(struct drm_device *dev);
819extern void nouveau_gpuobj_suspend_cleanup(struct drm_device *dev);
820extern void nouveau_gpuobj_resume(struct drm_device *dev); 899extern void nouveau_gpuobj_resume(struct drm_device *dev);
900extern int nouveau_gpuobj_class_new(struct drm_device *, u32 class, u32 eng);
901extern int nouveau_gpuobj_mthd_new(struct drm_device *, u32 class, u32 mthd,
902 int (*exec)(struct nouveau_channel *,
903 u32 class, u32 mthd, u32 data));
904extern int nouveau_gpuobj_mthd_call(struct nouveau_channel *, u32, u32, u32);
905extern int nouveau_gpuobj_mthd_call2(struct drm_device *, int, u32, u32, u32);
821extern int nouveau_gpuobj_channel_init(struct nouveau_channel *, 906extern int nouveau_gpuobj_channel_init(struct nouveau_channel *,
822 uint32_t vram_h, uint32_t tt_h); 907 uint32_t vram_h, uint32_t tt_h);
823extern void nouveau_gpuobj_channel_takedown(struct nouveau_channel *); 908extern void nouveau_gpuobj_channel_takedown(struct nouveau_channel *);
@@ -832,21 +917,25 @@ extern int nouveau_gpuobj_new_fake(struct drm_device *, u32 pinst, u64 vinst,
832extern int nouveau_gpuobj_dma_new(struct nouveau_channel *, int class, 917extern int nouveau_gpuobj_dma_new(struct nouveau_channel *, int class,
833 uint64_t offset, uint64_t size, int access, 918 uint64_t offset, uint64_t size, int access,
834 int target, struct nouveau_gpuobj **); 919 int target, struct nouveau_gpuobj **);
835extern int nouveau_gpuobj_gart_dma_new(struct nouveau_channel *, 920extern int nouveau_gpuobj_gr_new(struct nouveau_channel *, u32 handle, int class);
836 uint64_t offset, uint64_t size, 921extern int nv50_gpuobj_dma_new(struct nouveau_channel *, int class, u64 base,
837 int access, struct nouveau_gpuobj **, 922 u64 size, int target, int access, u32 type,
838 uint32_t *o_ret); 923 u32 comp, struct nouveau_gpuobj **pobj);
839extern int nouveau_gpuobj_gr_new(struct nouveau_channel *, int class, 924extern void nv50_gpuobj_dma_init(struct nouveau_gpuobj *, u32 offset,
840 struct nouveau_gpuobj **); 925 int class, u64 base, u64 size, int target,
841extern int nouveau_gpuobj_sw_new(struct nouveau_channel *, int class, 926 int access, u32 type, u32 comp);
842 struct nouveau_gpuobj **);
843extern int nouveau_ioctl_grobj_alloc(struct drm_device *, void *data, 927extern int nouveau_ioctl_grobj_alloc(struct drm_device *, void *data,
844 struct drm_file *); 928 struct drm_file *);
845extern int nouveau_ioctl_gpuobj_free(struct drm_device *, void *data, 929extern int nouveau_ioctl_gpuobj_free(struct drm_device *, void *data,
846 struct drm_file *); 930 struct drm_file *);
847 931
848/* nouveau_irq.c */ 932/* nouveau_irq.c */
933extern int nouveau_irq_init(struct drm_device *);
934extern void nouveau_irq_fini(struct drm_device *);
849extern irqreturn_t nouveau_irq_handler(DRM_IRQ_ARGS); 935extern irqreturn_t nouveau_irq_handler(DRM_IRQ_ARGS);
936extern void nouveau_irq_register(struct drm_device *, int status_bit,
937 void (*)(struct drm_device *));
938extern void nouveau_irq_unregister(struct drm_device *, int status_bit);
850extern void nouveau_irq_preinstall(struct drm_device *); 939extern void nouveau_irq_preinstall(struct drm_device *);
851extern int nouveau_irq_postinstall(struct drm_device *); 940extern int nouveau_irq_postinstall(struct drm_device *);
852extern void nouveau_irq_uninstall(struct drm_device *); 941extern void nouveau_irq_uninstall(struct drm_device *);
@@ -854,8 +943,8 @@ extern void nouveau_irq_uninstall(struct drm_device *);
854/* nouveau_sgdma.c */ 943/* nouveau_sgdma.c */
855extern int nouveau_sgdma_init(struct drm_device *); 944extern int nouveau_sgdma_init(struct drm_device *);
856extern void nouveau_sgdma_takedown(struct drm_device *); 945extern void nouveau_sgdma_takedown(struct drm_device *);
857extern int nouveau_sgdma_get_page(struct drm_device *, uint32_t offset, 946extern uint32_t nouveau_sgdma_get_physical(struct drm_device *,
858 uint32_t *page); 947 uint32_t offset);
859extern struct ttm_backend *nouveau_sgdma_init_ttm(struct drm_device *); 948extern struct ttm_backend *nouveau_sgdma_init_ttm(struct drm_device *);
860 949
861/* nouveau_debugfs.c */ 950/* nouveau_debugfs.c */
@@ -966,18 +1055,25 @@ extern void nv04_fb_takedown(struct drm_device *);
966/* nv10_fb.c */ 1055/* nv10_fb.c */
967extern int nv10_fb_init(struct drm_device *); 1056extern int nv10_fb_init(struct drm_device *);
968extern void nv10_fb_takedown(struct drm_device *); 1057extern void nv10_fb_takedown(struct drm_device *);
969extern void nv10_fb_set_region_tiling(struct drm_device *, int, uint32_t, 1058extern void nv10_fb_init_tile_region(struct drm_device *dev, int i,
970 uint32_t, uint32_t); 1059 uint32_t addr, uint32_t size,
1060 uint32_t pitch, uint32_t flags);
1061extern void nv10_fb_set_tile_region(struct drm_device *dev, int i);
1062extern void nv10_fb_free_tile_region(struct drm_device *dev, int i);
971 1063
972/* nv30_fb.c */ 1064/* nv30_fb.c */
973extern int nv30_fb_init(struct drm_device *); 1065extern int nv30_fb_init(struct drm_device *);
974extern void nv30_fb_takedown(struct drm_device *); 1066extern void nv30_fb_takedown(struct drm_device *);
1067extern void nv30_fb_init_tile_region(struct drm_device *dev, int i,
1068 uint32_t addr, uint32_t size,
1069 uint32_t pitch, uint32_t flags);
1070extern void nv30_fb_free_tile_region(struct drm_device *dev, int i);
975 1071
976/* nv40_fb.c */ 1072/* nv40_fb.c */
977extern int nv40_fb_init(struct drm_device *); 1073extern int nv40_fb_init(struct drm_device *);
978extern void nv40_fb_takedown(struct drm_device *); 1074extern void nv40_fb_takedown(struct drm_device *);
979extern void nv40_fb_set_region_tiling(struct drm_device *, int, uint32_t, 1075extern void nv40_fb_set_tile_region(struct drm_device *dev, int i);
980 uint32_t, uint32_t); 1076
981/* nv50_fb.c */ 1077/* nv50_fb.c */
982extern int nv50_fb_init(struct drm_device *); 1078extern int nv50_fb_init(struct drm_device *);
983extern void nv50_fb_takedown(struct drm_device *); 1079extern void nv50_fb_takedown(struct drm_device *);
@@ -989,6 +1085,7 @@ extern void nvc0_fb_takedown(struct drm_device *);
989 1085
990/* nv04_fifo.c */ 1086/* nv04_fifo.c */
991extern int nv04_fifo_init(struct drm_device *); 1087extern int nv04_fifo_init(struct drm_device *);
1088extern void nv04_fifo_fini(struct drm_device *);
992extern void nv04_fifo_disable(struct drm_device *); 1089extern void nv04_fifo_disable(struct drm_device *);
993extern void nv04_fifo_enable(struct drm_device *); 1090extern void nv04_fifo_enable(struct drm_device *);
994extern bool nv04_fifo_reassign(struct drm_device *, bool); 1091extern bool nv04_fifo_reassign(struct drm_device *, bool);
@@ -998,19 +1095,18 @@ extern int nv04_fifo_create_context(struct nouveau_channel *);
998extern void nv04_fifo_destroy_context(struct nouveau_channel *); 1095extern void nv04_fifo_destroy_context(struct nouveau_channel *);
999extern int nv04_fifo_load_context(struct nouveau_channel *); 1096extern int nv04_fifo_load_context(struct nouveau_channel *);
1000extern int nv04_fifo_unload_context(struct drm_device *); 1097extern int nv04_fifo_unload_context(struct drm_device *);
1098extern void nv04_fifo_isr(struct drm_device *);
1001 1099
1002/* nv10_fifo.c */ 1100/* nv10_fifo.c */
1003extern int nv10_fifo_init(struct drm_device *); 1101extern int nv10_fifo_init(struct drm_device *);
1004extern int nv10_fifo_channel_id(struct drm_device *); 1102extern int nv10_fifo_channel_id(struct drm_device *);
1005extern int nv10_fifo_create_context(struct nouveau_channel *); 1103extern int nv10_fifo_create_context(struct nouveau_channel *);
1006extern void nv10_fifo_destroy_context(struct nouveau_channel *);
1007extern int nv10_fifo_load_context(struct nouveau_channel *); 1104extern int nv10_fifo_load_context(struct nouveau_channel *);
1008extern int nv10_fifo_unload_context(struct drm_device *); 1105extern int nv10_fifo_unload_context(struct drm_device *);
1009 1106
1010/* nv40_fifo.c */ 1107/* nv40_fifo.c */
1011extern int nv40_fifo_init(struct drm_device *); 1108extern int nv40_fifo_init(struct drm_device *);
1012extern int nv40_fifo_create_context(struct nouveau_channel *); 1109extern int nv40_fifo_create_context(struct nouveau_channel *);
1013extern void nv40_fifo_destroy_context(struct nouveau_channel *);
1014extern int nv40_fifo_load_context(struct nouveau_channel *); 1110extern int nv40_fifo_load_context(struct nouveau_channel *);
1015extern int nv40_fifo_unload_context(struct drm_device *); 1111extern int nv40_fifo_unload_context(struct drm_device *);
1016 1112
@@ -1038,7 +1134,6 @@ extern int nvc0_fifo_load_context(struct nouveau_channel *);
1038extern int nvc0_fifo_unload_context(struct drm_device *); 1134extern int nvc0_fifo_unload_context(struct drm_device *);
1039 1135
1040/* nv04_graph.c */ 1136/* nv04_graph.c */
1041extern struct nouveau_pgraph_object_class nv04_graph_grclass[];
1042extern int nv04_graph_init(struct drm_device *); 1137extern int nv04_graph_init(struct drm_device *);
1043extern void nv04_graph_takedown(struct drm_device *); 1138extern void nv04_graph_takedown(struct drm_device *);
1044extern void nv04_graph_fifo_access(struct drm_device *, bool); 1139extern void nv04_graph_fifo_access(struct drm_device *, bool);
@@ -1047,10 +1142,11 @@ extern int nv04_graph_create_context(struct nouveau_channel *);
1047extern void nv04_graph_destroy_context(struct nouveau_channel *); 1142extern void nv04_graph_destroy_context(struct nouveau_channel *);
1048extern int nv04_graph_load_context(struct nouveau_channel *); 1143extern int nv04_graph_load_context(struct nouveau_channel *);
1049extern int nv04_graph_unload_context(struct drm_device *); 1144extern int nv04_graph_unload_context(struct drm_device *);
1050extern void nv04_graph_context_switch(struct drm_device *); 1145extern int nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
1146 u32 class, u32 mthd, u32 data);
1147extern struct nouveau_bitfield nv04_graph_nsource[];
1051 1148
1052/* nv10_graph.c */ 1149/* nv10_graph.c */
1053extern struct nouveau_pgraph_object_class nv10_graph_grclass[];
1054extern int nv10_graph_init(struct drm_device *); 1150extern int nv10_graph_init(struct drm_device *);
1055extern void nv10_graph_takedown(struct drm_device *); 1151extern void nv10_graph_takedown(struct drm_device *);
1056extern struct nouveau_channel *nv10_graph_channel(struct drm_device *); 1152extern struct nouveau_channel *nv10_graph_channel(struct drm_device *);
@@ -1058,13 +1154,11 @@ extern int nv10_graph_create_context(struct nouveau_channel *);
1058extern void nv10_graph_destroy_context(struct nouveau_channel *); 1154extern void nv10_graph_destroy_context(struct nouveau_channel *);
1059extern int nv10_graph_load_context(struct nouveau_channel *); 1155extern int nv10_graph_load_context(struct nouveau_channel *);
1060extern int nv10_graph_unload_context(struct drm_device *); 1156extern int nv10_graph_unload_context(struct drm_device *);
1061extern void nv10_graph_context_switch(struct drm_device *); 1157extern void nv10_graph_set_tile_region(struct drm_device *dev, int i);
1062extern void nv10_graph_set_region_tiling(struct drm_device *, int, uint32_t, 1158extern struct nouveau_bitfield nv10_graph_intr[];
1063 uint32_t, uint32_t); 1159extern struct nouveau_bitfield nv10_graph_nstatus[];
1064 1160
1065/* nv20_graph.c */ 1161/* nv20_graph.c */
1066extern struct nouveau_pgraph_object_class nv20_graph_grclass[];
1067extern struct nouveau_pgraph_object_class nv30_graph_grclass[];
1068extern int nv20_graph_create_context(struct nouveau_channel *); 1162extern int nv20_graph_create_context(struct nouveau_channel *);
1069extern void nv20_graph_destroy_context(struct nouveau_channel *); 1163extern void nv20_graph_destroy_context(struct nouveau_channel *);
1070extern int nv20_graph_load_context(struct nouveau_channel *); 1164extern int nv20_graph_load_context(struct nouveau_channel *);
@@ -1072,11 +1166,9 @@ extern int nv20_graph_unload_context(struct drm_device *);
1072extern int nv20_graph_init(struct drm_device *); 1166extern int nv20_graph_init(struct drm_device *);
1073extern void nv20_graph_takedown(struct drm_device *); 1167extern void nv20_graph_takedown(struct drm_device *);
1074extern int nv30_graph_init(struct drm_device *); 1168extern int nv30_graph_init(struct drm_device *);
1075extern void nv20_graph_set_region_tiling(struct drm_device *, int, uint32_t, 1169extern void nv20_graph_set_tile_region(struct drm_device *dev, int i);
1076 uint32_t, uint32_t);
1077 1170
1078/* nv40_graph.c */ 1171/* nv40_graph.c */
1079extern struct nouveau_pgraph_object_class nv40_graph_grclass[];
1080extern int nv40_graph_init(struct drm_device *); 1172extern int nv40_graph_init(struct drm_device *);
1081extern void nv40_graph_takedown(struct drm_device *); 1173extern void nv40_graph_takedown(struct drm_device *);
1082extern struct nouveau_channel *nv40_graph_channel(struct drm_device *); 1174extern struct nouveau_channel *nv40_graph_channel(struct drm_device *);
@@ -1085,11 +1177,9 @@ extern void nv40_graph_destroy_context(struct nouveau_channel *);
1085extern int nv40_graph_load_context(struct nouveau_channel *); 1177extern int nv40_graph_load_context(struct nouveau_channel *);
1086extern int nv40_graph_unload_context(struct drm_device *); 1178extern int nv40_graph_unload_context(struct drm_device *);
1087extern void nv40_grctx_init(struct nouveau_grctx *); 1179extern void nv40_grctx_init(struct nouveau_grctx *);
1088extern void nv40_graph_set_region_tiling(struct drm_device *, int, uint32_t, 1180extern void nv40_graph_set_tile_region(struct drm_device *dev, int i);
1089 uint32_t, uint32_t);
1090 1181
1091/* nv50_graph.c */ 1182/* nv50_graph.c */
1092extern struct nouveau_pgraph_object_class nv50_graph_grclass[];
1093extern int nv50_graph_init(struct drm_device *); 1183extern int nv50_graph_init(struct drm_device *);
1094extern void nv50_graph_takedown(struct drm_device *); 1184extern void nv50_graph_takedown(struct drm_device *);
1095extern void nv50_graph_fifo_access(struct drm_device *, bool); 1185extern void nv50_graph_fifo_access(struct drm_device *, bool);
@@ -1098,10 +1188,10 @@ extern int nv50_graph_create_context(struct nouveau_channel *);
1098extern void nv50_graph_destroy_context(struct nouveau_channel *); 1188extern void nv50_graph_destroy_context(struct nouveau_channel *);
1099extern int nv50_graph_load_context(struct nouveau_channel *); 1189extern int nv50_graph_load_context(struct nouveau_channel *);
1100extern int nv50_graph_unload_context(struct drm_device *); 1190extern int nv50_graph_unload_context(struct drm_device *);
1101extern void nv50_graph_context_switch(struct drm_device *);
1102extern int nv50_grctx_init(struct nouveau_grctx *); 1191extern int nv50_grctx_init(struct nouveau_grctx *);
1103extern void nv50_graph_tlb_flush(struct drm_device *dev); 1192extern void nv50_graph_tlb_flush(struct drm_device *dev);
1104extern void nv86_graph_tlb_flush(struct drm_device *dev); 1193extern void nv86_graph_tlb_flush(struct drm_device *dev);
1194extern struct nouveau_enum nv50_data_error_names[];
1105 1195
1106/* nvc0_graph.c */ 1196/* nvc0_graph.c */
1107extern int nvc0_graph_init(struct drm_device *); 1197extern int nvc0_graph_init(struct drm_device *);
@@ -1113,16 +1203,22 @@ extern void nvc0_graph_destroy_context(struct nouveau_channel *);
1113extern int nvc0_graph_load_context(struct nouveau_channel *); 1203extern int nvc0_graph_load_context(struct nouveau_channel *);
1114extern int nvc0_graph_unload_context(struct drm_device *); 1204extern int nvc0_graph_unload_context(struct drm_device *);
1115 1205
1206/* nv84_crypt.c */
1207extern int nv84_crypt_init(struct drm_device *dev);
1208extern void nv84_crypt_fini(struct drm_device *dev);
1209extern int nv84_crypt_create_context(struct nouveau_channel *);
1210extern void nv84_crypt_destroy_context(struct nouveau_channel *);
1211extern void nv84_crypt_tlb_flush(struct drm_device *dev);
1212
1116/* nv04_instmem.c */ 1213/* nv04_instmem.c */
1117extern int nv04_instmem_init(struct drm_device *); 1214extern int nv04_instmem_init(struct drm_device *);
1118extern void nv04_instmem_takedown(struct drm_device *); 1215extern void nv04_instmem_takedown(struct drm_device *);
1119extern int nv04_instmem_suspend(struct drm_device *); 1216extern int nv04_instmem_suspend(struct drm_device *);
1120extern void nv04_instmem_resume(struct drm_device *); 1217extern void nv04_instmem_resume(struct drm_device *);
1121extern int nv04_instmem_populate(struct drm_device *, struct nouveau_gpuobj *, 1218extern int nv04_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
1122 uint32_t *size); 1219extern void nv04_instmem_put(struct nouveau_gpuobj *);
1123extern void nv04_instmem_clear(struct drm_device *, struct nouveau_gpuobj *); 1220extern int nv04_instmem_map(struct nouveau_gpuobj *);
1124extern int nv04_instmem_bind(struct drm_device *, struct nouveau_gpuobj *); 1221extern void nv04_instmem_unmap(struct nouveau_gpuobj *);
1125extern int nv04_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
1126extern void nv04_instmem_flush(struct drm_device *); 1222extern void nv04_instmem_flush(struct drm_device *);
1127 1223
1128/* nv50_instmem.c */ 1224/* nv50_instmem.c */
@@ -1130,26 +1226,18 @@ extern int nv50_instmem_init(struct drm_device *);
1130extern void nv50_instmem_takedown(struct drm_device *); 1226extern void nv50_instmem_takedown(struct drm_device *);
1131extern int nv50_instmem_suspend(struct drm_device *); 1227extern int nv50_instmem_suspend(struct drm_device *);
1132extern void nv50_instmem_resume(struct drm_device *); 1228extern void nv50_instmem_resume(struct drm_device *);
1133extern int nv50_instmem_populate(struct drm_device *, struct nouveau_gpuobj *, 1229extern int nv50_instmem_get(struct nouveau_gpuobj *, u32 size, u32 align);
1134 uint32_t *size); 1230extern void nv50_instmem_put(struct nouveau_gpuobj *);
1135extern void nv50_instmem_clear(struct drm_device *, struct nouveau_gpuobj *); 1231extern int nv50_instmem_map(struct nouveau_gpuobj *);
1136extern int nv50_instmem_bind(struct drm_device *, struct nouveau_gpuobj *); 1232extern void nv50_instmem_unmap(struct nouveau_gpuobj *);
1137extern int nv50_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
1138extern void nv50_instmem_flush(struct drm_device *); 1233extern void nv50_instmem_flush(struct drm_device *);
1139extern void nv84_instmem_flush(struct drm_device *); 1234extern void nv84_instmem_flush(struct drm_device *);
1140extern void nv50_vm_flush(struct drm_device *, int engine);
1141 1235
1142/* nvc0_instmem.c */ 1236/* nvc0_instmem.c */
1143extern int nvc0_instmem_init(struct drm_device *); 1237extern int nvc0_instmem_init(struct drm_device *);
1144extern void nvc0_instmem_takedown(struct drm_device *); 1238extern void nvc0_instmem_takedown(struct drm_device *);
1145extern int nvc0_instmem_suspend(struct drm_device *); 1239extern int nvc0_instmem_suspend(struct drm_device *);
1146extern void nvc0_instmem_resume(struct drm_device *); 1240extern void nvc0_instmem_resume(struct drm_device *);
1147extern int nvc0_instmem_populate(struct drm_device *, struct nouveau_gpuobj *,
1148 uint32_t *size);
1149extern void nvc0_instmem_clear(struct drm_device *, struct nouveau_gpuobj *);
1150extern int nvc0_instmem_bind(struct drm_device *, struct nouveau_gpuobj *);
1151extern int nvc0_instmem_unbind(struct drm_device *, struct nouveau_gpuobj *);
1152extern void nvc0_instmem_flush(struct drm_device *);
1153 1241
1154/* nv04_mc.c */ 1242/* nv04_mc.c */
1155extern int nv04_mc_init(struct drm_device *); 1243extern int nv04_mc_init(struct drm_device *);
@@ -1219,6 +1307,9 @@ extern u16 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index);
1219extern void nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val); 1307extern void nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val);
1220extern u32 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index); 1308extern u32 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index);
1221extern void nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val); 1309extern void nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val);
1310extern void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
1311extern int nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
1312 bool no_wait_reserve, bool no_wait_gpu);
1222 1313
1223/* nouveau_fence.c */ 1314/* nouveau_fence.c */
1224struct nouveau_fence; 1315struct nouveau_fence;
@@ -1234,12 +1325,35 @@ extern void nouveau_fence_work(struct nouveau_fence *fence,
1234 void (*work)(void *priv, bool signalled), 1325 void (*work)(void *priv, bool signalled),
1235 void *priv); 1326 void *priv);
1236struct nouveau_channel *nouveau_fence_channel(struct nouveau_fence *); 1327struct nouveau_channel *nouveau_fence_channel(struct nouveau_fence *);
1237extern bool nouveau_fence_signalled(void *obj, void *arg); 1328
1238extern int nouveau_fence_wait(void *obj, void *arg, bool lazy, bool intr); 1329extern bool __nouveau_fence_signalled(void *obj, void *arg);
1330extern int __nouveau_fence_wait(void *obj, void *arg, bool lazy, bool intr);
1331extern int __nouveau_fence_flush(void *obj, void *arg);
1332extern void __nouveau_fence_unref(void **obj);
1333extern void *__nouveau_fence_ref(void *obj);
1334
1335static inline bool nouveau_fence_signalled(struct nouveau_fence *obj)
1336{
1337 return __nouveau_fence_signalled(obj, NULL);
1338}
1339static inline int
1340nouveau_fence_wait(struct nouveau_fence *obj, bool lazy, bool intr)
1341{
1342 return __nouveau_fence_wait(obj, NULL, lazy, intr);
1343}
1239extern int nouveau_fence_sync(struct nouveau_fence *, struct nouveau_channel *); 1344extern int nouveau_fence_sync(struct nouveau_fence *, struct nouveau_channel *);
1240extern int nouveau_fence_flush(void *obj, void *arg); 1345static inline int nouveau_fence_flush(struct nouveau_fence *obj)
1241extern void nouveau_fence_unref(void **obj); 1346{
1242extern void *nouveau_fence_ref(void *obj); 1347 return __nouveau_fence_flush(obj, NULL);
1348}
1349static inline void nouveau_fence_unref(struct nouveau_fence **obj)
1350{
1351 __nouveau_fence_unref((void **)obj);
1352}
1353static inline struct nouveau_fence *nouveau_fence_ref(struct nouveau_fence *obj)
1354{
1355 return __nouveau_fence_ref(obj);
1356}
1243 1357
1244/* nouveau_gem.c */ 1358/* nouveau_gem.c */
1245extern int nouveau_gem_new(struct drm_device *, struct nouveau_channel *, 1359extern int nouveau_gem_new(struct drm_device *, struct nouveau_channel *,
@@ -1259,15 +1373,28 @@ extern int nouveau_gem_ioctl_cpu_fini(struct drm_device *, void *,
1259extern int nouveau_gem_ioctl_info(struct drm_device *, void *, 1373extern int nouveau_gem_ioctl_info(struct drm_device *, void *,
1260 struct drm_file *); 1374 struct drm_file *);
1261 1375
1376/* nouveau_display.c */
1377int nouveau_vblank_enable(struct drm_device *dev, int crtc);
1378void nouveau_vblank_disable(struct drm_device *dev, int crtc);
1379int nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1380 struct drm_pending_vblank_event *event);
1381int nouveau_finish_page_flip(struct nouveau_channel *,
1382 struct nouveau_page_flip_state *);
1383
1262/* nv10_gpio.c */ 1384/* nv10_gpio.c */
1263int nv10_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag); 1385int nv10_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag);
1264int nv10_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state); 1386int nv10_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state);
1265 1387
1266/* nv50_gpio.c */ 1388/* nv50_gpio.c */
1267int nv50_gpio_init(struct drm_device *dev); 1389int nv50_gpio_init(struct drm_device *dev);
1390void nv50_gpio_fini(struct drm_device *dev);
1268int nv50_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag); 1391int nv50_gpio_get(struct drm_device *dev, enum dcb_gpio_tag tag);
1269int nv50_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state); 1392int nv50_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state);
1270void nv50_gpio_irq_enable(struct drm_device *, enum dcb_gpio_tag, bool on); 1393int nv50_gpio_irq_register(struct drm_device *, enum dcb_gpio_tag,
1394 void (*)(void *, int), void *);
1395void nv50_gpio_irq_unregister(struct drm_device *, enum dcb_gpio_tag,
1396 void (*)(void *, int), void *);
1397bool nv50_gpio_irq_enable(struct drm_device *, enum dcb_gpio_tag, bool on);
1271 1398
1272/* nv50_calc. */ 1399/* nv50_calc. */
1273int nv50_calc_pll(struct drm_device *, struct pll_lims *, int clk, 1400int nv50_calc_pll(struct drm_device *, struct pll_lims *, int clk,
@@ -1334,7 +1461,9 @@ static inline void nv_wr08(struct drm_device *dev, unsigned reg, u8 val)
1334} 1461}
1335 1462
1336#define nv_wait(dev, reg, mask, val) \ 1463#define nv_wait(dev, reg, mask, val) \
1337 nouveau_wait_until(dev, 2000000000ULL, (reg), (mask), (val)) 1464 nouveau_wait_eq(dev, 2000000000ULL, (reg), (mask), (val))
1465#define nv_wait_ne(dev, reg, mask, val) \
1466 nouveau_wait_ne(dev, 2000000000ULL, (reg), (mask), (val))
1338 1467
1339/* PRAMIN access */ 1468/* PRAMIN access */
1340static inline u32 nv_ri32(struct drm_device *dev, unsigned offset) 1469static inline u32 nv_ri32(struct drm_device *dev, unsigned offset)
@@ -1447,6 +1576,23 @@ nv_match_device(struct drm_device *dev, unsigned device,
1447 dev->pdev->subsystem_device == sub_device; 1576 dev->pdev->subsystem_device == sub_device;
1448} 1577}
1449 1578
1579/* memory type/access flags, do not match hardware values */
1580#define NV_MEM_ACCESS_RO 1
1581#define NV_MEM_ACCESS_WO 2
1582#define NV_MEM_ACCESS_RW (NV_MEM_ACCESS_RO | NV_MEM_ACCESS_WO)
1583#define NV_MEM_ACCESS_SYS 4
1584#define NV_MEM_ACCESS_VM 8
1585
1586#define NV_MEM_TARGET_VRAM 0
1587#define NV_MEM_TARGET_PCI 1
1588#define NV_MEM_TARGET_PCI_NOSNOOP 2
1589#define NV_MEM_TARGET_VM 3
1590#define NV_MEM_TARGET_GART 4
1591
1592#define NV_MEM_TYPE_VM 0x7f
1593#define NV_MEM_COMP_VM 0x03
1594
1595/* NV_SW object class */
1450#define NV_SW 0x0000506e 1596#define NV_SW 0x0000506e
1451#define NV_SW_DMA_SEMAPHORE 0x00000060 1597#define NV_SW_DMA_SEMAPHORE 0x00000060
1452#define NV_SW_SEMAPHORE_OFFSET 0x00000064 1598#define NV_SW_SEMAPHORE_OFFSET 0x00000064
@@ -1457,5 +1603,6 @@ nv_match_device(struct drm_device *dev, unsigned device,
1457#define NV_SW_VBLSEM_OFFSET 0x00000400 1603#define NV_SW_VBLSEM_OFFSET 0x00000400
1458#define NV_SW_VBLSEM_RELEASE_VALUE 0x00000404 1604#define NV_SW_VBLSEM_RELEASE_VALUE 0x00000404
1459#define NV_SW_VBLSEM_RELEASE 0x00000408 1605#define NV_SW_VBLSEM_RELEASE 0x00000408
1606#define NV_SW_PAGE_FLIP 0x00000500
1460 1607
1461#endif /* __NOUVEAU_DRV_H__ */ 1608#endif /* __NOUVEAU_DRV_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 02a4d1fd4845..a26d04740c88 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -49,6 +49,102 @@
49#include "nouveau_fbcon.h" 49#include "nouveau_fbcon.h"
50#include "nouveau_dma.h" 50#include "nouveau_dma.h"
51 51
52static void
53nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
54{
55 struct nouveau_fbdev *nfbdev = info->par;
56 struct drm_device *dev = nfbdev->dev;
57 struct drm_nouveau_private *dev_priv = dev->dev_private;
58 int ret;
59
60 if (info->state != FBINFO_STATE_RUNNING)
61 return;
62
63 ret = -ENODEV;
64 if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
65 mutex_trylock(&dev_priv->channel->mutex)) {
66 if (dev_priv->card_type < NV_50)
67 ret = nv04_fbcon_fillrect(info, rect);
68 else
69 if (dev_priv->card_type < NV_C0)
70 ret = nv50_fbcon_fillrect(info, rect);
71 else
72 ret = nvc0_fbcon_fillrect(info, rect);
73 mutex_unlock(&dev_priv->channel->mutex);
74 }
75
76 if (ret == 0)
77 return;
78
79 if (ret != -ENODEV)
80 nouveau_fbcon_gpu_lockup(info);
81 cfb_fillrect(info, rect);
82}
83
84static void
85nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
86{
87 struct nouveau_fbdev *nfbdev = info->par;
88 struct drm_device *dev = nfbdev->dev;
89 struct drm_nouveau_private *dev_priv = dev->dev_private;
90 int ret;
91
92 if (info->state != FBINFO_STATE_RUNNING)
93 return;
94
95 ret = -ENODEV;
96 if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
97 mutex_trylock(&dev_priv->channel->mutex)) {
98 if (dev_priv->card_type < NV_50)
99 ret = nv04_fbcon_copyarea(info, image);
100 else
101 if (dev_priv->card_type < NV_C0)
102 ret = nv50_fbcon_copyarea(info, image);
103 else
104 ret = nvc0_fbcon_copyarea(info, image);
105 mutex_unlock(&dev_priv->channel->mutex);
106 }
107
108 if (ret == 0)
109 return;
110
111 if (ret != -ENODEV)
112 nouveau_fbcon_gpu_lockup(info);
113 cfb_copyarea(info, image);
114}
115
116static void
117nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
118{
119 struct nouveau_fbdev *nfbdev = info->par;
120 struct drm_device *dev = nfbdev->dev;
121 struct drm_nouveau_private *dev_priv = dev->dev_private;
122 int ret;
123
124 if (info->state != FBINFO_STATE_RUNNING)
125 return;
126
127 ret = -ENODEV;
128 if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
129 mutex_trylock(&dev_priv->channel->mutex)) {
130 if (dev_priv->card_type < NV_50)
131 ret = nv04_fbcon_imageblit(info, image);
132 else
133 if (dev_priv->card_type < NV_C0)
134 ret = nv50_fbcon_imageblit(info, image);
135 else
136 ret = nvc0_fbcon_imageblit(info, image);
137 mutex_unlock(&dev_priv->channel->mutex);
138 }
139
140 if (ret == 0)
141 return;
142
143 if (ret != -ENODEV)
144 nouveau_fbcon_gpu_lockup(info);
145 cfb_imageblit(info, image);
146}
147
52static int 148static int
53nouveau_fbcon_sync(struct fb_info *info) 149nouveau_fbcon_sync(struct fb_info *info)
54{ 150{
@@ -58,22 +154,36 @@ nouveau_fbcon_sync(struct fb_info *info)
58 struct nouveau_channel *chan = dev_priv->channel; 154 struct nouveau_channel *chan = dev_priv->channel;
59 int ret, i; 155 int ret, i;
60 156
61 if (!chan || !chan->accel_done || 157 if (!chan || !chan->accel_done || in_interrupt() ||
62 info->state != FBINFO_STATE_RUNNING || 158 info->state != FBINFO_STATE_RUNNING ||
63 info->flags & FBINFO_HWACCEL_DISABLED) 159 info->flags & FBINFO_HWACCEL_DISABLED)
64 return 0; 160 return 0;
65 161
66 if (RING_SPACE(chan, 4)) { 162 if (!mutex_trylock(&chan->mutex))
163 return 0;
164
165 ret = RING_SPACE(chan, 4);
166 if (ret) {
167 mutex_unlock(&chan->mutex);
67 nouveau_fbcon_gpu_lockup(info); 168 nouveau_fbcon_gpu_lockup(info);
68 return 0; 169 return 0;
69 } 170 }
70 171
71 BEGIN_RING(chan, 0, 0x0104, 1); 172 if (dev_priv->card_type >= NV_C0) {
72 OUT_RING(chan, 0); 173 BEGIN_NVC0(chan, 2, NvSub2D, 0x010c, 1);
73 BEGIN_RING(chan, 0, 0x0100, 1); 174 OUT_RING (chan, 0);
74 OUT_RING(chan, 0); 175 BEGIN_NVC0(chan, 2, NvSub2D, 0x0100, 1);
176 OUT_RING (chan, 0);
177 } else {
178 BEGIN_RING(chan, 0, 0x0104, 1);
179 OUT_RING (chan, 0);
180 BEGIN_RING(chan, 0, 0x0100, 1);
181 OUT_RING (chan, 0);
182 }
183
75 nouveau_bo_wr32(chan->notifier_bo, chan->m2mf_ntfy + 3, 0xffffffff); 184 nouveau_bo_wr32(chan->notifier_bo, chan->m2mf_ntfy + 3, 0xffffffff);
76 FIRE_RING(chan); 185 FIRE_RING(chan);
186 mutex_unlock(&chan->mutex);
77 187
78 ret = -EBUSY; 188 ret = -EBUSY;
79 for (i = 0; i < 100000; i++) { 189 for (i = 0; i < 100000; i++) {
@@ -97,9 +207,9 @@ static struct fb_ops nouveau_fbcon_ops = {
97 .owner = THIS_MODULE, 207 .owner = THIS_MODULE,
98 .fb_check_var = drm_fb_helper_check_var, 208 .fb_check_var = drm_fb_helper_check_var,
99 .fb_set_par = drm_fb_helper_set_par, 209 .fb_set_par = drm_fb_helper_set_par,
100 .fb_fillrect = cfb_fillrect, 210 .fb_fillrect = nouveau_fbcon_fillrect,
101 .fb_copyarea = cfb_copyarea, 211 .fb_copyarea = nouveau_fbcon_copyarea,
102 .fb_imageblit = cfb_imageblit, 212 .fb_imageblit = nouveau_fbcon_imageblit,
103 .fb_sync = nouveau_fbcon_sync, 213 .fb_sync = nouveau_fbcon_sync,
104 .fb_pan_display = drm_fb_helper_pan_display, 214 .fb_pan_display = drm_fb_helper_pan_display,
105 .fb_blank = drm_fb_helper_blank, 215 .fb_blank = drm_fb_helper_blank,
@@ -108,29 +218,13 @@ static struct fb_ops nouveau_fbcon_ops = {
108 .fb_debug_leave = drm_fb_helper_debug_leave, 218 .fb_debug_leave = drm_fb_helper_debug_leave,
109}; 219};
110 220
111static struct fb_ops nv04_fbcon_ops = { 221static struct fb_ops nouveau_fbcon_sw_ops = {
112 .owner = THIS_MODULE, 222 .owner = THIS_MODULE,
113 .fb_check_var = drm_fb_helper_check_var, 223 .fb_check_var = drm_fb_helper_check_var,
114 .fb_set_par = drm_fb_helper_set_par, 224 .fb_set_par = drm_fb_helper_set_par,
115 .fb_fillrect = nv04_fbcon_fillrect, 225 .fb_fillrect = cfb_fillrect,
116 .fb_copyarea = nv04_fbcon_copyarea, 226 .fb_copyarea = cfb_copyarea,
117 .fb_imageblit = nv04_fbcon_imageblit, 227 .fb_imageblit = cfb_imageblit,
118 .fb_sync = nouveau_fbcon_sync,
119 .fb_pan_display = drm_fb_helper_pan_display,
120 .fb_blank = drm_fb_helper_blank,
121 .fb_setcmap = drm_fb_helper_setcmap,
122 .fb_debug_enter = drm_fb_helper_debug_enter,
123 .fb_debug_leave = drm_fb_helper_debug_leave,
124};
125
126static struct fb_ops nv50_fbcon_ops = {
127 .owner = THIS_MODULE,
128 .fb_check_var = drm_fb_helper_check_var,
129 .fb_set_par = drm_fb_helper_set_par,
130 .fb_fillrect = nv50_fbcon_fillrect,
131 .fb_copyarea = nv50_fbcon_copyarea,
132 .fb_imageblit = nv50_fbcon_imageblit,
133 .fb_sync = nouveau_fbcon_sync,
134 .fb_pan_display = drm_fb_helper_pan_display, 228 .fb_pan_display = drm_fb_helper_pan_display,
135 .fb_blank = drm_fb_helper_blank, 229 .fb_blank = drm_fb_helper_blank,
136 .fb_setcmap = drm_fb_helper_setcmap, 230 .fb_setcmap = drm_fb_helper_setcmap,
@@ -257,21 +351,16 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
257 FBINFO_HWACCEL_FILLRECT | 351 FBINFO_HWACCEL_FILLRECT |
258 FBINFO_HWACCEL_IMAGEBLIT; 352 FBINFO_HWACCEL_IMAGEBLIT;
259 info->flags |= FBINFO_CAN_FORCE_OUTPUT; 353 info->flags |= FBINFO_CAN_FORCE_OUTPUT;
260 info->fbops = &nouveau_fbcon_ops; 354 info->fbops = &nouveau_fbcon_sw_ops;
261 info->fix.smem_start = dev->mode_config.fb_base + nvbo->bo.offset - 355 info->fix.smem_start = dev->mode_config.fb_base +
262 dev_priv->vm_vram_base; 356 (nvbo->bo.mem.start << PAGE_SHIFT);
263 info->fix.smem_len = size; 357 info->fix.smem_len = size;
264 358
265 info->screen_base = nvbo_kmap_obj_iovirtual(nouveau_fb->nvbo); 359 info->screen_base = nvbo_kmap_obj_iovirtual(nouveau_fb->nvbo);
266 info->screen_size = size; 360 info->screen_size = size;
267 361
268 drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
269 drm_fb_helper_fill_var(info, &nfbdev->helper, sizes->fb_width, sizes->fb_height); 362 drm_fb_helper_fill_var(info, &nfbdev->helper, sizes->fb_width, sizes->fb_height);
270 363
271 /* FIXME: we really shouldn't expose mmio space at all */
272 info->fix.mmio_start = pci_resource_start(pdev, 1);
273 info->fix.mmio_len = pci_resource_len(pdev, 1);
274
275 /* Set aperture base/size for vesafb takeover */ 364 /* Set aperture base/size for vesafb takeover */
276 info->apertures = dev_priv->apertures; 365 info->apertures = dev_priv->apertures;
277 if (!info->apertures) { 366 if (!info->apertures) {
@@ -285,19 +374,20 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
285 info->pixmap.flags = FB_PIXMAP_SYSTEM; 374 info->pixmap.flags = FB_PIXMAP_SYSTEM;
286 info->pixmap.scan_align = 1; 375 info->pixmap.scan_align = 1;
287 376
377 mutex_unlock(&dev->struct_mutex);
378
288 if (dev_priv->channel && !nouveau_nofbaccel) { 379 if (dev_priv->channel && !nouveau_nofbaccel) {
289 switch (dev_priv->card_type) { 380 ret = -ENODEV;
290 case NV_C0: 381 if (dev_priv->card_type < NV_50)
291 break; 382 ret = nv04_fbcon_accel_init(info);
292 case NV_50: 383 else
293 nv50_fbcon_accel_init(info); 384 if (dev_priv->card_type < NV_C0)
294 info->fbops = &nv50_fbcon_ops; 385 ret = nv50_fbcon_accel_init(info);
295 break; 386 else
296 default: 387 ret = nvc0_fbcon_accel_init(info);
297 nv04_fbcon_accel_init(info); 388
298 info->fbops = &nv04_fbcon_ops; 389 if (ret == 0)
299 break; 390 info->fbops = &nouveau_fbcon_ops;
300 };
301 } 391 }
302 392
303 nouveau_fbcon_zfill(dev, nfbdev); 393 nouveau_fbcon_zfill(dev, nfbdev);
@@ -308,7 +398,6 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
308 nouveau_fb->base.height, 398 nouveau_fb->base.height,
309 nvbo->bo.offset, nvbo); 399 nvbo->bo.offset, nvbo);
310 400
311 mutex_unlock(&dev->struct_mutex);
312 vga_switcheroo_client_fb_set(dev->pdev, info); 401 vga_switcheroo_client_fb_set(dev->pdev, info);
313 return 0; 402 return 0;
314 403
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index e7e12684c37e..b73c29f87fc3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -40,15 +40,21 @@ struct nouveau_fbdev {
40 40
41void nouveau_fbcon_restore(void); 41void nouveau_fbcon_restore(void);
42 42
43void nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region); 43int nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
44void nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect); 44int nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
45void nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image); 45int nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
46int nv04_fbcon_accel_init(struct fb_info *info); 46int nv04_fbcon_accel_init(struct fb_info *info);
47void nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect); 47
48void nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region); 48int nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
49void nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image); 49int nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
50int nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
50int nv50_fbcon_accel_init(struct fb_info *info); 51int nv50_fbcon_accel_init(struct fb_info *info);
51 52
53int nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
54int nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region);
55int nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image);
56int nvc0_fbcon_accel_init(struct fb_info *info);
57
52void nouveau_fbcon_gpu_lockup(struct fb_info *info); 58void nouveau_fbcon_gpu_lockup(struct fb_info *info);
53 59
54int nouveau_fbcon_init(struct drm_device *dev); 60int nouveau_fbcon_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index ab1bbfbf266e..221b8462ea37 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -32,7 +32,8 @@
32#include "nouveau_dma.h" 32#include "nouveau_dma.h"
33 33
34#define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10) 34#define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
35#define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17) 35#define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17 && \
36 nouveau_private(dev)->card_type < NV_C0)
36 37
37struct nouveau_fence { 38struct nouveau_fence {
38 struct nouveau_channel *channel; 39 struct nouveau_channel *channel;
@@ -64,6 +65,7 @@ nouveau_fence_del(struct kref *ref)
64 struct nouveau_fence *fence = 65 struct nouveau_fence *fence =
65 container_of(ref, struct nouveau_fence, refcount); 66 container_of(ref, struct nouveau_fence, refcount);
66 67
68 nouveau_channel_ref(NULL, &fence->channel);
67 kfree(fence); 69 kfree(fence);
68} 70}
69 71
@@ -76,14 +78,17 @@ nouveau_fence_update(struct nouveau_channel *chan)
76 78
77 spin_lock(&chan->fence.lock); 79 spin_lock(&chan->fence.lock);
78 80
79 if (USE_REFCNT(dev)) 81 /* Fetch the last sequence if the channel is still up and running */
80 sequence = nvchan_rd32(chan, 0x48); 82 if (likely(!list_empty(&chan->fence.pending))) {
81 else 83 if (USE_REFCNT(dev))
82 sequence = atomic_read(&chan->fence.last_sequence_irq); 84 sequence = nvchan_rd32(chan, 0x48);
85 else
86 sequence = atomic_read(&chan->fence.last_sequence_irq);
83 87
84 if (chan->fence.sequence_ack == sequence) 88 if (chan->fence.sequence_ack == sequence)
85 goto out; 89 goto out;
86 chan->fence.sequence_ack = sequence; 90 chan->fence.sequence_ack = sequence;
91 }
87 92
88 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 93 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
89 sequence = fence->sequence; 94 sequence = fence->sequence;
@@ -113,13 +118,13 @@ nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
113 if (!fence) 118 if (!fence)
114 return -ENOMEM; 119 return -ENOMEM;
115 kref_init(&fence->refcount); 120 kref_init(&fence->refcount);
116 fence->channel = chan; 121 nouveau_channel_ref(chan, &fence->channel);
117 122
118 if (emit) 123 if (emit)
119 ret = nouveau_fence_emit(fence); 124 ret = nouveau_fence_emit(fence);
120 125
121 if (ret) 126 if (ret)
122 nouveau_fence_unref((void *)&fence); 127 nouveau_fence_unref(&fence);
123 *pfence = fence; 128 *pfence = fence;
124 return ret; 129 return ret;
125} 130}
@@ -127,7 +132,7 @@ nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
127struct nouveau_channel * 132struct nouveau_channel *
128nouveau_fence_channel(struct nouveau_fence *fence) 133nouveau_fence_channel(struct nouveau_fence *fence)
129{ 134{
130 return fence ? fence->channel : NULL; 135 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
131} 136}
132 137
133int 138int
@@ -135,6 +140,7 @@ nouveau_fence_emit(struct nouveau_fence *fence)
135{ 140{
136 struct nouveau_channel *chan = fence->channel; 141 struct nouveau_channel *chan = fence->channel;
137 struct drm_device *dev = chan->dev; 142 struct drm_device *dev = chan->dev;
143 struct drm_nouveau_private *dev_priv = dev->dev_private;
138 int ret; 144 int ret;
139 145
140 ret = RING_SPACE(chan, 2); 146 ret = RING_SPACE(chan, 2);
@@ -155,8 +161,15 @@ nouveau_fence_emit(struct nouveau_fence *fence)
155 list_add_tail(&fence->entry, &chan->fence.pending); 161 list_add_tail(&fence->entry, &chan->fence.pending);
156 spin_unlock(&chan->fence.lock); 162 spin_unlock(&chan->fence.lock);
157 163
158 BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1); 164 if (USE_REFCNT(dev)) {
159 OUT_RING(chan, fence->sequence); 165 if (dev_priv->card_type < NV_C0)
166 BEGIN_RING(chan, NvSubSw, 0x0050, 1);
167 else
168 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0050, 1);
169 } else {
170 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
171 }
172 OUT_RING (chan, fence->sequence);
160 FIRE_RING(chan); 173 FIRE_RING(chan);
161 174
162 return 0; 175 return 0;
@@ -182,7 +195,7 @@ nouveau_fence_work(struct nouveau_fence *fence,
182} 195}
183 196
184void 197void
185nouveau_fence_unref(void **sync_obj) 198__nouveau_fence_unref(void **sync_obj)
186{ 199{
187 struct nouveau_fence *fence = nouveau_fence(*sync_obj); 200 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
188 201
@@ -192,7 +205,7 @@ nouveau_fence_unref(void **sync_obj)
192} 205}
193 206
194void * 207void *
195nouveau_fence_ref(void *sync_obj) 208__nouveau_fence_ref(void *sync_obj)
196{ 209{
197 struct nouveau_fence *fence = nouveau_fence(sync_obj); 210 struct nouveau_fence *fence = nouveau_fence(sync_obj);
198 211
@@ -201,7 +214,7 @@ nouveau_fence_ref(void *sync_obj)
201} 214}
202 215
203bool 216bool
204nouveau_fence_signalled(void *sync_obj, void *sync_arg) 217__nouveau_fence_signalled(void *sync_obj, void *sync_arg)
205{ 218{
206 struct nouveau_fence *fence = nouveau_fence(sync_obj); 219 struct nouveau_fence *fence = nouveau_fence(sync_obj);
207 struct nouveau_channel *chan = fence->channel; 220 struct nouveau_channel *chan = fence->channel;
@@ -214,13 +227,14 @@ nouveau_fence_signalled(void *sync_obj, void *sync_arg)
214} 227}
215 228
216int 229int
217nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr) 230__nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
218{ 231{
219 unsigned long timeout = jiffies + (3 * DRM_HZ); 232 unsigned long timeout = jiffies + (3 * DRM_HZ);
233 unsigned long sleep_time = jiffies + 1;
220 int ret = 0; 234 int ret = 0;
221 235
222 while (1) { 236 while (1) {
223 if (nouveau_fence_signalled(sync_obj, sync_arg)) 237 if (__nouveau_fence_signalled(sync_obj, sync_arg))
224 break; 238 break;
225 239
226 if (time_after_eq(jiffies, timeout)) { 240 if (time_after_eq(jiffies, timeout)) {
@@ -230,7 +244,7 @@ nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
230 244
231 __set_current_state(intr ? TASK_INTERRUPTIBLE 245 __set_current_state(intr ? TASK_INTERRUPTIBLE
232 : TASK_UNINTERRUPTIBLE); 246 : TASK_UNINTERRUPTIBLE);
233 if (lazy) 247 if (lazy && time_after_eq(jiffies, sleep_time))
234 schedule_timeout(1); 248 schedule_timeout(1);
235 249
236 if (intr && signal_pending(current)) { 250 if (intr && signal_pending(current)) {
@@ -368,7 +382,7 @@ emit_semaphore(struct nouveau_channel *chan, int method,
368 382
369 kref_get(&sema->ref); 383 kref_get(&sema->ref);
370 nouveau_fence_work(fence, semaphore_work, sema); 384 nouveau_fence_work(fence, semaphore_work, sema);
371 nouveau_fence_unref((void *)&fence); 385 nouveau_fence_unref(&fence);
372 386
373 return 0; 387 return 0;
374} 388}
@@ -380,33 +394,49 @@ nouveau_fence_sync(struct nouveau_fence *fence,
380 struct nouveau_channel *chan = nouveau_fence_channel(fence); 394 struct nouveau_channel *chan = nouveau_fence_channel(fence);
381 struct drm_device *dev = wchan->dev; 395 struct drm_device *dev = wchan->dev;
382 struct nouveau_semaphore *sema; 396 struct nouveau_semaphore *sema;
383 int ret; 397 int ret = 0;
384 398
385 if (likely(!fence || chan == wchan || 399 if (likely(!chan || chan == wchan ||
386 nouveau_fence_signalled(fence, NULL))) 400 nouveau_fence_signalled(fence)))
387 return 0; 401 goto out;
388 402
389 sema = alloc_semaphore(dev); 403 sema = alloc_semaphore(dev);
390 if (!sema) { 404 if (!sema) {
391 /* Early card or broken userspace, fall back to 405 /* Early card or broken userspace, fall back to
392 * software sync. */ 406 * software sync. */
393 return nouveau_fence_wait(fence, NULL, false, false); 407 ret = nouveau_fence_wait(fence, true, false);
408 goto out;
409 }
410
411 /* try to take chan's mutex, if we can't take it right away
412 * we have to fallback to software sync to prevent locking
413 * order issues
414 */
415 if (!mutex_trylock(&chan->mutex)) {
416 ret = nouveau_fence_wait(fence, true, false);
417 goto out_unref;
394 } 418 }
395 419
396 /* Make wchan wait until it gets signalled */ 420 /* Make wchan wait until it gets signalled */
397 ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema); 421 ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
398 if (ret) 422 if (ret)
399 goto out; 423 goto out_unlock;
400 424
401 /* Signal the semaphore from chan */ 425 /* Signal the semaphore from chan */
402 ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema); 426 ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
403out: 427
428out_unlock:
429 mutex_unlock(&chan->mutex);
430out_unref:
404 kref_put(&sema->ref, free_semaphore); 431 kref_put(&sema->ref, free_semaphore);
432out:
433 if (chan)
434 nouveau_channel_put_unlocked(&chan);
405 return ret; 435 return ret;
406} 436}
407 437
408int 438int
409nouveau_fence_flush(void *sync_obj, void *sync_arg) 439__nouveau_fence_flush(void *sync_obj, void *sync_arg)
410{ 440{
411 return 0; 441 return 0;
412} 442}
@@ -420,30 +450,27 @@ nouveau_fence_channel_init(struct nouveau_channel *chan)
420 int ret; 450 int ret;
421 451
422 /* Create an NV_SW object for various sync purposes */ 452 /* Create an NV_SW object for various sync purposes */
423 ret = nouveau_gpuobj_sw_new(chan, NV_SW, &obj); 453 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
424 if (ret) 454 if (ret)
425 return ret; 455 return ret;
426 456
427 ret = nouveau_ramht_insert(chan, NvSw, obj); 457 /* we leave subchannel empty for nvc0 */
428 nouveau_gpuobj_ref(NULL, &obj); 458 if (dev_priv->card_type < NV_C0) {
429 if (ret) 459 ret = RING_SPACE(chan, 2);
430 return ret; 460 if (ret)
431 461 return ret;
432 ret = RING_SPACE(chan, 2); 462 BEGIN_RING(chan, NvSubSw, 0, 1);
433 if (ret) 463 OUT_RING(chan, NvSw);
434 return ret; 464 }
435 BEGIN_RING(chan, NvSubSw, 0, 1);
436 OUT_RING(chan, NvSw);
437 465
438 /* Create a DMA object for the shared cross-channel sync area. */ 466 /* Create a DMA object for the shared cross-channel sync area. */
439 if (USE_SEMA(dev)) { 467 if (USE_SEMA(dev)) {
440 struct drm_mm_node *mem = dev_priv->fence.bo->bo.mem.mm_node; 468 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
441 469
442 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 470 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
443 mem->start << PAGE_SHIFT, 471 mem->start << PAGE_SHIFT,
444 mem->size << PAGE_SHIFT, 472 mem->size, NV_MEM_ACCESS_RW,
445 NV_DMA_ACCESS_RW, 473 NV_MEM_TARGET_VRAM, &obj);
446 NV_DMA_TARGET_VIDMEM, &obj);
447 if (ret) 474 if (ret)
448 return ret; 475 return ret;
449 476
@@ -473,6 +500,8 @@ nouveau_fence_channel_fini(struct nouveau_channel *chan)
473{ 500{
474 struct nouveau_fence *tmp, *fence; 501 struct nouveau_fence *tmp, *fence;
475 502
503 spin_lock(&chan->fence.lock);
504
476 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) { 505 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
477 fence->signalled = true; 506 fence->signalled = true;
478 list_del(&fence->entry); 507 list_del(&fence->entry);
@@ -482,6 +511,8 @@ nouveau_fence_channel_fini(struct nouveau_channel *chan)
482 511
483 kref_put(&fence->refcount, nouveau_fence_del); 512 kref_put(&fence->refcount, nouveau_fence_del);
484 } 513 }
514
515 spin_unlock(&chan->fence.lock);
485} 516}
486 517
487int 518int
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 9a1fdcf400c2..506c508b7eda 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -48,9 +48,6 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
48 return; 48 return;
49 nvbo->gem = NULL; 49 nvbo->gem = NULL;
50 50
51 if (unlikely(nvbo->cpu_filp))
52 ttm_bo_synccpu_write_release(bo);
53
54 if (unlikely(nvbo->pin_refcnt)) { 51 if (unlikely(nvbo->pin_refcnt)) {
55 nvbo->pin_refcnt = 1; 52 nvbo->pin_refcnt = 1;
56 nouveau_bo_unpin(nvbo); 53 nouveau_bo_unpin(nvbo);
@@ -106,32 +103,6 @@ nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep)
106 return 0; 103 return 0;
107} 104}
108 105
109static bool
110nouveau_gem_tile_flags_valid(struct drm_device *dev, uint32_t tile_flags)
111{
112 struct drm_nouveau_private *dev_priv = dev->dev_private;
113
114 if (dev_priv->card_type >= NV_50) {
115 switch (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) {
116 case 0x0000:
117 case 0x1800:
118 case 0x2800:
119 case 0x4800:
120 case 0x7000:
121 case 0x7400:
122 case 0x7a00:
123 case 0xe000:
124 return true;
125 }
126 } else {
127 if (!(tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK))
128 return true;
129 }
130
131 NV_ERROR(dev, "bad page flags: 0x%08x\n", tile_flags);
132 return false;
133}
134
135int 106int
136nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 107nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
137 struct drm_file *file_priv) 108 struct drm_file *file_priv)
@@ -146,11 +117,6 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
146 if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL)) 117 if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL))
147 dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping; 118 dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping;
148 119
149 if (req->channel_hint) {
150 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel_hint,
151 file_priv, chan);
152 }
153
154 if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM) 120 if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM)
155 flags |= TTM_PL_FLAG_VRAM; 121 flags |= TTM_PL_FLAG_VRAM;
156 if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART) 122 if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART)
@@ -158,13 +124,23 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
158 if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU) 124 if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU)
159 flags |= TTM_PL_FLAG_SYSTEM; 125 flags |= TTM_PL_FLAG_SYSTEM;
160 126
161 if (!nouveau_gem_tile_flags_valid(dev, req->info.tile_flags)) 127 if (!dev_priv->engine.vram.flags_valid(dev, req->info.tile_flags)) {
128 NV_ERROR(dev, "bad page flags: 0x%08x\n", req->info.tile_flags);
162 return -EINVAL; 129 return -EINVAL;
130 }
131
132 if (req->channel_hint) {
133 chan = nouveau_channel_get(dev, file_priv, req->channel_hint);
134 if (IS_ERR(chan))
135 return PTR_ERR(chan);
136 }
163 137
164 ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags, 138 ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags,
165 req->info.tile_mode, req->info.tile_flags, false, 139 req->info.tile_mode, req->info.tile_flags, false,
166 (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE), 140 (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE),
167 &nvbo); 141 &nvbo);
142 if (chan)
143 nouveau_channel_put(&chan);
168 if (ret) 144 if (ret)
169 return ret; 145 return ret;
170 146
@@ -231,15 +207,8 @@ validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
231 207
232 list_for_each_safe(entry, tmp, list) { 208 list_for_each_safe(entry, tmp, list) {
233 nvbo = list_entry(entry, struct nouveau_bo, entry); 209 nvbo = list_entry(entry, struct nouveau_bo, entry);
234 if (likely(fence)) { 210
235 struct nouveau_fence *prev_fence; 211 nouveau_bo_fence(nvbo, fence);
236
237 spin_lock(&nvbo->bo.lock);
238 prev_fence = nvbo->bo.sync_obj;
239 nvbo->bo.sync_obj = nouveau_fence_ref(fence);
240 spin_unlock(&nvbo->bo.lock);
241 nouveau_fence_unref((void *)&prev_fence);
242 }
243 212
244 if (unlikely(nvbo->validate_mapped)) { 213 if (unlikely(nvbo->validate_mapped)) {
245 ttm_bo_kunmap(&nvbo->kmap); 214 ttm_bo_kunmap(&nvbo->kmap);
@@ -299,14 +268,15 @@ retry:
299 return -EINVAL; 268 return -EINVAL;
300 } 269 }
301 270
302 ret = ttm_bo_reserve(&nvbo->bo, false, false, true, sequence); 271 ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);
303 if (ret) { 272 if (ret) {
304 validate_fini(op, NULL); 273 validate_fini(op, NULL);
305 if (ret == -EAGAIN) 274 if (unlikely(ret == -EAGAIN))
306 ret = ttm_bo_wait_unreserved(&nvbo->bo, false); 275 ret = ttm_bo_wait_unreserved(&nvbo->bo, true);
307 drm_gem_object_unreference_unlocked(gem); 276 drm_gem_object_unreference_unlocked(gem);
308 if (ret) { 277 if (unlikely(ret)) {
309 NV_ERROR(dev, "fail reserve\n"); 278 if (ret != -ERESTARTSYS)
279 NV_ERROR(dev, "fail reserve\n");
310 return ret; 280 return ret;
311 } 281 }
312 goto retry; 282 goto retry;
@@ -331,25 +301,6 @@ retry:
331 validate_fini(op, NULL); 301 validate_fini(op, NULL);
332 return -EINVAL; 302 return -EINVAL;
333 } 303 }
334
335 if (unlikely(atomic_read(&nvbo->bo.cpu_writers) > 0)) {
336 validate_fini(op, NULL);
337
338 if (nvbo->cpu_filp == file_priv) {
339 NV_ERROR(dev, "bo %p mapped by process trying "
340 "to validate it!\n", nvbo);
341 return -EINVAL;
342 }
343
344 mutex_unlock(&drm_global_mutex);
345 ret = ttm_bo_wait_cpu(&nvbo->bo, false);
346 mutex_lock(&drm_global_mutex);
347 if (ret) {
348 NV_ERROR(dev, "fail wait_cpu\n");
349 return ret;
350 }
351 goto retry;
352 }
353 } 304 }
354 305
355 return 0; 306 return 0;
@@ -383,11 +334,11 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
383 } 334 }
384 335
385 nvbo->channel = (b->read_domains & (1 << 31)) ? NULL : chan; 336 nvbo->channel = (b->read_domains & (1 << 31)) ? NULL : chan;
386 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, 337 ret = nouveau_bo_validate(nvbo, true, false, false);
387 false, false, false);
388 nvbo->channel = NULL; 338 nvbo->channel = NULL;
389 if (unlikely(ret)) { 339 if (unlikely(ret)) {
390 NV_ERROR(dev, "fail ttm_validate\n"); 340 if (ret != -ERESTARTSYS)
341 NV_ERROR(dev, "fail ttm_validate\n");
391 return ret; 342 return ret;
392 } 343 }
393 344
@@ -439,13 +390,15 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
439 390
440 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 391 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
441 if (unlikely(ret)) { 392 if (unlikely(ret)) {
442 NV_ERROR(dev, "validate_init\n"); 393 if (ret != -ERESTARTSYS)
394 NV_ERROR(dev, "validate_init\n");
443 return ret; 395 return ret;
444 } 396 }
445 397
446 ret = validate_list(chan, &op->vram_list, pbbo, user_buffers); 398 ret = validate_list(chan, &op->vram_list, pbbo, user_buffers);
447 if (unlikely(ret < 0)) { 399 if (unlikely(ret < 0)) {
448 NV_ERROR(dev, "validate vram_list\n"); 400 if (ret != -ERESTARTSYS)
401 NV_ERROR(dev, "validate vram_list\n");
449 validate_fini(op, NULL); 402 validate_fini(op, NULL);
450 return ret; 403 return ret;
451 } 404 }
@@ -453,7 +406,8 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
453 406
454 ret = validate_list(chan, &op->gart_list, pbbo, user_buffers); 407 ret = validate_list(chan, &op->gart_list, pbbo, user_buffers);
455 if (unlikely(ret < 0)) { 408 if (unlikely(ret < 0)) {
456 NV_ERROR(dev, "validate gart_list\n"); 409 if (ret != -ERESTARTSYS)
410 NV_ERROR(dev, "validate gart_list\n");
457 validate_fini(op, NULL); 411 validate_fini(op, NULL);
458 return ret; 412 return ret;
459 } 413 }
@@ -461,7 +415,8 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
461 415
462 ret = validate_list(chan, &op->both_list, pbbo, user_buffers); 416 ret = validate_list(chan, &op->both_list, pbbo, user_buffers);
463 if (unlikely(ret < 0)) { 417 if (unlikely(ret < 0)) {
464 NV_ERROR(dev, "validate both_list\n"); 418 if (ret != -ERESTARTSYS)
419 NV_ERROR(dev, "validate both_list\n");
465 validate_fini(op, NULL); 420 validate_fini(op, NULL);
466 return ret; 421 return ret;
467 } 422 }
@@ -557,9 +512,9 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
557 data |= r->vor; 512 data |= r->vor;
558 } 513 }
559 514
560 spin_lock(&nvbo->bo.lock); 515 spin_lock(&nvbo->bo.bdev->fence_lock);
561 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 516 ret = ttm_bo_wait(&nvbo->bo, false, false, false);
562 spin_unlock(&nvbo->bo.lock); 517 spin_unlock(&nvbo->bo.bdev->fence_lock);
563 if (ret) { 518 if (ret) {
564 NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret); 519 NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret);
565 break; 520 break;
@@ -585,7 +540,9 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
585 struct nouveau_fence *fence = NULL; 540 struct nouveau_fence *fence = NULL;
586 int i, j, ret = 0, do_reloc = 0; 541 int i, j, ret = 0, do_reloc = 0;
587 542
588 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan); 543 chan = nouveau_channel_get(dev, file_priv, req->channel);
544 if (IS_ERR(chan))
545 return PTR_ERR(chan);
589 546
590 req->vram_available = dev_priv->fb_aper_free; 547 req->vram_available = dev_priv->fb_aper_free;
591 req->gart_available = dev_priv->gart_info.aper_free; 548 req->gart_available = dev_priv->gart_info.aper_free;
@@ -595,28 +552,34 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
595 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 552 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
596 NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n", 553 NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n",
597 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 554 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
555 nouveau_channel_put(&chan);
598 return -EINVAL; 556 return -EINVAL;
599 } 557 }
600 558
601 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 559 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
602 NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n", 560 NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n",
603 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 561 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
562 nouveau_channel_put(&chan);
604 return -EINVAL; 563 return -EINVAL;
605 } 564 }
606 565
607 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 566 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
608 NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n", 567 NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n",
609 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 568 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
569 nouveau_channel_put(&chan);
610 return -EINVAL; 570 return -EINVAL;
611 } 571 }
612 572
613 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 573 push = u_memcpya(req->push, req->nr_push, sizeof(*push));
614 if (IS_ERR(push)) 574 if (IS_ERR(push)) {
575 nouveau_channel_put(&chan);
615 return PTR_ERR(push); 576 return PTR_ERR(push);
577 }
616 578
617 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 579 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
618 if (IS_ERR(bo)) { 580 if (IS_ERR(bo)) {
619 kfree(push); 581 kfree(push);
582 nouveau_channel_put(&chan);
620 return PTR_ERR(bo); 583 return PTR_ERR(bo);
621 } 584 }
622 585
@@ -639,7 +602,8 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
639 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 602 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
640 req->nr_buffers, &op, &do_reloc); 603 req->nr_buffers, &op, &do_reloc);
641 if (ret) { 604 if (ret) {
642 NV_ERROR(dev, "validate: %d\n", ret); 605 if (ret != -ERESTARTSYS)
606 NV_ERROR(dev, "validate: %d\n", ret);
643 goto out; 607 goto out;
644 } 608 }
645 609
@@ -732,7 +696,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
732 696
733out: 697out:
734 validate_fini(&op, fence); 698 validate_fini(&op, fence);
735 nouveau_fence_unref((void**)&fence); 699 nouveau_fence_unref(&fence);
736 kfree(bo); 700 kfree(bo);
737 kfree(push); 701 kfree(push);
738 702
@@ -750,6 +714,7 @@ out_next:
750 req->suffix1 = 0x00000000; 714 req->suffix1 = 0x00000000;
751 } 715 }
752 716
717 nouveau_channel_put(&chan);
753 return ret; 718 return ret;
754} 719}
755 720
@@ -781,26 +746,9 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
781 return -ENOENT; 746 return -ENOENT;
782 nvbo = nouveau_gem_object(gem); 747 nvbo = nouveau_gem_object(gem);
783 748
784 if (nvbo->cpu_filp) { 749 spin_lock(&nvbo->bo.bdev->fence_lock);
785 if (nvbo->cpu_filp == file_priv) 750 ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait);
786 goto out; 751 spin_unlock(&nvbo->bo.bdev->fence_lock);
787
788 ret = ttm_bo_wait_cpu(&nvbo->bo, no_wait);
789 if (ret)
790 goto out;
791 }
792
793 if (req->flags & NOUVEAU_GEM_CPU_PREP_NOBLOCK) {
794 spin_lock(&nvbo->bo.lock);
795 ret = ttm_bo_wait(&nvbo->bo, false, false, no_wait);
796 spin_unlock(&nvbo->bo.lock);
797 } else {
798 ret = ttm_bo_synccpu_write_grab(&nvbo->bo, no_wait);
799 if (ret == 0)
800 nvbo->cpu_filp = file_priv;
801 }
802
803out:
804 drm_gem_object_unreference_unlocked(gem); 752 drm_gem_object_unreference_unlocked(gem);
805 return ret; 753 return ret;
806} 754}
@@ -809,26 +757,7 @@ int
809nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 757nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
810 struct drm_file *file_priv) 758 struct drm_file *file_priv)
811{ 759{
812 struct drm_nouveau_gem_cpu_prep *req = data; 760 return 0;
813 struct drm_gem_object *gem;
814 struct nouveau_bo *nvbo;
815 int ret = -EINVAL;
816
817 gem = drm_gem_object_lookup(dev, file_priv, req->handle);
818 if (!gem)
819 return -ENOENT;
820 nvbo = nouveau_gem_object(gem);
821
822 if (nvbo->cpu_filp != file_priv)
823 goto out;
824 nvbo->cpu_filp = NULL;
825
826 ttm_bo_synccpu_write_release(&nvbo->bo);
827 ret = 0;
828
829out:
830 drm_gem_object_unreference_unlocked(gem);
831 return ret;
832} 761}
833 762
834int 763int
diff --git a/drivers/gpu/drm/nouveau/nouveau_hw.c b/drivers/gpu/drm/nouveau/nouveau_hw.c
index b9672a05c411..053edf9d2f67 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hw.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hw.c
@@ -953,7 +953,7 @@ nv_load_state_ext(struct drm_device *dev, int head,
953 NVWriteCRTC(dev, head, NV_PCRTC_850, regp->crtc_850); 953 NVWriteCRTC(dev, head, NV_PCRTC_850, regp->crtc_850);
954 954
955 reg900 = NVReadRAMDAC(dev, head, NV_PRAMDAC_900); 955 reg900 = NVReadRAMDAC(dev, head, NV_PRAMDAC_900);
956 if (regp->crtc_cfg == NV_PCRTC_CONFIG_START_ADDRESS_HSYNC) 956 if (regp->crtc_cfg == NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC)
957 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 | 0x10000); 957 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 | 0x10000);
958 else 958 else
959 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 & ~0x10000); 959 NVWriteRAMDAC(dev, head, NV_PRAMDAC_900, reg900 & ~0x10000);
@@ -999,8 +999,8 @@ nv_load_state_ext(struct drm_device *dev, int head,
999 if (dev_priv->card_type == NV_10) { 999 if (dev_priv->card_type == NV_10) {
1000 /* Not waiting for vertical retrace before modifying 1000 /* Not waiting for vertical retrace before modifying
1001 CRE_53/CRE_54 causes lockups. */ 1001 CRE_53/CRE_54 causes lockups. */
1002 nouveau_wait_until(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8); 1002 nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
1003 nouveau_wait_until(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0); 1003 nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0);
1004 } 1004 }
1005 1005
1006 wr_cio_state(dev, head, regp, NV_CIO_CRE_53); 1006 wr_cio_state(dev, head, regp, NV_CIO_CRE_53);
@@ -1017,8 +1017,9 @@ nv_load_state_ext(struct drm_device *dev, int head,
1017 1017
1018 NVWriteCRTC(dev, head, NV_PCRTC_START, regp->fb_start); 1018 NVWriteCRTC(dev, head, NV_PCRTC_START, regp->fb_start);
1019 1019
1020 /* Setting 1 on this value gives you interrupts for every vblank period. */ 1020 /* Enable vblank interrupts. */
1021 NVWriteCRTC(dev, head, NV_PCRTC_INTR_EN_0, 0); 1021 NVWriteCRTC(dev, head, NV_PCRTC_INTR_EN_0,
1022 (dev->vblank_enabled[head] ? 1 : 0));
1022 NVWriteCRTC(dev, head, NV_PCRTC_INTR_0, NV_PCRTC_INTR_0_VBLANK); 1023 NVWriteCRTC(dev, head, NV_PCRTC_INTR_0, NV_PCRTC_INTR_0_VBLANK);
1023} 1024}
1024 1025
diff --git a/drivers/gpu/drm/nouveau/nouveau_irq.c b/drivers/gpu/drm/nouveau/nouveau_irq.c
index 7bfd9e6c9d67..2ba7265bc967 100644
--- a/drivers/gpu/drm/nouveau/nouveau_irq.c
+++ b/drivers/gpu/drm/nouveau/nouveau_irq.c
@@ -36,18 +36,7 @@
36#include "nouveau_drv.h" 36#include "nouveau_drv.h"
37#include "nouveau_reg.h" 37#include "nouveau_reg.h"
38#include "nouveau_ramht.h" 38#include "nouveau_ramht.h"
39#include <linux/ratelimit.h> 39#include "nouveau_util.h"
40
41/* needed for hotplug irq */
42#include "nouveau_connector.h"
43#include "nv50_display.h"
44
45static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
46
47static int nouveau_ratelimit(void)
48{
49 return __ratelimit(&nouveau_ratelimit_state);
50}
51 40
52void 41void
53nouveau_irq_preinstall(struct drm_device *dev) 42nouveau_irq_preinstall(struct drm_device *dev)
@@ -57,19 +46,19 @@ nouveau_irq_preinstall(struct drm_device *dev)
57 /* Master disable */ 46 /* Master disable */
58 nv_wr32(dev, NV03_PMC_INTR_EN_0, 0); 47 nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
59 48
60 if (dev_priv->card_type >= NV_50) { 49 INIT_LIST_HEAD(&dev_priv->vbl_waiting);
61 INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
62 INIT_WORK(&dev_priv->hpd_work, nv50_display_irq_hotplug_bh);
63 spin_lock_init(&dev_priv->hpd_state.lock);
64 INIT_LIST_HEAD(&dev_priv->vbl_waiting);
65 }
66} 50}
67 51
68int 52int
69nouveau_irq_postinstall(struct drm_device *dev) 53nouveau_irq_postinstall(struct drm_device *dev)
70{ 54{
55 struct drm_nouveau_private *dev_priv = dev->dev_private;
56
71 /* Master enable */ 57 /* Master enable */
72 nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE); 58 nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
59 if (dev_priv->msi_enabled)
60 nv_wr08(dev, 0x00088068, 0xff);
61
73 return 0; 62 return 0;
74} 63}
75 64
@@ -80,1178 +69,83 @@ nouveau_irq_uninstall(struct drm_device *dev)
80 nv_wr32(dev, NV03_PMC_INTR_EN_0, 0); 69 nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
81} 70}
82 71
83static int 72irqreturn_t
84nouveau_call_method(struct nouveau_channel *chan, int class, int mthd, int data) 73nouveau_irq_handler(DRM_IRQ_ARGS)
85{
86 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
87 struct nouveau_pgraph_object_method *grm;
88 struct nouveau_pgraph_object_class *grc;
89
90 grc = dev_priv->engine.graph.grclass;
91 while (grc->id) {
92 if (grc->id == class)
93 break;
94 grc++;
95 }
96
97 if (grc->id != class || !grc->methods)
98 return -ENOENT;
99
100 grm = grc->methods;
101 while (grm->id) {
102 if (grm->id == mthd)
103 return grm->exec(chan, class, mthd, data);
104 grm++;
105 }
106
107 return -ENOENT;
108}
109
110static bool
111nouveau_fifo_swmthd(struct nouveau_channel *chan, uint32_t addr, uint32_t data)
112{
113 struct drm_device *dev = chan->dev;
114 const int subc = (addr >> 13) & 0x7;
115 const int mthd = addr & 0x1ffc;
116
117 if (mthd == 0x0000) {
118 struct nouveau_gpuobj *gpuobj;
119
120 gpuobj = nouveau_ramht_find(chan, data);
121 if (!gpuobj)
122 return false;
123
124 if (gpuobj->engine != NVOBJ_ENGINE_SW)
125 return false;
126
127 chan->sw_subchannel[subc] = gpuobj->class;
128 nv_wr32(dev, NV04_PFIFO_CACHE1_ENGINE, nv_rd32(dev,
129 NV04_PFIFO_CACHE1_ENGINE) & ~(0xf << subc * 4));
130 return true;
131 }
132
133 /* hw object */
134 if (nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE) & (1 << (subc*4)))
135 return false;
136
137 if (nouveau_call_method(chan, chan->sw_subchannel[subc], mthd, data))
138 return false;
139
140 return true;
141}
142
143static void
144nouveau_fifo_irq_handler(struct drm_device *dev)
145{
146 struct drm_nouveau_private *dev_priv = dev->dev_private;
147 struct nouveau_engine *engine = &dev_priv->engine;
148 uint32_t status, reassign;
149 int cnt = 0;
150
151 reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
152 while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
153 struct nouveau_channel *chan = NULL;
154 uint32_t chid, get;
155
156 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
157
158 chid = engine->fifo.channel_id(dev);
159 if (chid >= 0 && chid < engine->fifo.channels)
160 chan = dev_priv->fifos[chid];
161 get = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
162
163 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
164 uint32_t mthd, data;
165 int ptr;
166
167 /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
168 * wrapping on my G80 chips, but CACHE1 isn't big
169 * enough for this much data.. Tests show that it
170 * wraps around to the start at GET=0x800.. No clue
171 * as to why..
172 */
173 ptr = (get & 0x7ff) >> 2;
174
175 if (dev_priv->card_type < NV_40) {
176 mthd = nv_rd32(dev,
177 NV04_PFIFO_CACHE1_METHOD(ptr));
178 data = nv_rd32(dev,
179 NV04_PFIFO_CACHE1_DATA(ptr));
180 } else {
181 mthd = nv_rd32(dev,
182 NV40_PFIFO_CACHE1_METHOD(ptr));
183 data = nv_rd32(dev,
184 NV40_PFIFO_CACHE1_DATA(ptr));
185 }
186
187 if (!chan || !nouveau_fifo_swmthd(chan, mthd, data)) {
188 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
189 "Mthd 0x%04x Data 0x%08x\n",
190 chid, (mthd >> 13) & 7, mthd & 0x1ffc,
191 data);
192 }
193
194 nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
195 nv_wr32(dev, NV03_PFIFO_INTR_0,
196 NV_PFIFO_INTR_CACHE_ERROR);
197
198 nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
199 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
200 nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
201 nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
202 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
203 nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
204
205 nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
206 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
207 nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
208
209 status &= ~NV_PFIFO_INTR_CACHE_ERROR;
210 }
211
212 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
213 u32 dma_get = nv_rd32(dev, 0x003244);
214 u32 dma_put = nv_rd32(dev, 0x003240);
215 u32 push = nv_rd32(dev, 0x003220);
216 u32 state = nv_rd32(dev, 0x003228);
217
218 if (dev_priv->card_type == NV_50) {
219 u32 ho_get = nv_rd32(dev, 0x003328);
220 u32 ho_put = nv_rd32(dev, 0x003320);
221 u32 ib_get = nv_rd32(dev, 0x003334);
222 u32 ib_put = nv_rd32(dev, 0x003330);
223
224 if (nouveau_ratelimit())
225 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
226 "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
227 "State 0x%08x Push 0x%08x\n",
228 chid, ho_get, dma_get, ho_put,
229 dma_put, ib_get, ib_put, state,
230 push);
231
232 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
233 nv_wr32(dev, 0x003364, 0x00000000);
234 if (dma_get != dma_put || ho_get != ho_put) {
235 nv_wr32(dev, 0x003244, dma_put);
236 nv_wr32(dev, 0x003328, ho_put);
237 } else
238 if (ib_get != ib_put) {
239 nv_wr32(dev, 0x003334, ib_put);
240 }
241 } else {
242 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
243 "Put 0x%08x State 0x%08x Push 0x%08x\n",
244 chid, dma_get, dma_put, state, push);
245
246 if (dma_get != dma_put)
247 nv_wr32(dev, 0x003244, dma_put);
248 }
249
250 nv_wr32(dev, 0x003228, 0x00000000);
251 nv_wr32(dev, 0x003220, 0x00000001);
252 nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
253 status &= ~NV_PFIFO_INTR_DMA_PUSHER;
254 }
255
256 if (status & NV_PFIFO_INTR_SEMAPHORE) {
257 uint32_t sem;
258
259 status &= ~NV_PFIFO_INTR_SEMAPHORE;
260 nv_wr32(dev, NV03_PFIFO_INTR_0,
261 NV_PFIFO_INTR_SEMAPHORE);
262
263 sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
264 nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
265
266 nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
267 nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
268 }
269
270 if (dev_priv->card_type == NV_50) {
271 if (status & 0x00000010) {
272 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
273 status &= ~0x00000010;
274 nv_wr32(dev, 0x002100, 0x00000010);
275 }
276 }
277
278 if (status) {
279 if (nouveau_ratelimit())
280 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
281 status, chid);
282 nv_wr32(dev, NV03_PFIFO_INTR_0, status);
283 status = 0;
284 }
285
286 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
287 }
288
289 if (status) {
290 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
291 nv_wr32(dev, 0x2140, 0);
292 nv_wr32(dev, 0x140, 0);
293 }
294
295 nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
296}
297
298struct nouveau_bitfield_names {
299 uint32_t mask;
300 const char *name;
301};
302
303static struct nouveau_bitfield_names nstatus_names[] =
304{
305 { NV04_PGRAPH_NSTATUS_STATE_IN_USE, "STATE_IN_USE" },
306 { NV04_PGRAPH_NSTATUS_INVALID_STATE, "INVALID_STATE" },
307 { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT, "BAD_ARGUMENT" },
308 { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT, "PROTECTION_FAULT" }
309};
310
311static struct nouveau_bitfield_names nstatus_names_nv10[] =
312{
313 { NV10_PGRAPH_NSTATUS_STATE_IN_USE, "STATE_IN_USE" },
314 { NV10_PGRAPH_NSTATUS_INVALID_STATE, "INVALID_STATE" },
315 { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT, "BAD_ARGUMENT" },
316 { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT, "PROTECTION_FAULT" }
317};
318
319static struct nouveau_bitfield_names nsource_names[] =
320{
321 { NV03_PGRAPH_NSOURCE_NOTIFICATION, "NOTIFICATION" },
322 { NV03_PGRAPH_NSOURCE_DATA_ERROR, "DATA_ERROR" },
323 { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR, "PROTECTION_ERROR" },
324 { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION, "RANGE_EXCEPTION" },
325 { NV03_PGRAPH_NSOURCE_LIMIT_COLOR, "LIMIT_COLOR" },
326 { NV03_PGRAPH_NSOURCE_LIMIT_ZETA, "LIMIT_ZETA" },
327 { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD, "ILLEGAL_MTHD" },
328 { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION, "DMA_R_PROTECTION" },
329 { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION, "DMA_W_PROTECTION" },
330 { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION, "FORMAT_EXCEPTION" },
331 { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION, "PATCH_EXCEPTION" },
332 { NV03_PGRAPH_NSOURCE_STATE_INVALID, "STATE_INVALID" },
333 { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY, "DOUBLE_NOTIFY" },
334 { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE, "NOTIFY_IN_USE" },
335 { NV03_PGRAPH_NSOURCE_METHOD_CNT, "METHOD_CNT" },
336 { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION, "BFR_NOTIFICATION" },
337 { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
338 { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A, "DMA_WIDTH_A" },
339 { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B, "DMA_WIDTH_B" },
340};
341
342static void
343nouveau_print_bitfield_names_(uint32_t value,
344 const struct nouveau_bitfield_names *namelist,
345 const int namelist_len)
346{
347 /*
348 * Caller must have already printed the KERN_* log level for us.
349 * Also the caller is responsible for adding the newline.
350 */
351 int i;
352 for (i = 0; i < namelist_len; ++i) {
353 uint32_t mask = namelist[i].mask;
354 if (value & mask) {
355 printk(" %s", namelist[i].name);
356 value &= ~mask;
357 }
358 }
359 if (value)
360 printk(" (unknown bits 0x%08x)", value);
361}
362#define nouveau_print_bitfield_names(val, namelist) \
363 nouveau_print_bitfield_names_((val), (namelist), ARRAY_SIZE(namelist))
364
365struct nouveau_enum_names {
366 uint32_t value;
367 const char *name;
368};
369
370static void
371nouveau_print_enum_names_(uint32_t value,
372 const struct nouveau_enum_names *namelist,
373 const int namelist_len)
374{
375 /*
376 * Caller must have already printed the KERN_* log level for us.
377 * Also the caller is responsible for adding the newline.
378 */
379 int i;
380 for (i = 0; i < namelist_len; ++i) {
381 if (value == namelist[i].value) {
382 printk("%s", namelist[i].name);
383 return;
384 }
385 }
386 printk("unknown value 0x%08x", value);
387}
388#define nouveau_print_enum_names(val, namelist) \
389 nouveau_print_enum_names_((val), (namelist), ARRAY_SIZE(namelist))
390
391static int
392nouveau_graph_chid_from_grctx(struct drm_device *dev)
393{ 74{
75 struct drm_device *dev = (struct drm_device *)arg;
394 struct drm_nouveau_private *dev_priv = dev->dev_private; 76 struct drm_nouveau_private *dev_priv = dev->dev_private;
395 uint32_t inst; 77 unsigned long flags;
78 u32 stat;
396 int i; 79 int i;
397 80
398 if (dev_priv->card_type < NV_40) 81 stat = nv_rd32(dev, NV03_PMC_INTR_0);
399 return dev_priv->engine.fifo.channels; 82 if (!stat)
400 else 83 return IRQ_NONE;
401 if (dev_priv->card_type < NV_50) {
402 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 4;
403
404 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
405 struct nouveau_channel *chan = dev_priv->fifos[i];
406
407 if (!chan || !chan->ramin_grctx)
408 continue;
409
410 if (inst == chan->ramin_grctx->pinst)
411 break;
412 }
413 } else {
414 inst = (nv_rd32(dev, 0x40032c) & 0xfffff) << 12;
415
416 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
417 struct nouveau_channel *chan = dev_priv->fifos[i];
418
419 if (!chan || !chan->ramin)
420 continue;
421
422 if (inst == chan->ramin->vinst)
423 break;
424 }
425 }
426
427
428 return i;
429}
430
431static int
432nouveau_graph_trapped_channel(struct drm_device *dev, int *channel_ret)
433{
434 struct drm_nouveau_private *dev_priv = dev->dev_private;
435 struct nouveau_engine *engine = &dev_priv->engine;
436 int channel;
437
438 if (dev_priv->card_type < NV_10)
439 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0xf;
440 else
441 if (dev_priv->card_type < NV_40)
442 channel = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
443 else
444 channel = nouveau_graph_chid_from_grctx(dev);
445
446 if (channel >= engine->fifo.channels || !dev_priv->fifos[channel]) {
447 NV_ERROR(dev, "AIII, invalid/inactive channel id %d\n", channel);
448 return -EINVAL;
449 }
450
451 *channel_ret = channel;
452 return 0;
453}
454
455struct nouveau_pgraph_trap {
456 int channel;
457 int class;
458 int subc, mthd, size;
459 uint32_t data, data2;
460 uint32_t nsource, nstatus;
461};
462
463static void
464nouveau_graph_trap_info(struct drm_device *dev,
465 struct nouveau_pgraph_trap *trap)
466{
467 struct drm_nouveau_private *dev_priv = dev->dev_private;
468 uint32_t address;
469
470 trap->nsource = trap->nstatus = 0;
471 if (dev_priv->card_type < NV_50) {
472 trap->nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
473 trap->nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
474 }
475
476 if (nouveau_graph_trapped_channel(dev, &trap->channel))
477 trap->channel = -1;
478 address = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
479
480 trap->mthd = address & 0x1FFC;
481 trap->data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
482 if (dev_priv->card_type < NV_10) {
483 trap->subc = (address >> 13) & 0x7;
484 } else {
485 trap->subc = (address >> 16) & 0x7;
486 trap->data2 = nv_rd32(dev, NV10_PGRAPH_TRAPPED_DATA_HIGH);
487 }
488
489 if (dev_priv->card_type < NV_10)
490 trap->class = nv_rd32(dev, 0x400180 + trap->subc*4) & 0xFF;
491 else if (dev_priv->card_type < NV_40)
492 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFF;
493 else if (dev_priv->card_type < NV_50)
494 trap->class = nv_rd32(dev, 0x400160 + trap->subc*4) & 0xFFFF;
495 else
496 trap->class = nv_rd32(dev, 0x400814);
497}
498
499static void
500nouveau_graph_dump_trap_info(struct drm_device *dev, const char *id,
501 struct nouveau_pgraph_trap *trap)
502{
503 struct drm_nouveau_private *dev_priv = dev->dev_private;
504 uint32_t nsource = trap->nsource, nstatus = trap->nstatus;
505
506 if (dev_priv->card_type < NV_50) {
507 NV_INFO(dev, "%s - nSource:", id);
508 nouveau_print_bitfield_names(nsource, nsource_names);
509 printk(", nStatus:");
510 if (dev_priv->card_type < NV_10)
511 nouveau_print_bitfield_names(nstatus, nstatus_names);
512 else
513 nouveau_print_bitfield_names(nstatus, nstatus_names_nv10);
514 printk("\n");
515 }
516
517 NV_INFO(dev, "%s - Ch %d/%d Class 0x%04x Mthd 0x%04x "
518 "Data 0x%08x:0x%08x\n",
519 id, trap->channel, trap->subc,
520 trap->class, trap->mthd,
521 trap->data2, trap->data);
522}
523
524static int
525nouveau_pgraph_intr_swmthd(struct drm_device *dev,
526 struct nouveau_pgraph_trap *trap)
527{
528 struct drm_nouveau_private *dev_priv = dev->dev_private;
529
530 if (trap->channel < 0 ||
531 trap->channel >= dev_priv->engine.fifo.channels ||
532 !dev_priv->fifos[trap->channel])
533 return -ENODEV;
534
535 return nouveau_call_method(dev_priv->fifos[trap->channel],
536 trap->class, trap->mthd, trap->data);
537}
538
539static inline void
540nouveau_pgraph_intr_notify(struct drm_device *dev, uint32_t nsource)
541{
542 struct nouveau_pgraph_trap trap;
543 int unhandled = 0;
544 84
545 nouveau_graph_trap_info(dev, &trap); 85 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
86 for (i = 0; i < 32 && stat; i++) {
87 if (!(stat & (1 << i)) || !dev_priv->irq_handler[i])
88 continue;
546 89
547 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) { 90 dev_priv->irq_handler[i](dev);
548 if (nouveau_pgraph_intr_swmthd(dev, &trap)) 91 stat &= ~(1 << i);
549 unhandled = 1;
550 } else {
551 unhandled = 1;
552 } 92 }
553 93
554 if (unhandled) 94 if (dev_priv->msi_enabled)
555 nouveau_graph_dump_trap_info(dev, "PGRAPH_NOTIFY", &trap); 95 nv_wr08(dev, 0x00088068, 0xff);
556} 96 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
557
558
559static inline void
560nouveau_pgraph_intr_error(struct drm_device *dev, uint32_t nsource)
561{
562 struct nouveau_pgraph_trap trap;
563 int unhandled = 0;
564
565 nouveau_graph_trap_info(dev, &trap);
566 trap.nsource = nsource;
567
568 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
569 if (nouveau_pgraph_intr_swmthd(dev, &trap))
570 unhandled = 1;
571 } else if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
572 uint32_t v = nv_rd32(dev, 0x402000);
573 nv_wr32(dev, 0x402000, v);
574
575 /* dump the error anyway for now: it's useful for
576 Gallium development */
577 unhandled = 1;
578 } else {
579 unhandled = 1;
580 }
581 97
582 if (unhandled && nouveau_ratelimit()) 98 if (stat && nouveau_ratelimit())
583 nouveau_graph_dump_trap_info(dev, "PGRAPH_ERROR", &trap); 99 NV_ERROR(dev, "PMC - unhandled INTR 0x%08x\n", stat);
100 return IRQ_HANDLED;
584} 101}
585 102
586static inline void 103int
587nouveau_pgraph_intr_context_switch(struct drm_device *dev) 104nouveau_irq_init(struct drm_device *dev)
588{ 105{
589 struct drm_nouveau_private *dev_priv = dev->dev_private; 106 struct drm_nouveau_private *dev_priv = dev->dev_private;
590 struct nouveau_engine *engine = &dev_priv->engine; 107 int ret;
591 uint32_t chid;
592
593 chid = engine->fifo.channel_id(dev);
594 NV_DEBUG(dev, "PGRAPH context switch interrupt channel %x\n", chid);
595
596 switch (dev_priv->card_type) {
597 case NV_04:
598 nv04_graph_context_switch(dev);
599 break;
600 case NV_10:
601 nv10_graph_context_switch(dev);
602 break;
603 default:
604 NV_ERROR(dev, "Context switch not implemented\n");
605 break;
606 }
607}
608
609static void
610nouveau_pgraph_irq_handler(struct drm_device *dev)
611{
612 uint32_t status;
613
614 while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
615 uint32_t nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
616
617 if (status & NV_PGRAPH_INTR_NOTIFY) {
618 nouveau_pgraph_intr_notify(dev, nsource);
619
620 status &= ~NV_PGRAPH_INTR_NOTIFY;
621 nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_NOTIFY);
622 }
623
624 if (status & NV_PGRAPH_INTR_ERROR) {
625 nouveau_pgraph_intr_error(dev, nsource);
626 108
627 status &= ~NV_PGRAPH_INTR_ERROR; 109 if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
628 nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_ERROR); 110 ret = pci_enable_msi(dev->pdev);
111 if (ret == 0) {
112 NV_INFO(dev, "enabled MSI\n");
113 dev_priv->msi_enabled = true;
629 } 114 }
630
631 if (status & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
632 status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
633 nv_wr32(dev, NV03_PGRAPH_INTR,
634 NV_PGRAPH_INTR_CONTEXT_SWITCH);
635
636 nouveau_pgraph_intr_context_switch(dev);
637 }
638
639 if (status) {
640 NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n", status);
641 nv_wr32(dev, NV03_PGRAPH_INTR, status);
642 }
643
644 if ((nv_rd32(dev, NV04_PGRAPH_FIFO) & (1 << 0)) == 0)
645 nv_wr32(dev, NV04_PGRAPH_FIFO, 1);
646 } 115 }
647 116
648 nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING); 117 return drm_irq_install(dev);
649}
650
651static struct nouveau_enum_names nv50_mp_exec_error_names[] =
652{
653 { 3, "STACK_UNDERFLOW" },
654 { 4, "QUADON_ACTIVE" },
655 { 8, "TIMEOUT" },
656 { 0x10, "INVALID_OPCODE" },
657 { 0x40, "BREAKPOINT" },
658};
659
660static void
661nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
662{
663 struct drm_nouveau_private *dev_priv = dev->dev_private;
664 uint32_t units = nv_rd32(dev, 0x1540);
665 uint32_t addr, mp10, status, pc, oplow, ophigh;
666 int i;
667 int mps = 0;
668 for (i = 0; i < 4; i++) {
669 if (!(units & 1 << (i+24)))
670 continue;
671 if (dev_priv->chipset < 0xa0)
672 addr = 0x408200 + (tpid << 12) + (i << 7);
673 else
674 addr = 0x408100 + (tpid << 11) + (i << 7);
675 mp10 = nv_rd32(dev, addr + 0x10);
676 status = nv_rd32(dev, addr + 0x14);
677 if (!status)
678 continue;
679 if (display) {
680 nv_rd32(dev, addr + 0x20);
681 pc = nv_rd32(dev, addr + 0x24);
682 oplow = nv_rd32(dev, addr + 0x70);
683 ophigh= nv_rd32(dev, addr + 0x74);
684 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
685 "TP %d MP %d: ", tpid, i);
686 nouveau_print_enum_names(status,
687 nv50_mp_exec_error_names);
688 printk(" at %06x warp %d, opcode %08x %08x\n",
689 pc&0xffffff, pc >> 24,
690 oplow, ophigh);
691 }
692 nv_wr32(dev, addr + 0x10, mp10);
693 nv_wr32(dev, addr + 0x14, 0);
694 mps++;
695 }
696 if (!mps && display)
697 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
698 "No MPs claiming errors?\n", tpid);
699} 118}
700 119
701static void 120void
702nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old, 121nouveau_irq_fini(struct drm_device *dev)
703 uint32_t ustatus_new, int display, const char *name)
704{ 122{
705 struct drm_nouveau_private *dev_priv = dev->dev_private; 123 struct drm_nouveau_private *dev_priv = dev->dev_private;
706 int tps = 0;
707 uint32_t units = nv_rd32(dev, 0x1540);
708 int i, r;
709 uint32_t ustatus_addr, ustatus;
710 for (i = 0; i < 16; i++) {
711 if (!(units & (1 << i)))
712 continue;
713 if (dev_priv->chipset < 0xa0)
714 ustatus_addr = ustatus_old + (i << 12);
715 else
716 ustatus_addr = ustatus_new + (i << 11);
717 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
718 if (!ustatus)
719 continue;
720 tps++;
721 switch (type) {
722 case 6: /* texture error... unknown for now */
723 nv50_fb_vm_trap(dev, display, name);
724 if (display) {
725 NV_ERROR(dev, "magic set %d:\n", i);
726 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
727 NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
728 nv_rd32(dev, r));
729 }
730 break;
731 case 7: /* MP error */
732 if (ustatus & 0x00010000) {
733 nv50_pgraph_mp_trap(dev, i, display);
734 ustatus &= ~0x00010000;
735 }
736 break;
737 case 8: /* TPDMA error */
738 {
739 uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
740 uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
741 uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
742 uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
743 uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
744 uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
745 uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
746 nv50_fb_vm_trap(dev, display, name);
747 /* 2d engine destination */
748 if (ustatus & 0x00000010) {
749 if (display) {
750 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
751 i, e14, e10);
752 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
753 i, e0c, e18, e1c, e20, e24);
754 }
755 ustatus &= ~0x00000010;
756 }
757 /* Render target */
758 if (ustatus & 0x00000040) {
759 if (display) {
760 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
761 i, e14, e10);
762 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
763 i, e0c, e18, e1c, e20, e24);
764 }
765 ustatus &= ~0x00000040;
766 }
767 /* CUDA memory: l[], g[] or stack. */
768 if (ustatus & 0x00000080) {
769 if (display) {
770 if (e18 & 0x80000000) {
771 /* g[] read fault? */
772 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
773 i, e14, e10 | ((e18 >> 24) & 0x1f));
774 e18 &= ~0x1f000000;
775 } else if (e18 & 0xc) {
776 /* g[] write fault? */
777 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
778 i, e14, e10 | ((e18 >> 7) & 0x1f));
779 e18 &= ~0x00000f80;
780 } else {
781 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
782 i, e14, e10);
783 }
784 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
785 i, e0c, e18, e1c, e20, e24);
786 }
787 ustatus &= ~0x00000080;
788 }
789 }
790 break;
791 }
792 if (ustatus) {
793 if (display)
794 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
795 }
796 nv_wr32(dev, ustatus_addr, 0xc0000000);
797 }
798
799 if (!tps && display)
800 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
801}
802
803static void
804nv50_pgraph_trap_handler(struct drm_device *dev)
805{
806 struct nouveau_pgraph_trap trap;
807 uint32_t status = nv_rd32(dev, 0x400108);
808 uint32_t ustatus;
809 int display = nouveau_ratelimit();
810
811
812 if (!status && display) {
813 nouveau_graph_trap_info(dev, &trap);
814 nouveau_graph_dump_trap_info(dev, "PGRAPH_TRAP", &trap);
815 NV_INFO(dev, "PGRAPH_TRAP - no units reporting traps?\n");
816 }
817
818 /* DISPATCH: Relays commands to other units and handles NOTIFY,
819 * COND, QUERY. If you get a trap from it, the command is still stuck
820 * in DISPATCH and you need to do something about it. */
821 if (status & 0x001) {
822 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
823 if (!ustatus && display) {
824 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
825 }
826
827 /* Known to be triggered by screwed up NOTIFY and COND... */
828 if (ustatus & 0x00000001) {
829 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_FAULT");
830 nv_wr32(dev, 0x400500, 0);
831 if (nv_rd32(dev, 0x400808) & 0x80000000) {
832 if (display) {
833 if (nouveau_graph_trapped_channel(dev, &trap.channel))
834 trap.channel = -1;
835 trap.class = nv_rd32(dev, 0x400814);
836 trap.mthd = nv_rd32(dev, 0x400808) & 0x1ffc;
837 trap.subc = (nv_rd32(dev, 0x400808) >> 16) & 0x7;
838 trap.data = nv_rd32(dev, 0x40080c);
839 trap.data2 = nv_rd32(dev, 0x400810);
840 nouveau_graph_dump_trap_info(dev,
841 "PGRAPH_TRAP_DISPATCH_FAULT", &trap);
842 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400808: %08x\n", nv_rd32(dev, 0x400808));
843 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - 400848: %08x\n", nv_rd32(dev, 0x400848));
844 }
845 nv_wr32(dev, 0x400808, 0);
846 } else if (display) {
847 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_FAULT - No stuck command?\n");
848 }
849 nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
850 nv_wr32(dev, 0x400848, 0);
851 ustatus &= ~0x00000001;
852 }
853 if (ustatus & 0x00000002) {
854 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_DISPATCH_QUERY");
855 nv_wr32(dev, 0x400500, 0);
856 if (nv_rd32(dev, 0x40084c) & 0x80000000) {
857 if (display) {
858 if (nouveau_graph_trapped_channel(dev, &trap.channel))
859 trap.channel = -1;
860 trap.class = nv_rd32(dev, 0x400814);
861 trap.mthd = nv_rd32(dev, 0x40084c) & 0x1ffc;
862 trap.subc = (nv_rd32(dev, 0x40084c) >> 16) & 0x7;
863 trap.data = nv_rd32(dev, 0x40085c);
864 trap.data2 = 0;
865 nouveau_graph_dump_trap_info(dev,
866 "PGRAPH_TRAP_DISPATCH_QUERY", &trap);
867 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - 40084c: %08x\n", nv_rd32(dev, 0x40084c));
868 }
869 nv_wr32(dev, 0x40084c, 0);
870 } else if (display) {
871 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH_QUERY - No stuck command?\n");
872 }
873 ustatus &= ~0x00000002;
874 }
875 if (ustatus && display)
876 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - Unhandled ustatus 0x%08x\n", ustatus);
877 nv_wr32(dev, 0x400804, 0xc0000000);
878 nv_wr32(dev, 0x400108, 0x001);
879 status &= ~0x001;
880 }
881
882 /* TRAPs other than dispatch use the "normal" trap regs. */
883 if (status && display) {
884 nouveau_graph_trap_info(dev, &trap);
885 nouveau_graph_dump_trap_info(dev,
886 "PGRAPH_TRAP", &trap);
887 }
888
889 /* M2MF: Memory to memory copy engine. */
890 if (status & 0x002) {
891 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
892 if (!ustatus && display) {
893 NV_INFO(dev, "PGRAPH_TRAP_M2MF - no ustatus?\n");
894 }
895 if (ustatus & 0x00000001) {
896 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_NOTIFY");
897 ustatus &= ~0x00000001;
898 }
899 if (ustatus & 0x00000002) {
900 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_IN");
901 ustatus &= ~0x00000002;
902 }
903 if (ustatus & 0x00000004) {
904 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_M2MF_OUT");
905 ustatus &= ~0x00000004;
906 }
907 NV_INFO (dev, "PGRAPH_TRAP_M2MF - %08x %08x %08x %08x\n",
908 nv_rd32(dev, 0x406804),
909 nv_rd32(dev, 0x406808),
910 nv_rd32(dev, 0x40680c),
911 nv_rd32(dev, 0x406810));
912 if (ustatus && display)
913 NV_INFO(dev, "PGRAPH_TRAP_M2MF - Unhandled ustatus 0x%08x\n", ustatus);
914 /* No sane way found yet -- just reset the bugger. */
915 nv_wr32(dev, 0x400040, 2);
916 nv_wr32(dev, 0x400040, 0);
917 nv_wr32(dev, 0x406800, 0xc0000000);
918 nv_wr32(dev, 0x400108, 0x002);
919 status &= ~0x002;
920 }
921
922 /* VFETCH: Fetches data from vertex buffers. */
923 if (status & 0x004) {
924 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
925 if (!ustatus && display) {
926 NV_INFO(dev, "PGRAPH_TRAP_VFETCH - no ustatus?\n");
927 }
928 if (ustatus & 0x00000001) {
929 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_VFETCH_FAULT");
930 NV_INFO (dev, "PGRAPH_TRAP_VFETCH_FAULT - %08x %08x %08x %08x\n",
931 nv_rd32(dev, 0x400c00),
932 nv_rd32(dev, 0x400c08),
933 nv_rd32(dev, 0x400c0c),
934 nv_rd32(dev, 0x400c10));
935 ustatus &= ~0x00000001;
936 }
937 if (ustatus && display)
938 NV_INFO(dev, "PGRAPH_TRAP_VFETCH - Unhandled ustatus 0x%08x\n", ustatus);
939 nv_wr32(dev, 0x400c04, 0xc0000000);
940 nv_wr32(dev, 0x400108, 0x004);
941 status &= ~0x004;
942 }
943
944 /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
945 if (status & 0x008) {
946 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
947 if (!ustatus && display) {
948 NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - no ustatus?\n");
949 }
950 if (ustatus & 0x00000001) {
951 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_STRMOUT_FAULT");
952 NV_INFO (dev, "PGRAPH_TRAP_STRMOUT_FAULT - %08x %08x %08x %08x\n",
953 nv_rd32(dev, 0x401804),
954 nv_rd32(dev, 0x401808),
955 nv_rd32(dev, 0x40180c),
956 nv_rd32(dev, 0x401810));
957 ustatus &= ~0x00000001;
958 }
959 if (ustatus && display)
960 NV_INFO(dev, "PGRAPH_TRAP_STRMOUT - Unhandled ustatus 0x%08x\n", ustatus);
961 /* No sane way found yet -- just reset the bugger. */
962 nv_wr32(dev, 0x400040, 0x80);
963 nv_wr32(dev, 0x400040, 0);
964 nv_wr32(dev, 0x401800, 0xc0000000);
965 nv_wr32(dev, 0x400108, 0x008);
966 status &= ~0x008;
967 }
968
969 /* CCACHE: Handles code and c[] caches and fills them. */
970 if (status & 0x010) {
971 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
972 if (!ustatus && display) {
973 NV_INFO(dev, "PGRAPH_TRAP_CCACHE - no ustatus?\n");
974 }
975 if (ustatus & 0x00000001) {
976 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_CCACHE_FAULT");
977 NV_INFO (dev, "PGRAPH_TRAP_CCACHE_FAULT - %08x %08x %08x %08x %08x %08x %08x\n",
978 nv_rd32(dev, 0x405800),
979 nv_rd32(dev, 0x405804),
980 nv_rd32(dev, 0x405808),
981 nv_rd32(dev, 0x40580c),
982 nv_rd32(dev, 0x405810),
983 nv_rd32(dev, 0x405814),
984 nv_rd32(dev, 0x40581c));
985 ustatus &= ~0x00000001;
986 }
987 if (ustatus && display)
988 NV_INFO(dev, "PGRAPH_TRAP_CCACHE - Unhandled ustatus 0x%08x\n", ustatus);
989 nv_wr32(dev, 0x405018, 0xc0000000);
990 nv_wr32(dev, 0x400108, 0x010);
991 status &= ~0x010;
992 }
993
994 /* Unknown, not seen yet... 0x402000 is the only trap status reg
995 * remaining, so try to handle it anyway. Perhaps related to that
996 * unknown DMA slot on tesla? */
997 if (status & 0x20) {
998 nv50_fb_vm_trap(dev, display, "PGRAPH_TRAP_UNKC04");
999 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
1000 if (display)
1001 NV_INFO(dev, "PGRAPH_TRAP_UNKC04 - Unhandled ustatus 0x%08x\n", ustatus);
1002 nv_wr32(dev, 0x402000, 0xc0000000);
1003 /* no status modifiction on purpose */
1004 }
1005
1006 /* TEXTURE: CUDA texturing units */
1007 if (status & 0x040) {
1008 nv50_pgraph_tp_trap (dev, 6, 0x408900, 0x408600, display,
1009 "PGRAPH_TRAP_TEXTURE");
1010 nv_wr32(dev, 0x400108, 0x040);
1011 status &= ~0x040;
1012 }
1013
1014 /* MP: CUDA execution engines. */
1015 if (status & 0x080) {
1016 nv50_pgraph_tp_trap (dev, 7, 0x408314, 0x40831c, display,
1017 "PGRAPH_TRAP_MP");
1018 nv_wr32(dev, 0x400108, 0x080);
1019 status &= ~0x080;
1020 }
1021
1022 /* TPDMA: Handles TP-initiated uncached memory accesses:
1023 * l[], g[], stack, 2d surfaces, render targets. */
1024 if (status & 0x100) {
1025 nv50_pgraph_tp_trap (dev, 8, 0x408e08, 0x408708, display,
1026 "PGRAPH_TRAP_TPDMA");
1027 nv_wr32(dev, 0x400108, 0x100);
1028 status &= ~0x100;
1029 }
1030
1031 if (status) {
1032 if (display)
1033 NV_INFO(dev, "PGRAPH_TRAP - Unknown trap 0x%08x\n",
1034 status);
1035 nv_wr32(dev, 0x400108, status);
1036 }
1037}
1038
1039/* There must be a *lot* of these. Will take some time to gather them up. */
1040static struct nouveau_enum_names nv50_data_error_names[] =
1041{
1042 { 4, "INVALID_VALUE" },
1043 { 5, "INVALID_ENUM" },
1044 { 8, "INVALID_OBJECT" },
1045 { 0xc, "INVALID_BITFIELD" },
1046 { 0x28, "MP_NO_REG_SPACE" },
1047 { 0x2b, "MP_BLOCK_SIZE_MISMATCH" },
1048};
1049
1050static void
1051nv50_pgraph_irq_handler(struct drm_device *dev)
1052{
1053 struct nouveau_pgraph_trap trap;
1054 int unhandled = 0;
1055 uint32_t status;
1056
1057 while ((status = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1058 /* NOTIFY: You've set a NOTIFY an a command and it's done. */
1059 if (status & 0x00000001) {
1060 nouveau_graph_trap_info(dev, &trap);
1061 if (nouveau_ratelimit())
1062 nouveau_graph_dump_trap_info(dev,
1063 "PGRAPH_NOTIFY", &trap);
1064 status &= ~0x00000001;
1065 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000001);
1066 }
1067
1068 /* COMPUTE_QUERY: Purpose and exact cause unknown, happens
1069 * when you write 0x200 to 0x50c0 method 0x31c. */
1070 if (status & 0x00000002) {
1071 nouveau_graph_trap_info(dev, &trap);
1072 if (nouveau_ratelimit())
1073 nouveau_graph_dump_trap_info(dev,
1074 "PGRAPH_COMPUTE_QUERY", &trap);
1075 status &= ~0x00000002;
1076 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000002);
1077 }
1078
1079 /* Unknown, never seen: 0x4 */
1080
1081 /* ILLEGAL_MTHD: You used a wrong method for this class. */
1082 if (status & 0x00000010) {
1083 nouveau_graph_trap_info(dev, &trap);
1084 if (nouveau_pgraph_intr_swmthd(dev, &trap))
1085 unhandled = 1;
1086 if (unhandled && nouveau_ratelimit())
1087 nouveau_graph_dump_trap_info(dev,
1088 "PGRAPH_ILLEGAL_MTHD", &trap);
1089 status &= ~0x00000010;
1090 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000010);
1091 }
1092
1093 /* ILLEGAL_CLASS: You used a wrong class. */
1094 if (status & 0x00000020) {
1095 nouveau_graph_trap_info(dev, &trap);
1096 if (nouveau_ratelimit())
1097 nouveau_graph_dump_trap_info(dev,
1098 "PGRAPH_ILLEGAL_CLASS", &trap);
1099 status &= ~0x00000020;
1100 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000020);
1101 }
1102
1103 /* DOUBLE_NOTIFY: You tried to set a NOTIFY on another NOTIFY. */
1104 if (status & 0x00000040) {
1105 nouveau_graph_trap_info(dev, &trap);
1106 if (nouveau_ratelimit())
1107 nouveau_graph_dump_trap_info(dev,
1108 "PGRAPH_DOUBLE_NOTIFY", &trap);
1109 status &= ~0x00000040;
1110 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00000040);
1111 }
1112
1113 /* CONTEXT_SWITCH: PGRAPH needs us to load a new context */
1114 if (status & 0x00001000) {
1115 nv_wr32(dev, 0x400500, 0x00000000);
1116 nv_wr32(dev, NV03_PGRAPH_INTR,
1117 NV_PGRAPH_INTR_CONTEXT_SWITCH);
1118 nv_wr32(dev, NV40_PGRAPH_INTR_EN, nv_rd32(dev,
1119 NV40_PGRAPH_INTR_EN) &
1120 ~NV_PGRAPH_INTR_CONTEXT_SWITCH);
1121 nv_wr32(dev, 0x400500, 0x00010001);
1122
1123 nv50_graph_context_switch(dev);
1124
1125 status &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1126 }
1127
1128 /* BUFFER_NOTIFY: Your m2mf transfer finished */
1129 if (status & 0x00010000) {
1130 nouveau_graph_trap_info(dev, &trap);
1131 if (nouveau_ratelimit())
1132 nouveau_graph_dump_trap_info(dev,
1133 "PGRAPH_BUFFER_NOTIFY", &trap);
1134 status &= ~0x00010000;
1135 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00010000);
1136 }
1137
1138 /* DATA_ERROR: Invalid value for this method, or invalid
1139 * state in current PGRAPH context for this operation */
1140 if (status & 0x00100000) {
1141 nouveau_graph_trap_info(dev, &trap);
1142 if (nouveau_ratelimit()) {
1143 nouveau_graph_dump_trap_info(dev,
1144 "PGRAPH_DATA_ERROR", &trap);
1145 NV_INFO (dev, "PGRAPH_DATA_ERROR - ");
1146 nouveau_print_enum_names(nv_rd32(dev, 0x400110),
1147 nv50_data_error_names);
1148 printk("\n");
1149 }
1150 status &= ~0x00100000;
1151 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00100000);
1152 }
1153 124
1154 /* TRAP: Something bad happened in the middle of command 125 drm_irq_uninstall(dev);
1155 * execution. Has a billion types, subtypes, and even 126 if (dev_priv->msi_enabled)
1156 * subsubtypes. */ 127 pci_disable_msi(dev->pdev);
1157 if (status & 0x00200000) {
1158 nv50_pgraph_trap_handler(dev);
1159 status &= ~0x00200000;
1160 nv_wr32(dev, NV03_PGRAPH_INTR, 0x00200000);
1161 }
1162
1163 /* Unknown, never seen: 0x00400000 */
1164
1165 /* SINGLE_STEP: Happens on every method if you turned on
1166 * single stepping in 40008c */
1167 if (status & 0x01000000) {
1168 nouveau_graph_trap_info(dev, &trap);
1169 if (nouveau_ratelimit())
1170 nouveau_graph_dump_trap_info(dev,
1171 "PGRAPH_SINGLE_STEP", &trap);
1172 status &= ~0x01000000;
1173 nv_wr32(dev, NV03_PGRAPH_INTR, 0x01000000);
1174 }
1175
1176 /* 0x02000000 happens when you pause a ctxprog...
1177 * but the only way this can happen that I know is by
1178 * poking the relevant MMIO register, and we don't
1179 * do that. */
1180
1181 if (status) {
1182 NV_INFO(dev, "Unhandled PGRAPH_INTR - 0x%08x\n",
1183 status);
1184 nv_wr32(dev, NV03_PGRAPH_INTR, status);
1185 }
1186
1187 {
1188 const int isb = (1 << 16) | (1 << 0);
1189
1190 if ((nv_rd32(dev, 0x400500) & isb) != isb)
1191 nv_wr32(dev, 0x400500,
1192 nv_rd32(dev, 0x400500) | isb);
1193 }
1194 }
1195
1196 nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PGRAPH_PENDING);
1197 if (nv_rd32(dev, 0x400824) & (1 << 31))
1198 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1199} 128}
1200 129
1201static void 130void
1202nouveau_crtc_irq_handler(struct drm_device *dev, int crtc) 131nouveau_irq_register(struct drm_device *dev, int status_bit,
132 void (*handler)(struct drm_device *))
1203{ 133{
1204 if (crtc & 1) 134 struct drm_nouveau_private *dev_priv = dev->dev_private;
1205 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK); 135 unsigned long flags;
1206 136
1207 if (crtc & 2) 137 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1208 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK); 138 dev_priv->irq_handler[status_bit] = handler;
139 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1209} 140}
1210 141
1211irqreturn_t 142void
1212nouveau_irq_handler(DRM_IRQ_ARGS) 143nouveau_irq_unregister(struct drm_device *dev, int status_bit)
1213{ 144{
1214 struct drm_device *dev = (struct drm_device *)arg;
1215 struct drm_nouveau_private *dev_priv = dev->dev_private; 145 struct drm_nouveau_private *dev_priv = dev->dev_private;
1216 uint32_t status;
1217 unsigned long flags; 146 unsigned long flags;
1218 147
1219 status = nv_rd32(dev, NV03_PMC_INTR_0);
1220 if (!status)
1221 return IRQ_NONE;
1222
1223 spin_lock_irqsave(&dev_priv->context_switch_lock, flags); 148 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
1224 149 dev_priv->irq_handler[status_bit] = NULL;
1225 if (status & NV_PMC_INTR_0_PFIFO_PENDING) {
1226 nouveau_fifo_irq_handler(dev);
1227 status &= ~NV_PMC_INTR_0_PFIFO_PENDING;
1228 }
1229
1230 if (status & NV_PMC_INTR_0_PGRAPH_PENDING) {
1231 if (dev_priv->card_type >= NV_50)
1232 nv50_pgraph_irq_handler(dev);
1233 else
1234 nouveau_pgraph_irq_handler(dev);
1235
1236 status &= ~NV_PMC_INTR_0_PGRAPH_PENDING;
1237 }
1238
1239 if (status & NV_PMC_INTR_0_CRTCn_PENDING) {
1240 nouveau_crtc_irq_handler(dev, (status>>24)&3);
1241 status &= ~NV_PMC_INTR_0_CRTCn_PENDING;
1242 }
1243
1244 if (status & (NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1245 NV_PMC_INTR_0_NV50_I2C_PENDING)) {
1246 nv50_display_irq_handler(dev);
1247 status &= ~(NV_PMC_INTR_0_NV50_DISPLAY_PENDING |
1248 NV_PMC_INTR_0_NV50_I2C_PENDING);
1249 }
1250
1251 if (status)
1252 NV_ERROR(dev, "Unhandled PMC INTR status bits 0x%08x\n", status);
1253
1254 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags); 150 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
1255
1256 return IRQ_HANDLED;
1257} 151}
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index fe4a30dc4b42..69044eb104bb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -36,183 +36,112 @@
36 36
37#include "nouveau_drv.h" 37#include "nouveau_drv.h"
38#include "nouveau_pm.h" 38#include "nouveau_pm.h"
39#include "nouveau_mm.h"
40#include "nouveau_vm.h"
39 41
40/* 42/*
41 * NV10-NV40 tiling helpers 43 * NV10-NV40 tiling helpers
42 */ 44 */
43 45
44static void 46static void
45nv10_mem_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 47nv10_mem_update_tile_region(struct drm_device *dev,
46 uint32_t size, uint32_t pitch) 48 struct nouveau_tile_reg *tile, uint32_t addr,
49 uint32_t size, uint32_t pitch, uint32_t flags)
47{ 50{
48 struct drm_nouveau_private *dev_priv = dev->dev_private; 51 struct drm_nouveau_private *dev_priv = dev->dev_private;
49 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo; 52 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
50 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb; 53 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
51 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph; 54 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
52 struct nouveau_tile_reg *tile = &dev_priv->tile[i]; 55 int i = tile - dev_priv->tile.reg;
56 unsigned long save;
53 57
54 tile->addr = addr; 58 nouveau_fence_unref(&tile->fence);
55 tile->size = size;
56 tile->used = !!pitch;
57 nouveau_fence_unref((void **)&tile->fence);
58 59
60 if (tile->pitch)
61 pfb->free_tile_region(dev, i);
62
63 if (pitch)
64 pfb->init_tile_region(dev, i, addr, size, pitch, flags);
65
66 spin_lock_irqsave(&dev_priv->context_switch_lock, save);
59 pfifo->reassign(dev, false); 67 pfifo->reassign(dev, false);
60 pfifo->cache_pull(dev, false); 68 pfifo->cache_pull(dev, false);
61 69
62 nouveau_wait_for_idle(dev); 70 nouveau_wait_for_idle(dev);
63 71
64 pgraph->set_region_tiling(dev, i, addr, size, pitch); 72 pfb->set_tile_region(dev, i);
65 pfb->set_region_tiling(dev, i, addr, size, pitch); 73 pgraph->set_tile_region(dev, i);
66 74
67 pfifo->cache_pull(dev, true); 75 pfifo->cache_pull(dev, true);
68 pfifo->reassign(dev, true); 76 pfifo->reassign(dev, true);
77 spin_unlock_irqrestore(&dev_priv->context_switch_lock, save);
69} 78}
70 79
71struct nouveau_tile_reg * 80static struct nouveau_tile_reg *
72nv10_mem_set_tiling(struct drm_device *dev, uint32_t addr, uint32_t size, 81nv10_mem_get_tile_region(struct drm_device *dev, int i)
73 uint32_t pitch)
74{ 82{
75 struct drm_nouveau_private *dev_priv = dev->dev_private; 83 struct drm_nouveau_private *dev_priv = dev->dev_private;
76 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb; 84 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
77 struct nouveau_tile_reg *found = NULL;
78 unsigned long i, flags;
79
80 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
81
82 for (i = 0; i < pfb->num_tiles; i++) {
83 struct nouveau_tile_reg *tile = &dev_priv->tile[i];
84
85 if (tile->used)
86 /* Tile region in use. */
87 continue;
88 85
89 if (tile->fence && 86 spin_lock(&dev_priv->tile.lock);
90 !nouveau_fence_signalled(tile->fence, NULL))
91 /* Pending tile region. */
92 continue;
93
94 if (max(tile->addr, addr) <
95 min(tile->addr + tile->size, addr + size))
96 /* Kill an intersecting tile region. */
97 nv10_mem_set_region_tiling(dev, i, 0, 0, 0);
98
99 if (pitch && !found) {
100 /* Free tile region. */
101 nv10_mem_set_region_tiling(dev, i, addr, size, pitch);
102 found = tile;
103 }
104 }
105 87
106 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags); 88 if (!tile->used &&
89 (!tile->fence || nouveau_fence_signalled(tile->fence)))
90 tile->used = true;
91 else
92 tile = NULL;
107 93
108 return found; 94 spin_unlock(&dev_priv->tile.lock);
95 return tile;
109} 96}
110 97
111void 98void
112nv10_mem_expire_tiling(struct drm_device *dev, struct nouveau_tile_reg *tile, 99nv10_mem_put_tile_region(struct drm_device *dev, struct nouveau_tile_reg *tile,
113 struct nouveau_fence *fence) 100 struct nouveau_fence *fence)
114{
115 if (fence) {
116 /* Mark it as pending. */
117 tile->fence = fence;
118 nouveau_fence_ref(fence);
119 }
120
121 tile->used = false;
122}
123
124/*
125 * NV50 VM helpers
126 */
127int
128nv50_mem_vm_bind_linear(struct drm_device *dev, uint64_t virt, uint32_t size,
129 uint32_t flags, uint64_t phys)
130{ 101{
131 struct drm_nouveau_private *dev_priv = dev->dev_private; 102 struct drm_nouveau_private *dev_priv = dev->dev_private;
132 struct nouveau_gpuobj *pgt;
133 unsigned block;
134 int i;
135 103
136 virt = ((virt - dev_priv->vm_vram_base) >> 16) << 1; 104 if (tile) {
137 size = (size >> 16) << 1; 105 spin_lock(&dev_priv->tile.lock);
138 106 if (fence) {
139 phys |= ((uint64_t)flags << 32); 107 /* Mark it as pending. */
140 phys |= 1; 108 tile->fence = fence;
141 if (dev_priv->vram_sys_base) { 109 nouveau_fence_ref(fence);
142 phys += dev_priv->vram_sys_base;
143 phys |= 0x30;
144 }
145
146 while (size) {
147 unsigned offset_h = upper_32_bits(phys);
148 unsigned offset_l = lower_32_bits(phys);
149 unsigned pte, end;
150
151 for (i = 7; i >= 0; i--) {
152 block = 1 << (i + 1);
153 if (size >= block && !(virt & (block - 1)))
154 break;
155 } 110 }
156 offset_l |= (i << 7);
157
158 phys += block << 15;
159 size -= block;
160
161 while (block) {
162 pgt = dev_priv->vm_vram_pt[virt >> 14];
163 pte = virt & 0x3ffe;
164
165 end = pte + block;
166 if (end > 16384)
167 end = 16384;
168 block -= (end - pte);
169 virt += (end - pte);
170
171 while (pte < end) {
172 nv_wo32(pgt, (pte * 4) + 0, offset_l);
173 nv_wo32(pgt, (pte * 4) + 4, offset_h);
174 pte += 2;
175 }
176 }
177 }
178 111
179 dev_priv->engine.instmem.flush(dev); 112 tile->used = false;
180 dev_priv->engine.fifo.tlb_flush(dev); 113 spin_unlock(&dev_priv->tile.lock);
181 dev_priv->engine.graph.tlb_flush(dev); 114 }
182 nv50_vm_flush(dev, 6);
183 return 0;
184} 115}
185 116
186void 117struct nouveau_tile_reg *
187nv50_mem_vm_unbind(struct drm_device *dev, uint64_t virt, uint32_t size) 118nv10_mem_set_tiling(struct drm_device *dev, uint32_t addr, uint32_t size,
119 uint32_t pitch, uint32_t flags)
188{ 120{
189 struct drm_nouveau_private *dev_priv = dev->dev_private; 121 struct drm_nouveau_private *dev_priv = dev->dev_private;
190 struct nouveau_gpuobj *pgt; 122 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
191 unsigned pages, pte, end; 123 struct nouveau_tile_reg *tile, *found = NULL;
192 124 int i;
193 virt -= dev_priv->vm_vram_base;
194 pages = (size >> 16) << 1;
195 125
196 while (pages) { 126 for (i = 0; i < pfb->num_tiles; i++) {
197 pgt = dev_priv->vm_vram_pt[virt >> 29]; 127 tile = nv10_mem_get_tile_region(dev, i);
198 pte = (virt & 0x1ffe0000ULL) >> 15;
199 128
200 end = pte + pages; 129 if (pitch && !found) {
201 if (end > 16384) 130 found = tile;
202 end = 16384; 131 continue;
203 pages -= (end - pte);
204 virt += (end - pte) << 15;
205 132
206 while (pte < end) { 133 } else if (tile && tile->pitch) {
207 nv_wo32(pgt, (pte * 4), 0); 134 /* Kill an unused tile region. */
208 pte++; 135 nv10_mem_update_tile_region(dev, tile, 0, 0, 0, 0);
209 } 136 }
137
138 nv10_mem_put_tile_region(dev, tile, NULL);
210 } 139 }
211 140
212 dev_priv->engine.instmem.flush(dev); 141 if (found)
213 dev_priv->engine.fifo.tlb_flush(dev); 142 nv10_mem_update_tile_region(dev, found, addr, size,
214 dev_priv->engine.graph.tlb_flush(dev); 143 pitch, flags);
215 nv50_vm_flush(dev, 6); 144 return found;
216} 145}
217 146
218/* 147/*
@@ -312,62 +241,7 @@ nouveau_mem_detect_nforce(struct drm_device *dev)
312 return 0; 241 return 0;
313} 242}
314 243
315static void 244int
316nv50_vram_preinit(struct drm_device *dev)
317{
318 struct drm_nouveau_private *dev_priv = dev->dev_private;
319 int i, parts, colbits, rowbitsa, rowbitsb, banks;
320 u64 rowsize, predicted;
321 u32 r0, r4, rt, ru;
322
323 r0 = nv_rd32(dev, 0x100200);
324 r4 = nv_rd32(dev, 0x100204);
325 rt = nv_rd32(dev, 0x100250);
326 ru = nv_rd32(dev, 0x001540);
327 NV_DEBUG(dev, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n", r0, r4, rt, ru);
328
329 for (i = 0, parts = 0; i < 8; i++) {
330 if (ru & (0x00010000 << i))
331 parts++;
332 }
333
334 colbits = (r4 & 0x0000f000) >> 12;
335 rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
336 rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
337 banks = ((r4 & 0x01000000) ? 8 : 4);
338
339 rowsize = parts * banks * (1 << colbits) * 8;
340 predicted = rowsize << rowbitsa;
341 if (r0 & 0x00000004)
342 predicted += rowsize << rowbitsb;
343
344 if (predicted != dev_priv->vram_size) {
345 NV_WARN(dev, "memory controller reports %dMiB VRAM\n",
346 (u32)(dev_priv->vram_size >> 20));
347 NV_WARN(dev, "we calculated %dMiB VRAM\n",
348 (u32)(predicted >> 20));
349 }
350
351 dev_priv->vram_rblock_size = rowsize >> 12;
352 if (rt & 1)
353 dev_priv->vram_rblock_size *= 3;
354
355 NV_DEBUG(dev, "rblock %lld bytes\n",
356 (u64)dev_priv->vram_rblock_size << 12);
357}
358
359static void
360nvaa_vram_preinit(struct drm_device *dev)
361{
362 struct drm_nouveau_private *dev_priv = dev->dev_private;
363
364 /* To our knowledge, there's no large scale reordering of pages
365 * that occurs on IGP chipsets.
366 */
367 dev_priv->vram_rblock_size = 1;
368}
369
370static int
371nouveau_mem_detect(struct drm_device *dev) 245nouveau_mem_detect(struct drm_device *dev)
372{ 246{
373 struct drm_nouveau_private *dev_priv = dev->dev_private; 247 struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -381,33 +255,6 @@ nouveau_mem_detect(struct drm_device *dev)
381 if (dev_priv->card_type < NV_50) { 255 if (dev_priv->card_type < NV_50) {
382 dev_priv->vram_size = nv_rd32(dev, NV04_PFB_FIFO_DATA); 256 dev_priv->vram_size = nv_rd32(dev, NV04_PFB_FIFO_DATA);
383 dev_priv->vram_size &= NV10_PFB_FIFO_DATA_RAM_AMOUNT_MB_MASK; 257 dev_priv->vram_size &= NV10_PFB_FIFO_DATA_RAM_AMOUNT_MB_MASK;
384 } else
385 if (dev_priv->card_type < NV_C0) {
386 dev_priv->vram_size = nv_rd32(dev, NV04_PFB_FIFO_DATA);
387 dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
388 dev_priv->vram_size &= 0xffffffff00ll;
389
390 switch (dev_priv->chipset) {
391 case 0xaa:
392 case 0xac:
393 case 0xaf:
394 dev_priv->vram_sys_base = nv_rd32(dev, 0x100e10);
395 dev_priv->vram_sys_base <<= 12;
396 nvaa_vram_preinit(dev);
397 break;
398 default:
399 nv50_vram_preinit(dev);
400 break;
401 }
402 } else {
403 dev_priv->vram_size = nv_rd32(dev, 0x10f20c) << 20;
404 dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
405 }
406
407 NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
408 if (dev_priv->vram_sys_base) {
409 NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
410 dev_priv->vram_sys_base);
411 } 258 }
412 259
413 if (dev_priv->vram_size) 260 if (dev_priv->vram_size)
@@ -415,6 +262,15 @@ nouveau_mem_detect(struct drm_device *dev)
415 return -ENOMEM; 262 return -ENOMEM;
416} 263}
417 264
265bool
266nouveau_mem_flags_valid(struct drm_device *dev, u32 tile_flags)
267{
268 if (!(tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK))
269 return true;
270
271 return false;
272}
273
418#if __OS_HAS_AGP 274#if __OS_HAS_AGP
419static unsigned long 275static unsigned long
420get_agp_mode(struct drm_device *dev, unsigned long mode) 276get_agp_mode(struct drm_device *dev, unsigned long mode)
@@ -547,10 +403,6 @@ nouveau_mem_vram_init(struct drm_device *dev)
547 if (ret) 403 if (ret)
548 return ret; 404 return ret;
549 405
550 ret = nouveau_mem_detect(dev);
551 if (ret)
552 return ret;
553
554 dev_priv->fb_phys = pci_resource_start(dev->pdev, 1); 406 dev_priv->fb_phys = pci_resource_start(dev->pdev, 1);
555 407
556 ret = nouveau_ttm_global_init(dev_priv); 408 ret = nouveau_ttm_global_init(dev_priv);
@@ -566,13 +418,6 @@ nouveau_mem_vram_init(struct drm_device *dev)
566 return ret; 418 return ret;
567 } 419 }
568 420
569 dev_priv->fb_available_size = dev_priv->vram_size;
570 dev_priv->fb_mappable_pages = dev_priv->fb_available_size;
571 if (dev_priv->fb_mappable_pages > pci_resource_len(dev->pdev, 1))
572 dev_priv->fb_mappable_pages =
573 pci_resource_len(dev->pdev, 1);
574 dev_priv->fb_mappable_pages >>= PAGE_SHIFT;
575
576 /* reserve space at end of VRAM for PRAMIN */ 421 /* reserve space at end of VRAM for PRAMIN */
577 if (dev_priv->chipset == 0x40 || dev_priv->chipset == 0x47 || 422 if (dev_priv->chipset == 0x40 || dev_priv->chipset == 0x47 ||
578 dev_priv->chipset == 0x49 || dev_priv->chipset == 0x4b) 423 dev_priv->chipset == 0x49 || dev_priv->chipset == 0x4b)
@@ -583,6 +428,22 @@ nouveau_mem_vram_init(struct drm_device *dev)
583 else 428 else
584 dev_priv->ramin_rsvd_vram = (512 * 1024); 429 dev_priv->ramin_rsvd_vram = (512 * 1024);
585 430
431 ret = dev_priv->engine.vram.init(dev);
432 if (ret)
433 return ret;
434
435 NV_INFO(dev, "Detected %dMiB VRAM\n", (int)(dev_priv->vram_size >> 20));
436 if (dev_priv->vram_sys_base) {
437 NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
438 dev_priv->vram_sys_base);
439 }
440
441 dev_priv->fb_available_size = dev_priv->vram_size;
442 dev_priv->fb_mappable_pages = dev_priv->fb_available_size;
443 if (dev_priv->fb_mappable_pages > pci_resource_len(dev->pdev, 1))
444 dev_priv->fb_mappable_pages = pci_resource_len(dev->pdev, 1);
445 dev_priv->fb_mappable_pages >>= PAGE_SHIFT;
446
586 dev_priv->fb_available_size -= dev_priv->ramin_rsvd_vram; 447 dev_priv->fb_available_size -= dev_priv->ramin_rsvd_vram;
587 dev_priv->fb_aper_free = dev_priv->fb_available_size; 448 dev_priv->fb_aper_free = dev_priv->fb_available_size;
588 449
@@ -799,3 +660,118 @@ nouveau_mem_timing_fini(struct drm_device *dev)
799 660
800 kfree(mem->timing); 661 kfree(mem->timing);
801} 662}
663
664static int
665nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long p_size)
666{
667 struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
668 struct nouveau_mm *mm;
669 u32 b_size;
670 int ret;
671
672 p_size = (p_size << PAGE_SHIFT) >> 12;
673 b_size = dev_priv->vram_rblock_size >> 12;
674
675 ret = nouveau_mm_init(&mm, 0, p_size, b_size);
676 if (ret)
677 return ret;
678
679 man->priv = mm;
680 return 0;
681}
682
683static int
684nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
685{
686 struct nouveau_mm *mm = man->priv;
687 int ret;
688
689 ret = nouveau_mm_fini(&mm);
690 if (ret)
691 return ret;
692
693 man->priv = NULL;
694 return 0;
695}
696
697static void
698nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
699 struct ttm_mem_reg *mem)
700{
701 struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
702 struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
703 struct drm_device *dev = dev_priv->dev;
704
705 vram->put(dev, (struct nouveau_vram **)&mem->mm_node);
706}
707
708static int
709nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
710 struct ttm_buffer_object *bo,
711 struct ttm_placement *placement,
712 struct ttm_mem_reg *mem)
713{
714 struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
715 struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
716 struct drm_device *dev = dev_priv->dev;
717 struct nouveau_bo *nvbo = nouveau_bo(bo);
718 struct nouveau_vram *node;
719 u32 size_nc = 0;
720 int ret;
721
722 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
723 size_nc = 1 << nvbo->vma.node->type;
724
725 ret = vram->get(dev, mem->num_pages << PAGE_SHIFT,
726 mem->page_alignment << PAGE_SHIFT, size_nc,
727 (nvbo->tile_flags >> 8) & 0xff, &node);
728 if (ret)
729 return ret;
730
731 node->page_shift = 12;
732 if (nvbo->vma.node)
733 node->page_shift = nvbo->vma.node->type;
734
735 mem->mm_node = node;
736 mem->start = node->offset >> PAGE_SHIFT;
737 return 0;
738}
739
740void
741nouveau_vram_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
742{
743 struct nouveau_mm *mm = man->priv;
744 struct nouveau_mm_node *r;
745 u64 total = 0, ttotal[3] = {}, tused[3] = {}, tfree[3] = {};
746 int i;
747
748 mutex_lock(&mm->mutex);
749 list_for_each_entry(r, &mm->nodes, nl_entry) {
750 printk(KERN_DEBUG "%s %s-%d: 0x%010llx 0x%010llx\n",
751 prefix, r->free ? "free" : "used", r->type,
752 ((u64)r->offset << 12),
753 (((u64)r->offset + r->length) << 12));
754 total += r->length;
755 ttotal[r->type] += r->length;
756 if (r->free)
757 tfree[r->type] += r->length;
758 else
759 tused[r->type] += r->length;
760 }
761 mutex_unlock(&mm->mutex);
762
763 printk(KERN_DEBUG "%s total: 0x%010llx\n", prefix, total << 12);
764 for (i = 0; i < 3; i++) {
765 printk(KERN_DEBUG "%s type %d: 0x%010llx, "
766 "used 0x%010llx, free 0x%010llx\n", prefix,
767 i, ttotal[i] << 12, tused[i] << 12, tfree[i] << 12);
768 }
769}
770
771const struct ttm_mem_type_manager_func nouveau_vram_manager = {
772 nouveau_vram_manager_init,
773 nouveau_vram_manager_fini,
774 nouveau_vram_manager_new,
775 nouveau_vram_manager_del,
776 nouveau_vram_manager_debug
777};
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.c b/drivers/gpu/drm/nouveau/nouveau_mm.c
new file mode 100644
index 000000000000..cdbb11eb701b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.c
@@ -0,0 +1,271 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_mm.h"
28
29static inline void
30region_put(struct nouveau_mm *rmm, struct nouveau_mm_node *a)
31{
32 list_del(&a->nl_entry);
33 list_del(&a->fl_entry);
34 kfree(a);
35}
36
37static struct nouveau_mm_node *
38region_split(struct nouveau_mm *rmm, struct nouveau_mm_node *a, u32 size)
39{
40 struct nouveau_mm_node *b;
41
42 if (a->length == size)
43 return a;
44
45 b = kmalloc(sizeof(*b), GFP_KERNEL);
46 if (unlikely(b == NULL))
47 return NULL;
48
49 b->offset = a->offset;
50 b->length = size;
51 b->free = a->free;
52 b->type = a->type;
53 a->offset += size;
54 a->length -= size;
55 list_add_tail(&b->nl_entry, &a->nl_entry);
56 if (b->free)
57 list_add_tail(&b->fl_entry, &a->fl_entry);
58 return b;
59}
60
61static struct nouveau_mm_node *
62nouveau_mm_merge(struct nouveau_mm *rmm, struct nouveau_mm_node *this)
63{
64 struct nouveau_mm_node *prev, *next;
65
66 /* try to merge with free adjacent entries of same type */
67 prev = list_entry(this->nl_entry.prev, struct nouveau_mm_node, nl_entry);
68 if (this->nl_entry.prev != &rmm->nodes) {
69 if (prev->free && prev->type == this->type) {
70 prev->length += this->length;
71 region_put(rmm, this);
72 this = prev;
73 }
74 }
75
76 next = list_entry(this->nl_entry.next, struct nouveau_mm_node, nl_entry);
77 if (this->nl_entry.next != &rmm->nodes) {
78 if (next->free && next->type == this->type) {
79 next->offset = this->offset;
80 next->length += this->length;
81 region_put(rmm, this);
82 this = next;
83 }
84 }
85
86 return this;
87}
88
89void
90nouveau_mm_put(struct nouveau_mm *rmm, struct nouveau_mm_node *this)
91{
92 u32 block_s, block_l;
93
94 this->free = true;
95 list_add(&this->fl_entry, &rmm->free);
96 this = nouveau_mm_merge(rmm, this);
97
98 /* any entirely free blocks now? we'll want to remove typing
99 * on them now so they can be use for any memory allocation
100 */
101 block_s = roundup(this->offset, rmm->block_size);
102 if (block_s + rmm->block_size > this->offset + this->length)
103 return;
104
105 /* split off any still-typed region at the start */
106 if (block_s != this->offset) {
107 if (!region_split(rmm, this, block_s - this->offset))
108 return;
109 }
110
111 /* split off the soon-to-be-untyped block(s) */
112 block_l = rounddown(this->length, rmm->block_size);
113 if (block_l != this->length) {
114 this = region_split(rmm, this, block_l);
115 if (!this)
116 return;
117 }
118
119 /* mark as having no type, and retry merge with any adjacent
120 * untyped blocks
121 */
122 this->type = 0;
123 nouveau_mm_merge(rmm, this);
124}
125
126int
127nouveau_mm_get(struct nouveau_mm *rmm, int type, u32 size, u32 size_nc,
128 u32 align, struct nouveau_mm_node **pnode)
129{
130 struct nouveau_mm_node *this, *tmp, *next;
131 u32 splitoff, avail, alloc;
132
133 list_for_each_entry_safe(this, tmp, &rmm->free, fl_entry) {
134 next = list_entry(this->nl_entry.next, struct nouveau_mm_node, nl_entry);
135 if (this->nl_entry.next == &rmm->nodes)
136 next = NULL;
137
138 /* skip wrongly typed blocks */
139 if (this->type && this->type != type)
140 continue;
141
142 /* account for alignment */
143 splitoff = this->offset & (align - 1);
144 if (splitoff)
145 splitoff = align - splitoff;
146
147 if (this->length <= splitoff)
148 continue;
149
150 /* determine total memory available from this, and
151 * the next block (if appropriate)
152 */
153 avail = this->length;
154 if (next && next->free && (!next->type || next->type == type))
155 avail += next->length;
156
157 avail -= splitoff;
158
159 /* determine allocation size */
160 if (size_nc) {
161 alloc = min(avail, size);
162 alloc = rounddown(alloc, size_nc);
163 if (alloc == 0)
164 continue;
165 } else {
166 alloc = size;
167 if (avail < alloc)
168 continue;
169 }
170
171 /* untyped block, split off a chunk that's a multiple
172 * of block_size and type it
173 */
174 if (!this->type) {
175 u32 block = roundup(alloc + splitoff, rmm->block_size);
176 if (this->length < block)
177 continue;
178
179 this = region_split(rmm, this, block);
180 if (!this)
181 return -ENOMEM;
182
183 this->type = type;
184 }
185
186 /* stealing memory from adjacent block */
187 if (alloc > this->length) {
188 u32 amount = alloc - (this->length - splitoff);
189
190 if (!next->type) {
191 amount = roundup(amount, rmm->block_size);
192
193 next = region_split(rmm, next, amount);
194 if (!next)
195 return -ENOMEM;
196
197 next->type = type;
198 }
199
200 this->length += amount;
201 next->offset += amount;
202 next->length -= amount;
203 if (!next->length) {
204 list_del(&next->nl_entry);
205 list_del(&next->fl_entry);
206 kfree(next);
207 }
208 }
209
210 if (splitoff) {
211 if (!region_split(rmm, this, splitoff))
212 return -ENOMEM;
213 }
214
215 this = region_split(rmm, this, alloc);
216 if (this == NULL)
217 return -ENOMEM;
218
219 this->free = false;
220 list_del(&this->fl_entry);
221 *pnode = this;
222 return 0;
223 }
224
225 return -ENOMEM;
226}
227
228int
229nouveau_mm_init(struct nouveau_mm **prmm, u32 offset, u32 length, u32 block)
230{
231 struct nouveau_mm *rmm;
232 struct nouveau_mm_node *heap;
233
234 heap = kzalloc(sizeof(*heap), GFP_KERNEL);
235 if (!heap)
236 return -ENOMEM;
237 heap->free = true;
238 heap->offset = roundup(offset, block);
239 heap->length = rounddown(offset + length, block) - heap->offset;
240
241 rmm = kzalloc(sizeof(*rmm), GFP_KERNEL);
242 if (!rmm) {
243 kfree(heap);
244 return -ENOMEM;
245 }
246 rmm->block_size = block;
247 mutex_init(&rmm->mutex);
248 INIT_LIST_HEAD(&rmm->nodes);
249 INIT_LIST_HEAD(&rmm->free);
250 list_add(&heap->nl_entry, &rmm->nodes);
251 list_add(&heap->fl_entry, &rmm->free);
252
253 *prmm = rmm;
254 return 0;
255}
256
257int
258nouveau_mm_fini(struct nouveau_mm **prmm)
259{
260 struct nouveau_mm *rmm = *prmm;
261 struct nouveau_mm_node *heap =
262 list_first_entry(&rmm->nodes, struct nouveau_mm_node, nl_entry);
263
264 if (!list_is_singular(&rmm->nodes))
265 return -EBUSY;
266
267 kfree(heap);
268 kfree(rmm);
269 *prmm = NULL;
270 return 0;
271}
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.h b/drivers/gpu/drm/nouveau/nouveau_mm.h
new file mode 100644
index 000000000000..af3844933036
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_mm.h
@@ -0,0 +1,67 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#ifndef __NOUVEAU_REGION_H__
26#define __NOUVEAU_REGION_H__
27
28struct nouveau_mm_node {
29 struct list_head nl_entry;
30 struct list_head fl_entry;
31 struct list_head rl_entry;
32
33 bool free;
34 int type;
35
36 u32 offset;
37 u32 length;
38};
39
40struct nouveau_mm {
41 struct list_head nodes;
42 struct list_head free;
43
44 struct mutex mutex;
45
46 u32 block_size;
47};
48
49int nouveau_mm_init(struct nouveau_mm **, u32 offset, u32 length, u32 block);
50int nouveau_mm_fini(struct nouveau_mm **);
51int nouveau_mm_pre(struct nouveau_mm *);
52int nouveau_mm_get(struct nouveau_mm *, int type, u32 size, u32 size_nc,
53 u32 align, struct nouveau_mm_node **);
54void nouveau_mm_put(struct nouveau_mm *, struct nouveau_mm_node *);
55
56int nv50_vram_init(struct drm_device *);
57int nv50_vram_new(struct drm_device *, u64 size, u32 align, u32 size_nc,
58 u32 memtype, struct nouveau_vram **);
59void nv50_vram_del(struct drm_device *, struct nouveau_vram **);
60bool nv50_vram_flags_valid(struct drm_device *, u32 tile_flags);
61
62int nvc0_vram_init(struct drm_device *);
63int nvc0_vram_new(struct drm_device *, u64 size, u32 align, u32 ncmin,
64 u32 memtype, struct nouveau_vram **);
65bool nvc0_vram_flags_valid(struct drm_device *, u32 tile_flags);
66
67#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
index 2cc59f8c658b..fe29d604b820 100644
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ b/drivers/gpu/drm/nouveau/nouveau_notifier.c
@@ -99,7 +99,6 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
99 int size, uint32_t *b_offset) 99 int size, uint32_t *b_offset)
100{ 100{
101 struct drm_device *dev = chan->dev; 101 struct drm_device *dev = chan->dev;
102 struct drm_nouveau_private *dev_priv = dev->dev_private;
103 struct nouveau_gpuobj *nobj = NULL; 102 struct nouveau_gpuobj *nobj = NULL;
104 struct drm_mm_node *mem; 103 struct drm_mm_node *mem;
105 uint32_t offset; 104 uint32_t offset;
@@ -113,31 +112,15 @@ nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
113 return -ENOMEM; 112 return -ENOMEM;
114 } 113 }
115 114
116 offset = chan->notifier_bo->bo.mem.start << PAGE_SHIFT; 115 if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM)
117 if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM) { 116 target = NV_MEM_TARGET_VRAM;
118 target = NV_DMA_TARGET_VIDMEM; 117 else
119 } else 118 target = NV_MEM_TARGET_GART;
120 if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_TT) { 119 offset = chan->notifier_bo->bo.mem.start << PAGE_SHIFT;
121 if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA &&
122 dev_priv->card_type < NV_50) {
123 ret = nouveau_sgdma_get_page(dev, offset, &offset);
124 if (ret)
125 return ret;
126 target = NV_DMA_TARGET_PCI;
127 } else {
128 target = NV_DMA_TARGET_AGP;
129 if (dev_priv->card_type >= NV_50)
130 offset += dev_priv->vm_gart_base;
131 }
132 } else {
133 NV_ERROR(dev, "Bad DMA target, mem_type %d!\n",
134 chan->notifier_bo->bo.mem.mem_type);
135 return -EINVAL;
136 }
137 offset += mem->start; 120 offset += mem->start;
138 121
139 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset, 122 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset,
140 mem->size, NV_DMA_ACCESS_RW, target, 123 mem->size, NV_MEM_ACCESS_RW, target,
141 &nobj); 124 &nobj);
142 if (ret) { 125 if (ret) {
143 drm_mm_put_block(mem); 126 drm_mm_put_block(mem);
@@ -181,15 +164,20 @@ int
181nouveau_ioctl_notifier_alloc(struct drm_device *dev, void *data, 164nouveau_ioctl_notifier_alloc(struct drm_device *dev, void *data,
182 struct drm_file *file_priv) 165 struct drm_file *file_priv)
183{ 166{
167 struct drm_nouveau_private *dev_priv = dev->dev_private;
184 struct drm_nouveau_notifierobj_alloc *na = data; 168 struct drm_nouveau_notifierobj_alloc *na = data;
185 struct nouveau_channel *chan; 169 struct nouveau_channel *chan;
186 int ret; 170 int ret;
187 171
188 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(na->channel, file_priv, chan); 172 /* completely unnecessary for these chipsets... */
173 if (unlikely(dev_priv->card_type >= NV_C0))
174 return -EINVAL;
175
176 chan = nouveau_channel_get(dev, file_priv, na->channel);
177 if (IS_ERR(chan))
178 return PTR_ERR(chan);
189 179
190 ret = nouveau_notifier_alloc(chan, na->handle, na->size, &na->offset); 180 ret = nouveau_notifier_alloc(chan, na->handle, na->size, &na->offset);
191 if (ret) 181 nouveau_channel_put(&chan);
192 return ret; 182 return ret;
193
194 return 0;
195} 183}
diff --git a/drivers/gpu/drm/nouveau/nouveau_object.c b/drivers/gpu/drm/nouveau/nouveau_object.c
index dd572adca02a..30b6544467ca 100644
--- a/drivers/gpu/drm/nouveau/nouveau_object.c
+++ b/drivers/gpu/drm/nouveau/nouveau_object.c
@@ -35,6 +35,102 @@
35#include "nouveau_drv.h" 35#include "nouveau_drv.h"
36#include "nouveau_drm.h" 36#include "nouveau_drm.h"
37#include "nouveau_ramht.h" 37#include "nouveau_ramht.h"
38#include "nouveau_vm.h"
39
40struct nouveau_gpuobj_method {
41 struct list_head head;
42 u32 mthd;
43 int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
44};
45
46struct nouveau_gpuobj_class {
47 struct list_head head;
48 struct list_head methods;
49 u32 id;
50 u32 engine;
51};
52
53int
54nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
55{
56 struct drm_nouveau_private *dev_priv = dev->dev_private;
57 struct nouveau_gpuobj_class *oc;
58
59 oc = kzalloc(sizeof(*oc), GFP_KERNEL);
60 if (!oc)
61 return -ENOMEM;
62
63 INIT_LIST_HEAD(&oc->methods);
64 oc->id = class;
65 oc->engine = engine;
66 list_add(&oc->head, &dev_priv->classes);
67 return 0;
68}
69
70int
71nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
72 int (*exec)(struct nouveau_channel *, u32, u32, u32))
73{
74 struct drm_nouveau_private *dev_priv = dev->dev_private;
75 struct nouveau_gpuobj_method *om;
76 struct nouveau_gpuobj_class *oc;
77
78 list_for_each_entry(oc, &dev_priv->classes, head) {
79 if (oc->id == class)
80 goto found;
81 }
82
83 return -EINVAL;
84
85found:
86 om = kzalloc(sizeof(*om), GFP_KERNEL);
87 if (!om)
88 return -ENOMEM;
89
90 om->mthd = mthd;
91 om->exec = exec;
92 list_add(&om->head, &oc->methods);
93 return 0;
94}
95
96int
97nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
98 u32 class, u32 mthd, u32 data)
99{
100 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
101 struct nouveau_gpuobj_method *om;
102 struct nouveau_gpuobj_class *oc;
103
104 list_for_each_entry(oc, &dev_priv->classes, head) {
105 if (oc->id != class)
106 continue;
107
108 list_for_each_entry(om, &oc->methods, head) {
109 if (om->mthd == mthd)
110 return om->exec(chan, class, mthd, data);
111 }
112 }
113
114 return -ENOENT;
115}
116
117int
118nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
119 u32 class, u32 mthd, u32 data)
120{
121 struct drm_nouveau_private *dev_priv = dev->dev_private;
122 struct nouveau_channel *chan = NULL;
123 unsigned long flags;
124 int ret = -EINVAL;
125
126 spin_lock_irqsave(&dev_priv->channels.lock, flags);
127 if (chid > 0 && chid < dev_priv->engine.fifo.channels)
128 chan = dev_priv->channels.ptr[chid];
129 if (chan)
130 ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
131 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
132 return ret;
133}
38 134
39/* NVidia uses context objects to drive drawing operations. 135/* NVidia uses context objects to drive drawing operations.
40 136
@@ -73,17 +169,14 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
73 struct nouveau_gpuobj **gpuobj_ret) 169 struct nouveau_gpuobj **gpuobj_ret)
74{ 170{
75 struct drm_nouveau_private *dev_priv = dev->dev_private; 171 struct drm_nouveau_private *dev_priv = dev->dev_private;
76 struct nouveau_engine *engine = &dev_priv->engine; 172 struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
77 struct nouveau_gpuobj *gpuobj; 173 struct nouveau_gpuobj *gpuobj;
78 struct drm_mm_node *ramin = NULL; 174 struct drm_mm_node *ramin = NULL;
79 int ret; 175 int ret, i;
80 176
81 NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n", 177 NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
82 chan ? chan->id : -1, size, align, flags); 178 chan ? chan->id : -1, size, align, flags);
83 179
84 if (!dev_priv || !gpuobj_ret || *gpuobj_ret != NULL)
85 return -EINVAL;
86
87 gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL); 180 gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
88 if (!gpuobj) 181 if (!gpuobj)
89 return -ENOMEM; 182 return -ENOMEM;
@@ -98,88 +191,41 @@ nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
98 spin_unlock(&dev_priv->ramin_lock); 191 spin_unlock(&dev_priv->ramin_lock);
99 192
100 if (chan) { 193 if (chan) {
101 NV_DEBUG(dev, "channel heap\n");
102
103 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0); 194 ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
104 if (ramin) 195 if (ramin)
105 ramin = drm_mm_get_block(ramin, size, align); 196 ramin = drm_mm_get_block(ramin, size, align);
106
107 if (!ramin) { 197 if (!ramin) {
108 nouveau_gpuobj_ref(NULL, &gpuobj); 198 nouveau_gpuobj_ref(NULL, &gpuobj);
109 return -ENOMEM; 199 return -ENOMEM;
110 } 200 }
111 } else {
112 NV_DEBUG(dev, "global heap\n");
113
114 /* allocate backing pages, sets vinst */
115 ret = engine->instmem.populate(dev, gpuobj, &size);
116 if (ret) {
117 nouveau_gpuobj_ref(NULL, &gpuobj);
118 return ret;
119 }
120
121 /* try and get aperture space */
122 do {
123 if (drm_mm_pre_get(&dev_priv->ramin_heap))
124 return -ENOMEM;
125 201
126 spin_lock(&dev_priv->ramin_lock); 202 gpuobj->pinst = chan->ramin->pinst;
127 ramin = drm_mm_search_free(&dev_priv->ramin_heap, size, 203 if (gpuobj->pinst != ~0)
128 align, 0); 204 gpuobj->pinst += ramin->start;
129 if (ramin == NULL) {
130 spin_unlock(&dev_priv->ramin_lock);
131 nouveau_gpuobj_ref(NULL, &gpuobj);
132 return -ENOMEM;
133 }
134
135 ramin = drm_mm_get_block_atomic(ramin, size, align);
136 spin_unlock(&dev_priv->ramin_lock);
137 } while (ramin == NULL);
138
139 /* on nv50 it's ok to fail, we have a fallback path */
140 if (!ramin && dev_priv->card_type < NV_50) {
141 nouveau_gpuobj_ref(NULL, &gpuobj);
142 return -ENOMEM;
143 }
144 }
145 205
146 /* if we got a chunk of the aperture, map pages into it */ 206 gpuobj->cinst = ramin->start;
147 gpuobj->im_pramin = ramin; 207 gpuobj->vinst = ramin->start + chan->ramin->vinst;
148 if (!chan && gpuobj->im_pramin && dev_priv->ramin_available) { 208 gpuobj->node = ramin;
149 ret = engine->instmem.bind(dev, gpuobj); 209 } else {
210 ret = instmem->get(gpuobj, size, align);
150 if (ret) { 211 if (ret) {
151 nouveau_gpuobj_ref(NULL, &gpuobj); 212 nouveau_gpuobj_ref(NULL, &gpuobj);
152 return ret; 213 return ret;
153 } 214 }
154 }
155
156 /* calculate the various different addresses for the object */
157 if (chan) {
158 gpuobj->pinst = chan->ramin->pinst;
159 if (gpuobj->pinst != ~0)
160 gpuobj->pinst += gpuobj->im_pramin->start;
161 215
162 if (dev_priv->card_type < NV_50) { 216 ret = -ENOSYS;
163 gpuobj->cinst = gpuobj->pinst; 217 if (!(flags & NVOBJ_FLAG_DONT_MAP))
164 } else { 218 ret = instmem->map(gpuobj);
165 gpuobj->cinst = gpuobj->im_pramin->start; 219 if (ret)
166 gpuobj->vinst = gpuobj->im_pramin->start +
167 chan->ramin->vinst;
168 }
169 } else {
170 if (gpuobj->im_pramin)
171 gpuobj->pinst = gpuobj->im_pramin->start;
172 else
173 gpuobj->pinst = ~0; 220 gpuobj->pinst = ~0;
174 gpuobj->cinst = 0xdeadbeef; 221
222 gpuobj->cinst = NVOBJ_CINST_GLOBAL;
175 } 223 }
176 224
177 if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) { 225 if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
178 int i;
179
180 for (i = 0; i < gpuobj->size; i += 4) 226 for (i = 0; i < gpuobj->size; i += 4)
181 nv_wo32(gpuobj, i, 0); 227 nv_wo32(gpuobj, i, 0);
182 engine->instmem.flush(dev); 228 instmem->flush(dev);
183 } 229 }
184 230
185 231
@@ -195,6 +241,7 @@ nouveau_gpuobj_init(struct drm_device *dev)
195 NV_DEBUG(dev, "\n"); 241 NV_DEBUG(dev, "\n");
196 242
197 INIT_LIST_HEAD(&dev_priv->gpuobj_list); 243 INIT_LIST_HEAD(&dev_priv->gpuobj_list);
244 INIT_LIST_HEAD(&dev_priv->classes);
198 spin_lock_init(&dev_priv->ramin_lock); 245 spin_lock_init(&dev_priv->ramin_lock);
199 dev_priv->ramin_base = ~0; 246 dev_priv->ramin_base = ~0;
200 247
@@ -205,9 +252,20 @@ void
205nouveau_gpuobj_takedown(struct drm_device *dev) 252nouveau_gpuobj_takedown(struct drm_device *dev)
206{ 253{
207 struct drm_nouveau_private *dev_priv = dev->dev_private; 254 struct drm_nouveau_private *dev_priv = dev->dev_private;
255 struct nouveau_gpuobj_method *om, *tm;
256 struct nouveau_gpuobj_class *oc, *tc;
208 257
209 NV_DEBUG(dev, "\n"); 258 NV_DEBUG(dev, "\n");
210 259
260 list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
261 list_for_each_entry_safe(om, tm, &oc->methods, head) {
262 list_del(&om->head);
263 kfree(om);
264 }
265 list_del(&oc->head);
266 kfree(oc);
267 }
268
211 BUG_ON(!list_empty(&dev_priv->gpuobj_list)); 269 BUG_ON(!list_empty(&dev_priv->gpuobj_list));
212} 270}
213 271
@@ -219,26 +277,34 @@ nouveau_gpuobj_del(struct kref *ref)
219 container_of(ref, struct nouveau_gpuobj, refcount); 277 container_of(ref, struct nouveau_gpuobj, refcount);
220 struct drm_device *dev = gpuobj->dev; 278 struct drm_device *dev = gpuobj->dev;
221 struct drm_nouveau_private *dev_priv = dev->dev_private; 279 struct drm_nouveau_private *dev_priv = dev->dev_private;
222 struct nouveau_engine *engine = &dev_priv->engine; 280 struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
223 int i; 281 int i;
224 282
225 NV_DEBUG(dev, "gpuobj %p\n", gpuobj); 283 NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
226 284
227 if (gpuobj->im_pramin && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) { 285 if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
228 for (i = 0; i < gpuobj->size; i += 4) 286 for (i = 0; i < gpuobj->size; i += 4)
229 nv_wo32(gpuobj, i, 0); 287 nv_wo32(gpuobj, i, 0);
230 engine->instmem.flush(dev); 288 instmem->flush(dev);
231 } 289 }
232 290
233 if (gpuobj->dtor) 291 if (gpuobj->dtor)
234 gpuobj->dtor(dev, gpuobj); 292 gpuobj->dtor(dev, gpuobj);
235 293
236 if (gpuobj->im_backing) 294 if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
237 engine->instmem.clear(dev, gpuobj); 295 if (gpuobj->node) {
296 instmem->unmap(gpuobj);
297 instmem->put(gpuobj);
298 }
299 } else {
300 if (gpuobj->node) {
301 spin_lock(&dev_priv->ramin_lock);
302 drm_mm_put_block(gpuobj->node);
303 spin_unlock(&dev_priv->ramin_lock);
304 }
305 }
238 306
239 spin_lock(&dev_priv->ramin_lock); 307 spin_lock(&dev_priv->ramin_lock);
240 if (gpuobj->im_pramin)
241 drm_mm_put_block(gpuobj->im_pramin);
242 list_del(&gpuobj->list); 308 list_del(&gpuobj->list);
243 spin_unlock(&dev_priv->ramin_lock); 309 spin_unlock(&dev_priv->ramin_lock);
244 310
@@ -278,7 +344,7 @@ nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
278 kref_init(&gpuobj->refcount); 344 kref_init(&gpuobj->refcount);
279 gpuobj->size = size; 345 gpuobj->size = size;
280 gpuobj->pinst = pinst; 346 gpuobj->pinst = pinst;
281 gpuobj->cinst = 0xdeadbeef; 347 gpuobj->cinst = NVOBJ_CINST_GLOBAL;
282 gpuobj->vinst = vinst; 348 gpuobj->vinst = vinst;
283 349
284 if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) { 350 if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
@@ -335,113 +401,150 @@ nouveau_gpuobj_class_instmem_size(struct drm_device *dev, int class)
335 The method below creates a DMA object in instance RAM and returns a handle 401 The method below creates a DMA object in instance RAM and returns a handle
336 to it that can be used to set up context objects. 402 to it that can be used to set up context objects.
337*/ 403*/
338int 404
339nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, 405void
340 uint64_t offset, uint64_t size, int access, 406nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
341 int target, struct nouveau_gpuobj **gpuobj) 407 u64 base, u64 size, int target, int access,
408 u32 type, u32 comp)
342{ 409{
343 struct drm_device *dev = chan->dev; 410 struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
344 struct drm_nouveau_private *dev_priv = dev->dev_private; 411 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
345 struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem; 412 u32 flags0;
346 int ret;
347 413
348 NV_DEBUG(dev, "ch%d class=0x%04x offset=0x%llx size=0x%llx\n", 414 flags0 = (comp << 29) | (type << 22) | class;
349 chan->id, class, offset, size); 415 flags0 |= 0x00100000;
350 NV_DEBUG(dev, "access=%d target=%d\n", access, target); 416
417 switch (access) {
418 case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
419 case NV_MEM_ACCESS_RW:
420 case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
421 default:
422 break;
423 }
351 424
352 switch (target) { 425 switch (target) {
353 case NV_DMA_TARGET_AGP: 426 case NV_MEM_TARGET_VRAM:
354 offset += dev_priv->gart_info.aper_base; 427 flags0 |= 0x00010000;
428 break;
429 case NV_MEM_TARGET_PCI:
430 flags0 |= 0x00020000;
431 break;
432 case NV_MEM_TARGET_PCI_NOSNOOP:
433 flags0 |= 0x00030000;
355 break; 434 break;
435 case NV_MEM_TARGET_GART:
436 base += dev_priv->gart_info.aper_base;
356 default: 437 default:
438 flags0 &= ~0x00100000;
357 break; 439 break;
358 } 440 }
359 441
360 ret = nouveau_gpuobj_new(dev, chan, 442 /* convert to base + limit */
361 nouveau_gpuobj_class_instmem_size(dev, class), 443 size = (base + size) - 1;
362 16, NVOBJ_FLAG_ZERO_ALLOC |
363 NVOBJ_FLAG_ZERO_FREE, gpuobj);
364 if (ret) {
365 NV_ERROR(dev, "Error creating gpuobj: %d\n", ret);
366 return ret;
367 }
368 444
369 if (dev_priv->card_type < NV_50) { 445 nv_wo32(obj, offset + 0x00, flags0);
370 uint32_t frame, adjust, pte_flags = 0; 446 nv_wo32(obj, offset + 0x04, lower_32_bits(size));
371 447 nv_wo32(obj, offset + 0x08, lower_32_bits(base));
372 if (access != NV_DMA_ACCESS_RO) 448 nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
373 pte_flags |= (1<<1); 449 upper_32_bits(base));
374 adjust = offset & 0x00000fff; 450 nv_wo32(obj, offset + 0x10, 0x00000000);
375 frame = offset & ~0x00000fff; 451 nv_wo32(obj, offset + 0x14, 0x00000000);
376
377 nv_wo32(*gpuobj, 0, ((1<<12) | (1<<13) | (adjust << 20) |
378 (access << 14) | (target << 16) |
379 class));
380 nv_wo32(*gpuobj, 4, size - 1);
381 nv_wo32(*gpuobj, 8, frame | pte_flags);
382 nv_wo32(*gpuobj, 12, frame | pte_flags);
383 } else {
384 uint64_t limit = offset + size - 1;
385 uint32_t flags0, flags5;
386 452
387 if (target == NV_DMA_TARGET_VIDMEM) { 453 pinstmem->flush(obj->dev);
388 flags0 = 0x00190000; 454}
389 flags5 = 0x00010000;
390 } else {
391 flags0 = 0x7fc00000;
392 flags5 = 0x00080000;
393 }
394 455
395 nv_wo32(*gpuobj, 0, flags0 | class); 456int
396 nv_wo32(*gpuobj, 4, lower_32_bits(limit)); 457nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
397 nv_wo32(*gpuobj, 8, lower_32_bits(offset)); 458 int target, int access, u32 type, u32 comp,
398 nv_wo32(*gpuobj, 12, ((upper_32_bits(limit) & 0xff) << 24) | 459 struct nouveau_gpuobj **pobj)
399 (upper_32_bits(offset) & 0xff)); 460{
400 nv_wo32(*gpuobj, 20, flags5); 461 struct drm_device *dev = chan->dev;
401 } 462 int ret;
402 463
403 instmem->flush(dev); 464 ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
465 if (ret)
466 return ret;
404 467
405 (*gpuobj)->engine = NVOBJ_ENGINE_SW; 468 nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
406 (*gpuobj)->class = class; 469 access, type, comp);
407 return 0; 470 return 0;
408} 471}
409 472
410int 473int
411nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan, 474nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
412 uint64_t offset, uint64_t size, int access, 475 u64 size, int access, int target,
413 struct nouveau_gpuobj **gpuobj, 476 struct nouveau_gpuobj **pobj)
414 uint32_t *o_ret)
415{ 477{
478 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
416 struct drm_device *dev = chan->dev; 479 struct drm_device *dev = chan->dev;
417 struct drm_nouveau_private *dev_priv = dev->dev_private; 480 struct nouveau_gpuobj *obj;
481 u32 flags0, flags2;
418 int ret; 482 int ret;
419 483
420 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP || 484 if (dev_priv->card_type >= NV_50) {
421 (dev_priv->card_type >= NV_50 && 485 u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
422 dev_priv->gart_info.type == NOUVEAU_GART_SGDMA)) { 486 u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
423 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 487
424 offset + dev_priv->vm_gart_base, 488 return nv50_gpuobj_dma_new(chan, class, base, size,
425 size, access, NV_DMA_TARGET_AGP, 489 target, access, type, comp, pobj);
426 gpuobj); 490 }
427 if (o_ret) 491
428 *o_ret = 0; 492 if (target == NV_MEM_TARGET_GART) {
429 } else 493 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
430 if (dev_priv->gart_info.type == NOUVEAU_GART_SGDMA) { 494 target = NV_MEM_TARGET_PCI_NOSNOOP;
431 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma, gpuobj); 495 base += dev_priv->gart_info.aper_base;
432 if (offset & ~0xffffffffULL) { 496 } else
433 NV_ERROR(dev, "obj offset exceeds 32-bits\n"); 497 if (base != 0) {
434 return -EINVAL; 498 base = nouveau_sgdma_get_physical(dev, base);
499 target = NV_MEM_TARGET_PCI;
500 } else {
501 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma, pobj);
502 return 0;
435 } 503 }
436 if (o_ret)
437 *o_ret = (uint32_t)offset;
438 ret = (*gpuobj != NULL) ? 0 : -EINVAL;
439 } else {
440 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type);
441 return -EINVAL;
442 } 504 }
443 505
444 return ret; 506 flags0 = class;
507 flags0 |= 0x00003000; /* PT present, PT linear */
508 flags2 = 0;
509
510 switch (target) {
511 case NV_MEM_TARGET_PCI:
512 flags0 |= 0x00020000;
513 break;
514 case NV_MEM_TARGET_PCI_NOSNOOP:
515 flags0 |= 0x00030000;
516 break;
517 default:
518 break;
519 }
520
521 switch (access) {
522 case NV_MEM_ACCESS_RO:
523 flags0 |= 0x00004000;
524 break;
525 case NV_MEM_ACCESS_WO:
526 flags0 |= 0x00008000;
527 default:
528 flags2 |= 0x00000002;
529 break;
530 }
531
532 flags0 |= (base & 0x00000fff) << 20;
533 flags2 |= (base & 0xfffff000);
534
535 ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
536 if (ret)
537 return ret;
538
539 nv_wo32(obj, 0x00, flags0);
540 nv_wo32(obj, 0x04, size - 1);
541 nv_wo32(obj, 0x08, flags2);
542 nv_wo32(obj, 0x0c, flags2);
543
544 obj->engine = NVOBJ_ENGINE_SW;
545 obj->class = class;
546 *pobj = obj;
547 return 0;
445} 548}
446 549
447/* Context objects in the instance RAM have the following structure. 550/* Context objects in the instance RAM have the following structure.
@@ -495,82 +598,130 @@ nouveau_gpuobj_gart_dma_new(struct nouveau_channel *chan,
495 entry[5]: 598 entry[5]:
496 set to 0? 599 set to 0?
497*/ 600*/
601static int
602nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class,
603 struct nouveau_gpuobj **gpuobj_ret)
604{
605 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
606 struct nouveau_gpuobj *gpuobj;
607
608 gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
609 if (!gpuobj)
610 return -ENOMEM;
611 gpuobj->dev = chan->dev;
612 gpuobj->engine = NVOBJ_ENGINE_SW;
613 gpuobj->class = class;
614 kref_init(&gpuobj->refcount);
615 gpuobj->cinst = 0x40;
616
617 spin_lock(&dev_priv->ramin_lock);
618 list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
619 spin_unlock(&dev_priv->ramin_lock);
620 *gpuobj_ret = gpuobj;
621 return 0;
622}
623
498int 624int
499nouveau_gpuobj_gr_new(struct nouveau_channel *chan, int class, 625nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
500 struct nouveau_gpuobj **gpuobj)
501{ 626{
627 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
502 struct drm_device *dev = chan->dev; 628 struct drm_device *dev = chan->dev;
503 struct drm_nouveau_private *dev_priv = dev->dev_private; 629 struct nouveau_gpuobj_class *oc;
630 struct nouveau_gpuobj *gpuobj;
504 int ret; 631 int ret;
505 632
506 NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class); 633 NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
507 634
635 list_for_each_entry(oc, &dev_priv->classes, head) {
636 if (oc->id == class)
637 goto found;
638 }
639
640 NV_ERROR(dev, "illegal object class: 0x%x\n", class);
641 return -EINVAL;
642
643found:
644 switch (oc->engine) {
645 case NVOBJ_ENGINE_SW:
646 if (dev_priv->card_type < NV_C0) {
647 ret = nouveau_gpuobj_sw_new(chan, class, &gpuobj);
648 if (ret)
649 return ret;
650 goto insert;
651 }
652 break;
653 case NVOBJ_ENGINE_GR:
654 if ((dev_priv->card_type >= NV_20 && !chan->ramin_grctx) ||
655 (dev_priv->card_type < NV_20 && !chan->pgraph_ctx)) {
656 struct nouveau_pgraph_engine *pgraph =
657 &dev_priv->engine.graph;
658
659 ret = pgraph->create_context(chan);
660 if (ret)
661 return ret;
662 }
663 break;
664 case NVOBJ_ENGINE_CRYPT:
665 if (!chan->crypt_ctx) {
666 struct nouveau_crypt_engine *pcrypt =
667 &dev_priv->engine.crypt;
668
669 ret = pcrypt->create_context(chan);
670 if (ret)
671 return ret;
672 }
673 break;
674 }
675
676 /* we're done if this is fermi */
677 if (dev_priv->card_type >= NV_C0)
678 return 0;
679
508 ret = nouveau_gpuobj_new(dev, chan, 680 ret = nouveau_gpuobj_new(dev, chan,
509 nouveau_gpuobj_class_instmem_size(dev, class), 681 nouveau_gpuobj_class_instmem_size(dev, class),
510 16, 682 16,
511 NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE, 683 NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
512 gpuobj); 684 &gpuobj);
513 if (ret) { 685 if (ret) {
514 NV_ERROR(dev, "Error creating gpuobj: %d\n", ret); 686 NV_ERROR(dev, "error creating gpuobj: %d\n", ret);
515 return ret; 687 return ret;
516 } 688 }
517 689
518 if (dev_priv->card_type >= NV_50) { 690 if (dev_priv->card_type >= NV_50) {
519 nv_wo32(*gpuobj, 0, class); 691 nv_wo32(gpuobj, 0, class);
520 nv_wo32(*gpuobj, 20, 0x00010000); 692 nv_wo32(gpuobj, 20, 0x00010000);
521 } else { 693 } else {
522 switch (class) { 694 switch (class) {
523 case NV_CLASS_NULL: 695 case NV_CLASS_NULL:
524 nv_wo32(*gpuobj, 0, 0x00001030); 696 nv_wo32(gpuobj, 0, 0x00001030);
525 nv_wo32(*gpuobj, 4, 0xFFFFFFFF); 697 nv_wo32(gpuobj, 4, 0xFFFFFFFF);
526 break; 698 break;
527 default: 699 default:
528 if (dev_priv->card_type >= NV_40) { 700 if (dev_priv->card_type >= NV_40) {
529 nv_wo32(*gpuobj, 0, class); 701 nv_wo32(gpuobj, 0, class);
530#ifdef __BIG_ENDIAN 702#ifdef __BIG_ENDIAN
531 nv_wo32(*gpuobj, 8, 0x01000000); 703 nv_wo32(gpuobj, 8, 0x01000000);
532#endif 704#endif
533 } else { 705 } else {
534#ifdef __BIG_ENDIAN 706#ifdef __BIG_ENDIAN
535 nv_wo32(*gpuobj, 0, class | 0x00080000); 707 nv_wo32(gpuobj, 0, class | 0x00080000);
536#else 708#else
537 nv_wo32(*gpuobj, 0, class); 709 nv_wo32(gpuobj, 0, class);
538#endif 710#endif
539 } 711 }
540 } 712 }
541 } 713 }
542 dev_priv->engine.instmem.flush(dev); 714 dev_priv->engine.instmem.flush(dev);
543 715
544 (*gpuobj)->engine = NVOBJ_ENGINE_GR; 716 gpuobj->engine = oc->engine;
545 (*gpuobj)->class = class; 717 gpuobj->class = oc->id;
546 return 0;
547}
548 718
549int 719insert:
550nouveau_gpuobj_sw_new(struct nouveau_channel *chan, int class, 720 ret = nouveau_ramht_insert(chan, handle, gpuobj);
551 struct nouveau_gpuobj **gpuobj_ret) 721 if (ret)
552{ 722 NV_ERROR(dev, "error adding gpuobj to RAMHT: %d\n", ret);
553 struct drm_nouveau_private *dev_priv; 723 nouveau_gpuobj_ref(NULL, &gpuobj);
554 struct nouveau_gpuobj *gpuobj; 724 return ret;
555
556 if (!chan || !gpuobj_ret || *gpuobj_ret != NULL)
557 return -EINVAL;
558 dev_priv = chan->dev->dev_private;
559
560 gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
561 if (!gpuobj)
562 return -ENOMEM;
563 gpuobj->dev = chan->dev;
564 gpuobj->engine = NVOBJ_ENGINE_SW;
565 gpuobj->class = class;
566 kref_init(&gpuobj->refcount);
567 gpuobj->cinst = 0x40;
568
569 spin_lock(&dev_priv->ramin_lock);
570 list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
571 spin_unlock(&dev_priv->ramin_lock);
572 *gpuobj_ret = gpuobj;
573 return 0;
574} 725}
575 726
576static int 727static int
@@ -585,7 +736,7 @@ nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
585 NV_DEBUG(dev, "ch%d\n", chan->id); 736 NV_DEBUG(dev, "ch%d\n", chan->id);
586 737
587 /* Base amount for object storage (4KiB enough?) */ 738 /* Base amount for object storage (4KiB enough?) */
588 size = 0x1000; 739 size = 0x2000;
589 base = 0; 740 base = 0;
590 741
591 /* PGRAPH context */ 742 /* PGRAPH context */
@@ -624,12 +775,30 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
624{ 775{
625 struct drm_device *dev = chan->dev; 776 struct drm_device *dev = chan->dev;
626 struct drm_nouveau_private *dev_priv = dev->dev_private; 777 struct drm_nouveau_private *dev_priv = dev->dev_private;
627 struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
628 struct nouveau_gpuobj *vram = NULL, *tt = NULL; 778 struct nouveau_gpuobj *vram = NULL, *tt = NULL;
629 int ret, i; 779 int ret;
630 780
631 NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h); 781 NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
632 782
783 if (dev_priv->card_type == NV_C0) {
784 struct nouveau_vm *vm = dev_priv->chan_vm;
785 struct nouveau_vm_pgd *vpgd;
786
787 ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0,
788 &chan->ramin);
789 if (ret)
790 return ret;
791
792 nouveau_vm_ref(vm, &chan->vm, NULL);
793
794 vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
795 nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
796 nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
797 nv_wo32(chan->ramin, 0x0208, 0xffffffff);
798 nv_wo32(chan->ramin, 0x020c, 0x000000ff);
799 return 0;
800 }
801
633 /* Allocate a chunk of memory for per-channel object storage */ 802 /* Allocate a chunk of memory for per-channel object storage */
634 ret = nouveau_gpuobj_channel_init_pramin(chan); 803 ret = nouveau_gpuobj_channel_init_pramin(chan);
635 if (ret) { 804 if (ret) {
@@ -639,14 +808,12 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
639 808
640 /* NV50 VM 809 /* NV50 VM
641 * - Allocate per-channel page-directory 810 * - Allocate per-channel page-directory
642 * - Map GART and VRAM into the channel's address space at the 811 * - Link with shared channel VM
643 * locations determined during init.
644 */ 812 */
645 if (dev_priv->card_type >= NV_50) { 813 if (dev_priv->chan_vm) {
646 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200; 814 u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
647 u64 vm_vinst = chan->ramin->vinst + pgd_offs; 815 u64 vm_vinst = chan->ramin->vinst + pgd_offs;
648 u32 vm_pinst = chan->ramin->pinst; 816 u32 vm_pinst = chan->ramin->pinst;
649 u32 pde;
650 817
651 if (vm_pinst != ~0) 818 if (vm_pinst != ~0)
652 vm_pinst += pgd_offs; 819 vm_pinst += pgd_offs;
@@ -655,29 +822,8 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
655 0, &chan->vm_pd); 822 0, &chan->vm_pd);
656 if (ret) 823 if (ret)
657 return ret; 824 return ret;
658 for (i = 0; i < 0x4000; i += 8) {
659 nv_wo32(chan->vm_pd, i + 0, 0x00000000);
660 nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
661 }
662
663 nouveau_gpuobj_ref(dev_priv->gart_info.sg_ctxdma,
664 &chan->vm_gart_pt);
665 pde = (dev_priv->vm_gart_base / (512*1024*1024)) * 8;
666 nv_wo32(chan->vm_pd, pde + 0, chan->vm_gart_pt->vinst | 3);
667 nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
668
669 pde = (dev_priv->vm_vram_base / (512*1024*1024)) * 8;
670 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
671 nouveau_gpuobj_ref(dev_priv->vm_vram_pt[i],
672 &chan->vm_vram_pt[i]);
673
674 nv_wo32(chan->vm_pd, pde + 0,
675 chan->vm_vram_pt[i]->vinst | 0x61);
676 nv_wo32(chan->vm_pd, pde + 4, 0x00000000);
677 pde += 8;
678 }
679 825
680 instmem->flush(dev); 826 nouveau_vm_ref(dev_priv->chan_vm, &chan->vm, chan->vm_pd);
681 } 827 }
682 828
683 /* RAMHT */ 829 /* RAMHT */
@@ -700,9 +846,8 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
700 /* VRAM ctxdma */ 846 /* VRAM ctxdma */
701 if (dev_priv->card_type >= NV_50) { 847 if (dev_priv->card_type >= NV_50) {
702 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 848 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
703 0, dev_priv->vm_end, 849 0, (1ULL << 40), NV_MEM_ACCESS_RW,
704 NV_DMA_ACCESS_RW, 850 NV_MEM_TARGET_VM, &vram);
705 NV_DMA_TARGET_AGP, &vram);
706 if (ret) { 851 if (ret) {
707 NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret); 852 NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
708 return ret; 853 return ret;
@@ -710,8 +855,8 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
710 } else { 855 } else {
711 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 856 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
712 0, dev_priv->fb_available_size, 857 0, dev_priv->fb_available_size,
713 NV_DMA_ACCESS_RW, 858 NV_MEM_ACCESS_RW,
714 NV_DMA_TARGET_VIDMEM, &vram); 859 NV_MEM_TARGET_VRAM, &vram);
715 if (ret) { 860 if (ret) {
716 NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret); 861 NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
717 return ret; 862 return ret;
@@ -728,21 +873,13 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
728 /* TT memory ctxdma */ 873 /* TT memory ctxdma */
729 if (dev_priv->card_type >= NV_50) { 874 if (dev_priv->card_type >= NV_50) {
730 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 875 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
731 0, dev_priv->vm_end, 876 0, (1ULL << 40), NV_MEM_ACCESS_RW,
732 NV_DMA_ACCESS_RW, 877 NV_MEM_TARGET_VM, &tt);
733 NV_DMA_TARGET_AGP, &tt);
734 if (ret) {
735 NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
736 return ret;
737 }
738 } else
739 if (dev_priv->gart_info.type != NOUVEAU_GART_NONE) {
740 ret = nouveau_gpuobj_gart_dma_new(chan, 0,
741 dev_priv->gart_info.aper_size,
742 NV_DMA_ACCESS_RW, &tt, NULL);
743 } else { 878 } else {
744 NV_ERROR(dev, "Invalid GART type %d\n", dev_priv->gart_info.type); 879 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
745 ret = -EINVAL; 880 0, dev_priv->gart_info.aper_size,
881 NV_MEM_ACCESS_RW,
882 NV_MEM_TARGET_GART, &tt);
746 } 883 }
747 884
748 if (ret) { 885 if (ret) {
@@ -763,21 +900,14 @@ nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
763void 900void
764nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan) 901nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
765{ 902{
766 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
767 struct drm_device *dev = chan->dev; 903 struct drm_device *dev = chan->dev;
768 int i;
769 904
770 NV_DEBUG(dev, "ch%d\n", chan->id); 905 NV_DEBUG(dev, "ch%d\n", chan->id);
771 906
772 if (!chan->ramht)
773 return;
774
775 nouveau_ramht_ref(NULL, &chan->ramht, chan); 907 nouveau_ramht_ref(NULL, &chan->ramht, chan);
776 908
909 nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
777 nouveau_gpuobj_ref(NULL, &chan->vm_pd); 910 nouveau_gpuobj_ref(NULL, &chan->vm_pd);
778 nouveau_gpuobj_ref(NULL, &chan->vm_gart_pt);
779 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++)
780 nouveau_gpuobj_ref(NULL, &chan->vm_vram_pt[i]);
781 911
782 if (chan->ramin_heap.free_stack.next) 912 if (chan->ramin_heap.free_stack.next)
783 drm_mm_takedown(&chan->ramin_heap); 913 drm_mm_takedown(&chan->ramin_heap);
@@ -791,147 +921,91 @@ nouveau_gpuobj_suspend(struct drm_device *dev)
791 struct nouveau_gpuobj *gpuobj; 921 struct nouveau_gpuobj *gpuobj;
792 int i; 922 int i;
793 923
794 if (dev_priv->card_type < NV_50) {
795 dev_priv->susres.ramin_copy = vmalloc(dev_priv->ramin_rsvd_vram);
796 if (!dev_priv->susres.ramin_copy)
797 return -ENOMEM;
798
799 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
800 dev_priv->susres.ramin_copy[i/4] = nv_ri32(dev, i);
801 return 0;
802 }
803
804 list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) { 924 list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
805 if (!gpuobj->im_backing) 925 if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
806 continue; 926 continue;
807 927
808 gpuobj->im_backing_suspend = vmalloc(gpuobj->size); 928 gpuobj->suspend = vmalloc(gpuobj->size);
809 if (!gpuobj->im_backing_suspend) { 929 if (!gpuobj->suspend) {
810 nouveau_gpuobj_resume(dev); 930 nouveau_gpuobj_resume(dev);
811 return -ENOMEM; 931 return -ENOMEM;
812 } 932 }
813 933
814 for (i = 0; i < gpuobj->size; i += 4) 934 for (i = 0; i < gpuobj->size; i += 4)
815 gpuobj->im_backing_suspend[i/4] = nv_ro32(gpuobj, i); 935 gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
816 } 936 }
817 937
818 return 0; 938 return 0;
819} 939}
820 940
821void 941void
822nouveau_gpuobj_suspend_cleanup(struct drm_device *dev)
823{
824 struct drm_nouveau_private *dev_priv = dev->dev_private;
825 struct nouveau_gpuobj *gpuobj;
826
827 if (dev_priv->card_type < NV_50) {
828 vfree(dev_priv->susres.ramin_copy);
829 dev_priv->susres.ramin_copy = NULL;
830 return;
831 }
832
833 list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
834 if (!gpuobj->im_backing_suspend)
835 continue;
836
837 vfree(gpuobj->im_backing_suspend);
838 gpuobj->im_backing_suspend = NULL;
839 }
840}
841
842void
843nouveau_gpuobj_resume(struct drm_device *dev) 942nouveau_gpuobj_resume(struct drm_device *dev)
844{ 943{
845 struct drm_nouveau_private *dev_priv = dev->dev_private; 944 struct drm_nouveau_private *dev_priv = dev->dev_private;
846 struct nouveau_gpuobj *gpuobj; 945 struct nouveau_gpuobj *gpuobj;
847 int i; 946 int i;
848 947
849 if (dev_priv->card_type < NV_50) {
850 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4)
851 nv_wi32(dev, i, dev_priv->susres.ramin_copy[i/4]);
852 nouveau_gpuobj_suspend_cleanup(dev);
853 return;
854 }
855
856 list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) { 948 list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
857 if (!gpuobj->im_backing_suspend) 949 if (!gpuobj->suspend)
858 continue; 950 continue;
859 951
860 for (i = 0; i < gpuobj->size; i += 4) 952 for (i = 0; i < gpuobj->size; i += 4)
861 nv_wo32(gpuobj, i, gpuobj->im_backing_suspend[i/4]); 953 nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
862 dev_priv->engine.instmem.flush(dev); 954
955 vfree(gpuobj->suspend);
956 gpuobj->suspend = NULL;
863 } 957 }
864 958
865 nouveau_gpuobj_suspend_cleanup(dev); 959 dev_priv->engine.instmem.flush(dev);
866} 960}
867 961
868int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data, 962int nouveau_ioctl_grobj_alloc(struct drm_device *dev, void *data,
869 struct drm_file *file_priv) 963 struct drm_file *file_priv)
870{ 964{
871 struct drm_nouveau_private *dev_priv = dev->dev_private;
872 struct drm_nouveau_grobj_alloc *init = data; 965 struct drm_nouveau_grobj_alloc *init = data;
873 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
874 struct nouveau_pgraph_object_class *grc;
875 struct nouveau_gpuobj *gr = NULL;
876 struct nouveau_channel *chan; 966 struct nouveau_channel *chan;
877 int ret; 967 int ret;
878 968
879 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(init->channel, file_priv, chan);
880
881 if (init->handle == ~0) 969 if (init->handle == ~0)
882 return -EINVAL; 970 return -EINVAL;
883 971
884 grc = pgraph->grclass; 972 chan = nouveau_channel_get(dev, file_priv, init->channel);
885 while (grc->id) { 973 if (IS_ERR(chan))
886 if (grc->id == init->class) 974 return PTR_ERR(chan);
887 break;
888 grc++;
889 }
890 975
891 if (!grc->id) { 976 if (nouveau_ramht_find(chan, init->handle)) {
892 NV_ERROR(dev, "Illegal object class: 0x%x\n", init->class); 977 ret = -EEXIST;
893 return -EPERM; 978 goto out;
894 } 979 }
895 980
896 if (nouveau_ramht_find(chan, init->handle)) 981 ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
897 return -EEXIST;
898
899 if (!grc->software)
900 ret = nouveau_gpuobj_gr_new(chan, grc->id, &gr);
901 else
902 ret = nouveau_gpuobj_sw_new(chan, grc->id, &gr);
903 if (ret) { 982 if (ret) {
904 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n", 983 NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
905 ret, init->channel, init->handle); 984 ret, init->channel, init->handle);
906 return ret;
907 } 985 }
908 986
909 ret = nouveau_ramht_insert(chan, init->handle, gr); 987out:
910 nouveau_gpuobj_ref(NULL, &gr); 988 nouveau_channel_put(&chan);
911 if (ret) { 989 return ret;
912 NV_ERROR(dev, "Error referencing object: %d (%d/0x%08x)\n",
913 ret, init->channel, init->handle);
914 return ret;
915 }
916
917 return 0;
918} 990}
919 991
920int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data, 992int nouveau_ioctl_gpuobj_free(struct drm_device *dev, void *data,
921 struct drm_file *file_priv) 993 struct drm_file *file_priv)
922{ 994{
923 struct drm_nouveau_gpuobj_free *objfree = data; 995 struct drm_nouveau_gpuobj_free *objfree = data;
924 struct nouveau_gpuobj *gpuobj;
925 struct nouveau_channel *chan; 996 struct nouveau_channel *chan;
997 int ret;
926 998
927 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(objfree->channel, file_priv, chan); 999 chan = nouveau_channel_get(dev, file_priv, objfree->channel);
1000 if (IS_ERR(chan))
1001 return PTR_ERR(chan);
928 1002
929 gpuobj = nouveau_ramht_find(chan, objfree->handle); 1003 /* Synchronize with the user channel */
930 if (!gpuobj) 1004 nouveau_channel_idle(chan);
931 return -ENOENT;
932 1005
933 nouveau_ramht_remove(chan, objfree->handle); 1006 ret = nouveau_ramht_remove(chan, objfree->handle);
934 return 0; 1007 nouveau_channel_put(&chan);
1008 return ret;
935} 1009}
936 1010
937u32 1011u32
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.c b/drivers/gpu/drm/nouveau/nouveau_pm.c
index 9f7b158f5825..fb846a3fef15 100644
--- a/drivers/gpu/drm/nouveau/nouveau_pm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_pm.c
@@ -27,6 +27,10 @@
27#include "nouveau_drv.h" 27#include "nouveau_drv.h"
28#include "nouveau_pm.h" 28#include "nouveau_pm.h"
29 29
30#ifdef CONFIG_ACPI
31#include <linux/acpi.h>
32#endif
33#include <linux/power_supply.h>
30#include <linux/hwmon.h> 34#include <linux/hwmon.h>
31#include <linux/hwmon-sysfs.h> 35#include <linux/hwmon-sysfs.h>
32 36
@@ -418,8 +422,7 @@ nouveau_hwmon_init(struct drm_device *dev)
418 return ret; 422 return ret;
419 } 423 }
420 dev_set_drvdata(hwmon_dev, dev); 424 dev_set_drvdata(hwmon_dev, dev);
421 ret = sysfs_create_group(&hwmon_dev->kobj, 425 ret = sysfs_create_group(&dev->pdev->dev.kobj, &hwmon_attrgroup);
422 &hwmon_attrgroup);
423 if (ret) { 426 if (ret) {
424 NV_ERROR(dev, 427 NV_ERROR(dev,
425 "Unable to create hwmon sysfs file: %d\n", ret); 428 "Unable to create hwmon sysfs file: %d\n", ret);
@@ -446,6 +449,25 @@ nouveau_hwmon_fini(struct drm_device *dev)
446#endif 449#endif
447} 450}
448 451
452#ifdef CONFIG_ACPI
453static int
454nouveau_pm_acpi_event(struct notifier_block *nb, unsigned long val, void *data)
455{
456 struct drm_nouveau_private *dev_priv =
457 container_of(nb, struct drm_nouveau_private, engine.pm.acpi_nb);
458 struct drm_device *dev = dev_priv->dev;
459 struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
460
461 if (strcmp(entry->device_class, "ac_adapter") == 0) {
462 bool ac = power_supply_is_system_supplied();
463
464 NV_DEBUG(dev, "power supply changed: %s\n", ac ? "AC" : "DC");
465 }
466
467 return NOTIFY_OK;
468}
469#endif
470
449int 471int
450nouveau_pm_init(struct drm_device *dev) 472nouveau_pm_init(struct drm_device *dev)
451{ 473{
@@ -485,6 +507,10 @@ nouveau_pm_init(struct drm_device *dev)
485 507
486 nouveau_sysfs_init(dev); 508 nouveau_sysfs_init(dev);
487 nouveau_hwmon_init(dev); 509 nouveau_hwmon_init(dev);
510#ifdef CONFIG_ACPI
511 pm->acpi_nb.notifier_call = nouveau_pm_acpi_event;
512 register_acpi_notifier(&pm->acpi_nb);
513#endif
488 514
489 return 0; 515 return 0;
490} 516}
@@ -503,6 +529,9 @@ nouveau_pm_fini(struct drm_device *dev)
503 nouveau_perf_fini(dev); 529 nouveau_perf_fini(dev);
504 nouveau_volt_fini(dev); 530 nouveau_volt_fini(dev);
505 531
532#ifdef CONFIG_ACPI
533 unregister_acpi_notifier(&pm->acpi_nb);
534#endif
506 nouveau_hwmon_fini(dev); 535 nouveau_hwmon_fini(dev);
507 nouveau_sysfs_fini(dev); 536 nouveau_sysfs_fini(dev);
508} 537}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.c b/drivers/gpu/drm/nouveau/nouveau_ramht.c
index 2d8580927ca4..bef3e6910418 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ramht.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ramht.c
@@ -104,17 +104,17 @@ nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
104 nouveau_gpuobj_ref(gpuobj, &entry->gpuobj); 104 nouveau_gpuobj_ref(gpuobj, &entry->gpuobj);
105 105
106 if (dev_priv->card_type < NV_40) { 106 if (dev_priv->card_type < NV_40) {
107 ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->cinst >> 4) | 107 ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->pinst >> 4) |
108 (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) | 108 (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
109 (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT); 109 (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
110 } else 110 } else
111 if (dev_priv->card_type < NV_50) { 111 if (dev_priv->card_type < NV_50) {
112 ctx = (gpuobj->cinst >> 4) | 112 ctx = (gpuobj->pinst >> 4) |
113 (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) | 113 (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
114 (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT); 114 (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
115 } else { 115 } else {
116 if (gpuobj->engine == NVOBJ_ENGINE_DISPLAY) { 116 if (gpuobj->engine == NVOBJ_ENGINE_DISPLAY) {
117 ctx = (gpuobj->cinst << 10) | 2; 117 ctx = (gpuobj->cinst << 10) | chan->id;
118 } else { 118 } else {
119 ctx = (gpuobj->cinst >> 4) | 119 ctx = (gpuobj->cinst >> 4) |
120 ((gpuobj->engine << 120 ((gpuobj->engine <<
@@ -214,18 +214,19 @@ out:
214 spin_unlock_irqrestore(&chan->ramht->lock, flags); 214 spin_unlock_irqrestore(&chan->ramht->lock, flags);
215} 215}
216 216
217void 217int
218nouveau_ramht_remove(struct nouveau_channel *chan, u32 handle) 218nouveau_ramht_remove(struct nouveau_channel *chan, u32 handle)
219{ 219{
220 struct nouveau_ramht_entry *entry; 220 struct nouveau_ramht_entry *entry;
221 221
222 entry = nouveau_ramht_remove_entry(chan, handle); 222 entry = nouveau_ramht_remove_entry(chan, handle);
223 if (!entry) 223 if (!entry)
224 return; 224 return -ENOENT;
225 225
226 nouveau_ramht_remove_hash(chan, entry->handle); 226 nouveau_ramht_remove_hash(chan, entry->handle);
227 nouveau_gpuobj_ref(NULL, &entry->gpuobj); 227 nouveau_gpuobj_ref(NULL, &entry->gpuobj);
228 kfree(entry); 228 kfree(entry);
229 return 0;
229} 230}
230 231
231struct nouveau_gpuobj * 232struct nouveau_gpuobj *
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.h b/drivers/gpu/drm/nouveau/nouveau_ramht.h
index b79cb5e1a8f1..c82de98fee0e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ramht.h
+++ b/drivers/gpu/drm/nouveau/nouveau_ramht.h
@@ -48,7 +48,7 @@ extern void nouveau_ramht_ref(struct nouveau_ramht *, struct nouveau_ramht **,
48 48
49extern int nouveau_ramht_insert(struct nouveau_channel *, u32 handle, 49extern int nouveau_ramht_insert(struct nouveau_channel *, u32 handle,
50 struct nouveau_gpuobj *); 50 struct nouveau_gpuobj *);
51extern void nouveau_ramht_remove(struct nouveau_channel *, u32 handle); 51extern int nouveau_ramht_remove(struct nouveau_channel *, u32 handle);
52extern struct nouveau_gpuobj * 52extern struct nouveau_gpuobj *
53nouveau_ramht_find(struct nouveau_channel *chan, u32 handle); 53nouveau_ramht_find(struct nouveau_channel *chan, u32 handle);
54 54
diff --git a/drivers/gpu/drm/nouveau/nouveau_reg.h b/drivers/gpu/drm/nouveau/nouveau_reg.h
index 1b42541ca9e5..04e8fb795269 100644
--- a/drivers/gpu/drm/nouveau/nouveau_reg.h
+++ b/drivers/gpu/drm/nouveau/nouveau_reg.h
@@ -45,6 +45,11 @@
45# define NV04_PFB_REF_CMD_REFRESH (1 << 0) 45# define NV04_PFB_REF_CMD_REFRESH (1 << 0)
46#define NV04_PFB_PRE 0x001002d4 46#define NV04_PFB_PRE 0x001002d4
47# define NV04_PFB_PRE_CMD_PRECHARGE (1 << 0) 47# define NV04_PFB_PRE_CMD_PRECHARGE (1 << 0)
48#define NV20_PFB_ZCOMP(i) (0x00100300 + 4*(i))
49# define NV20_PFB_ZCOMP_MODE_32 (4 << 24)
50# define NV20_PFB_ZCOMP_EN (1 << 31)
51# define NV25_PFB_ZCOMP_MODE_16 (1 << 20)
52# define NV25_PFB_ZCOMP_MODE_32 (2 << 20)
48#define NV10_PFB_CLOSE_PAGE2 0x0010033c 53#define NV10_PFB_CLOSE_PAGE2 0x0010033c
49#define NV04_PFB_SCRAMBLE(i) (0x00100400 + 4 * (i)) 54#define NV04_PFB_SCRAMBLE(i) (0x00100400 + 4 * (i))
50#define NV40_PFB_TILE(i) (0x00100600 + (i*16)) 55#define NV40_PFB_TILE(i) (0x00100600 + (i*16))
@@ -74,17 +79,6 @@
74# define NV40_RAMHT_CONTEXT_ENGINE_SHIFT 20 79# define NV40_RAMHT_CONTEXT_ENGINE_SHIFT 20
75# define NV40_RAMHT_CONTEXT_INSTANCE_SHIFT 0 80# define NV40_RAMHT_CONTEXT_INSTANCE_SHIFT 0
76 81
77/* DMA object defines */
78#define NV_DMA_ACCESS_RW 0
79#define NV_DMA_ACCESS_RO 1
80#define NV_DMA_ACCESS_WO 2
81#define NV_DMA_TARGET_VIDMEM 0
82#define NV_DMA_TARGET_PCI 2
83#define NV_DMA_TARGET_AGP 3
84/* The following is not a real value used by the card, it's changed by
85 * nouveau_object_dma_create */
86#define NV_DMA_TARGET_PCI_NONLINEAR 8
87
88/* Some object classes we care about in the drm */ 82/* Some object classes we care about in the drm */
89#define NV_CLASS_DMA_FROM_MEMORY 0x00000002 83#define NV_CLASS_DMA_FROM_MEMORY 0x00000002
90#define NV_CLASS_DMA_TO_MEMORY 0x00000003 84#define NV_CLASS_DMA_TO_MEMORY 0x00000003
@@ -332,6 +326,7 @@
332#define NV04_PGRAPH_BSWIZZLE5 0x004006A0 326#define NV04_PGRAPH_BSWIZZLE5 0x004006A0
333#define NV03_PGRAPH_STATUS 0x004006B0 327#define NV03_PGRAPH_STATUS 0x004006B0
334#define NV04_PGRAPH_STATUS 0x00400700 328#define NV04_PGRAPH_STATUS 0x00400700
329# define NV40_PGRAPH_STATUS_SYNC_STALL 0x00004000
335#define NV04_PGRAPH_TRAPPED_ADDR 0x00400704 330#define NV04_PGRAPH_TRAPPED_ADDR 0x00400704
336#define NV04_PGRAPH_TRAPPED_DATA 0x00400708 331#define NV04_PGRAPH_TRAPPED_DATA 0x00400708
337#define NV04_PGRAPH_SURFACE 0x0040070C 332#define NV04_PGRAPH_SURFACE 0x0040070C
@@ -378,6 +373,7 @@
378#define NV20_PGRAPH_TLIMIT(i) (0x00400904 + (i*16)) 373#define NV20_PGRAPH_TLIMIT(i) (0x00400904 + (i*16))
379#define NV20_PGRAPH_TSIZE(i) (0x00400908 + (i*16)) 374#define NV20_PGRAPH_TSIZE(i) (0x00400908 + (i*16))
380#define NV20_PGRAPH_TSTATUS(i) (0x0040090C + (i*16)) 375#define NV20_PGRAPH_TSTATUS(i) (0x0040090C + (i*16))
376#define NV20_PGRAPH_ZCOMP(i) (0x00400980 + 4*(i))
381#define NV10_PGRAPH_TILE(i) (0x00400B00 + (i*16)) 377#define NV10_PGRAPH_TILE(i) (0x00400B00 + (i*16))
382#define NV10_PGRAPH_TLIMIT(i) (0x00400B04 + (i*16)) 378#define NV10_PGRAPH_TLIMIT(i) (0x00400B04 + (i*16))
383#define NV10_PGRAPH_TSIZE(i) (0x00400B08 + (i*16)) 379#define NV10_PGRAPH_TSIZE(i) (0x00400B08 + (i*16))
@@ -714,31 +710,32 @@
714#define NV50_PDISPLAY_INTR_1_CLK_UNK10 0x00000010 710#define NV50_PDISPLAY_INTR_1_CLK_UNK10 0x00000010
715#define NV50_PDISPLAY_INTR_1_CLK_UNK20 0x00000020 711#define NV50_PDISPLAY_INTR_1_CLK_UNK20 0x00000020
716#define NV50_PDISPLAY_INTR_1_CLK_UNK40 0x00000040 712#define NV50_PDISPLAY_INTR_1_CLK_UNK40 0x00000040
717#define NV50_PDISPLAY_INTR_EN 0x0061002c 713#define NV50_PDISPLAY_INTR_EN_0 0x00610028
718#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC 0x0000000c 714#define NV50_PDISPLAY_INTR_EN_1 0x0061002c
719#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(n) (1 << ((n) + 2)) 715#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC 0x0000000c
720#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_0 0x00000004 716#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(n) (1 << ((n) + 2))
721#define NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_1 0x00000008 717#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_0 0x00000004
722#define NV50_PDISPLAY_INTR_EN_CLK_UNK10 0x00000010 718#define NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_1 0x00000008
723#define NV50_PDISPLAY_INTR_EN_CLK_UNK20 0x00000020 719#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK10 0x00000010
724#define NV50_PDISPLAY_INTR_EN_CLK_UNK40 0x00000040 720#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK20 0x00000020
721#define NV50_PDISPLAY_INTR_EN_1_CLK_UNK40 0x00000040
725#define NV50_PDISPLAY_UNK30_CTRL 0x00610030 722#define NV50_PDISPLAY_UNK30_CTRL 0x00610030
726#define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK0 0x00000200 723#define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK0 0x00000200
727#define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK1 0x00000400 724#define NV50_PDISPLAY_UNK30_CTRL_UPDATE_VCLK1 0x00000400
728#define NV50_PDISPLAY_UNK30_CTRL_PENDING 0x80000000 725#define NV50_PDISPLAY_UNK30_CTRL_PENDING 0x80000000
729#define NV50_PDISPLAY_TRAPPED_ADDR 0x00610080 726#define NV50_PDISPLAY_TRAPPED_ADDR(i) ((i) * 0x08 + 0x00610080)
730#define NV50_PDISPLAY_TRAPPED_DATA 0x00610084 727#define NV50_PDISPLAY_TRAPPED_DATA(i) ((i) * 0x08 + 0x00610084)
731#define NV50_PDISPLAY_CHANNEL_STAT(i) ((i) * 0x10 + 0x00610200) 728#define NV50_PDISPLAY_EVO_CTRL(i) ((i) * 0x10 + 0x00610200)
732#define NV50_PDISPLAY_CHANNEL_STAT_DMA 0x00000010 729#define NV50_PDISPLAY_EVO_CTRL_DMA 0x00000010
733#define NV50_PDISPLAY_CHANNEL_STAT_DMA_DISABLED 0x00000000 730#define NV50_PDISPLAY_EVO_CTRL_DMA_DISABLED 0x00000000
734#define NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED 0x00000010 731#define NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED 0x00000010
735#define NV50_PDISPLAY_CHANNEL_DMA_CB(i) ((i) * 0x10 + 0x00610204) 732#define NV50_PDISPLAY_EVO_DMA_CB(i) ((i) * 0x10 + 0x00610204)
736#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION 0x00000002 733#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION 0x00000002
737#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_VRAM 0x00000000 734#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION_VRAM 0x00000000
738#define NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_SYSTEM 0x00000002 735#define NV50_PDISPLAY_EVO_DMA_CB_LOCATION_SYSTEM 0x00000002
739#define NV50_PDISPLAY_CHANNEL_DMA_CB_VALID 0x00000001 736#define NV50_PDISPLAY_EVO_DMA_CB_VALID 0x00000001
740#define NV50_PDISPLAY_CHANNEL_UNK2(i) ((i) * 0x10 + 0x00610208) 737#define NV50_PDISPLAY_EVO_UNK2(i) ((i) * 0x10 + 0x00610208)
741#define NV50_PDISPLAY_CHANNEL_UNK3(i) ((i) * 0x10 + 0x0061020c) 738#define NV50_PDISPLAY_EVO_HASH_TAG(i) ((i) * 0x10 + 0x0061020c)
742 739
743#define NV50_PDISPLAY_CURSOR 0x00610270 740#define NV50_PDISPLAY_CURSOR 0x00610270
744#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i) ((i) * 0x10 + 0x00610270) 741#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i) ((i) * 0x10 + 0x00610270)
@@ -746,15 +743,11 @@
746#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS 0x00030000 743#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS 0x00030000
747#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS_ACTIVE 0x00010000 744#define NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS_ACTIVE 0x00010000
748 745
749#define NV50_PDISPLAY_CTRL_STATE 0x00610300 746#define NV50_PDISPLAY_PIO_CTRL 0x00610300
750#define NV50_PDISPLAY_CTRL_STATE_PENDING 0x80000000 747#define NV50_PDISPLAY_PIO_CTRL_PENDING 0x80000000
751#define NV50_PDISPLAY_CTRL_STATE_METHOD 0x00001ffc 748#define NV50_PDISPLAY_PIO_CTRL_MTHD 0x00001ffc
752#define NV50_PDISPLAY_CTRL_STATE_ENABLE 0x00000001 749#define NV50_PDISPLAY_PIO_CTRL_ENABLED 0x00000001
753#define NV50_PDISPLAY_CTRL_VAL 0x00610304 750#define NV50_PDISPLAY_PIO_DATA 0x00610304
754#define NV50_PDISPLAY_UNK_380 0x00610380
755#define NV50_PDISPLAY_RAM_AMOUNT 0x00610384
756#define NV50_PDISPLAY_UNK_388 0x00610388
757#define NV50_PDISPLAY_UNK_38C 0x0061038c
758 751
759#define NV50_PDISPLAY_CRTC_P(i, r) ((i) * 0x540 + NV50_PDISPLAY_CRTC_##r) 752#define NV50_PDISPLAY_CRTC_P(i, r) ((i) * 0x540 + NV50_PDISPLAY_CRTC_##r)
760#define NV50_PDISPLAY_CRTC_C(i, r) (4 + (i) * 0x540 + NV50_PDISPLAY_CRTC_##r) 753#define NV50_PDISPLAY_CRTC_C(i, r) (4 + (i) * 0x540 + NV50_PDISPLAY_CRTC_##r)
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index d4ac97007038..9a250eb53098 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -14,7 +14,7 @@ struct nouveau_sgdma_be {
14 dma_addr_t *pages; 14 dma_addr_t *pages;
15 unsigned nr_pages; 15 unsigned nr_pages;
16 16
17 unsigned pte_start; 17 u64 offset;
18 bool bound; 18 bool bound;
19}; 19};
20 20
@@ -74,18 +74,6 @@ nouveau_sgdma_clear(struct ttm_backend *be)
74 } 74 }
75} 75}
76 76
77static inline unsigned
78nouveau_sgdma_pte(struct drm_device *dev, uint64_t offset)
79{
80 struct drm_nouveau_private *dev_priv = dev->dev_private;
81 unsigned pte = (offset >> NV_CTXDMA_PAGE_SHIFT);
82
83 if (dev_priv->card_type < NV_50)
84 return pte + 2;
85
86 return pte << 1;
87}
88
89static int 77static int
90nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem) 78nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
91{ 79{
@@ -97,32 +85,17 @@ nouveau_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
97 85
98 NV_DEBUG(dev, "pg=0x%lx\n", mem->start); 86 NV_DEBUG(dev, "pg=0x%lx\n", mem->start);
99 87
100 pte = nouveau_sgdma_pte(nvbe->dev, mem->start << PAGE_SHIFT); 88 nvbe->offset = mem->start << PAGE_SHIFT;
101 nvbe->pte_start = pte; 89 pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
102 for (i = 0; i < nvbe->nr_pages; i++) { 90 for (i = 0; i < nvbe->nr_pages; i++) {
103 dma_addr_t dma_offset = nvbe->pages[i]; 91 dma_addr_t dma_offset = nvbe->pages[i];
104 uint32_t offset_l = lower_32_bits(dma_offset); 92 uint32_t offset_l = lower_32_bits(dma_offset);
105 uint32_t offset_h = upper_32_bits(dma_offset);
106
107 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++) {
108 if (dev_priv->card_type < NV_50) {
109 nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
110 pte += 1;
111 } else {
112 nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 0x21);
113 nv_wo32(gpuobj, (pte * 4) + 4, offset_h & 0xff);
114 pte += 2;
115 }
116 93
94 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
95 nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
117 dma_offset += NV_CTXDMA_PAGE_SIZE; 96 dma_offset += NV_CTXDMA_PAGE_SIZE;
118 } 97 }
119 } 98 }
120 dev_priv->engine.instmem.flush(nvbe->dev);
121
122 if (dev_priv->card_type == NV_50) {
123 dev_priv->engine.fifo.tlb_flush(dev);
124 dev_priv->engine.graph.tlb_flush(dev);
125 }
126 99
127 nvbe->bound = true; 100 nvbe->bound = true;
128 return 0; 101 return 0;
@@ -142,28 +115,10 @@ nouveau_sgdma_unbind(struct ttm_backend *be)
142 if (!nvbe->bound) 115 if (!nvbe->bound)
143 return 0; 116 return 0;
144 117
145 pte = nvbe->pte_start; 118 pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
146 for (i = 0; i < nvbe->nr_pages; i++) { 119 for (i = 0; i < nvbe->nr_pages; i++) {
147 dma_addr_t dma_offset = dev_priv->gart_info.sg_dummy_bus; 120 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
148 121 nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
149 for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++) {
150 if (dev_priv->card_type < NV_50) {
151 nv_wo32(gpuobj, (pte * 4) + 0, dma_offset | 3);
152 pte += 1;
153 } else {
154 nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
155 nv_wo32(gpuobj, (pte * 4) + 4, 0x00000000);
156 pte += 2;
157 }
158
159 dma_offset += NV_CTXDMA_PAGE_SIZE;
160 }
161 }
162 dev_priv->engine.instmem.flush(nvbe->dev);
163
164 if (dev_priv->card_type == NV_50) {
165 dev_priv->engine.fifo.tlb_flush(dev);
166 dev_priv->engine.graph.tlb_flush(dev);
167 } 122 }
168 123
169 nvbe->bound = false; 124 nvbe->bound = false;
@@ -186,6 +141,35 @@ nouveau_sgdma_destroy(struct ttm_backend *be)
186 } 141 }
187} 142}
188 143
144static int
145nv50_sgdma_bind(struct ttm_backend *be, struct ttm_mem_reg *mem)
146{
147 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
148 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
149
150 nvbe->offset = mem->start << PAGE_SHIFT;
151
152 nouveau_vm_map_sg(&dev_priv->gart_info.vma, nvbe->offset,
153 nvbe->nr_pages << PAGE_SHIFT, nvbe->pages);
154 nvbe->bound = true;
155 return 0;
156}
157
158static int
159nv50_sgdma_unbind(struct ttm_backend *be)
160{
161 struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)be;
162 struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
163
164 if (!nvbe->bound)
165 return 0;
166
167 nouveau_vm_unmap_at(&dev_priv->gart_info.vma, nvbe->offset,
168 nvbe->nr_pages << PAGE_SHIFT);
169 nvbe->bound = false;
170 return 0;
171}
172
189static struct ttm_backend_func nouveau_sgdma_backend = { 173static struct ttm_backend_func nouveau_sgdma_backend = {
190 .populate = nouveau_sgdma_populate, 174 .populate = nouveau_sgdma_populate,
191 .clear = nouveau_sgdma_clear, 175 .clear = nouveau_sgdma_clear,
@@ -194,23 +178,30 @@ static struct ttm_backend_func nouveau_sgdma_backend = {
194 .destroy = nouveau_sgdma_destroy 178 .destroy = nouveau_sgdma_destroy
195}; 179};
196 180
181static struct ttm_backend_func nv50_sgdma_backend = {
182 .populate = nouveau_sgdma_populate,
183 .clear = nouveau_sgdma_clear,
184 .bind = nv50_sgdma_bind,
185 .unbind = nv50_sgdma_unbind,
186 .destroy = nouveau_sgdma_destroy
187};
188
197struct ttm_backend * 189struct ttm_backend *
198nouveau_sgdma_init_ttm(struct drm_device *dev) 190nouveau_sgdma_init_ttm(struct drm_device *dev)
199{ 191{
200 struct drm_nouveau_private *dev_priv = dev->dev_private; 192 struct drm_nouveau_private *dev_priv = dev->dev_private;
201 struct nouveau_sgdma_be *nvbe; 193 struct nouveau_sgdma_be *nvbe;
202 194
203 if (!dev_priv->gart_info.sg_ctxdma)
204 return NULL;
205
206 nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL); 195 nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
207 if (!nvbe) 196 if (!nvbe)
208 return NULL; 197 return NULL;
209 198
210 nvbe->dev = dev; 199 nvbe->dev = dev;
211 200
212 nvbe->backend.func = &nouveau_sgdma_backend; 201 if (dev_priv->card_type < NV_50)
213 202 nvbe->backend.func = &nouveau_sgdma_backend;
203 else
204 nvbe->backend.func = &nv50_sgdma_backend;
214 return &nvbe->backend; 205 return &nvbe->backend;
215} 206}
216 207
@@ -218,7 +209,6 @@ int
218nouveau_sgdma_init(struct drm_device *dev) 209nouveau_sgdma_init(struct drm_device *dev)
219{ 210{
220 struct drm_nouveau_private *dev_priv = dev->dev_private; 211 struct drm_nouveau_private *dev_priv = dev->dev_private;
221 struct pci_dev *pdev = dev->pdev;
222 struct nouveau_gpuobj *gpuobj = NULL; 212 struct nouveau_gpuobj *gpuobj = NULL;
223 uint32_t aper_size, obj_size; 213 uint32_t aper_size, obj_size;
224 int i, ret; 214 int i, ret;
@@ -231,68 +221,40 @@ nouveau_sgdma_init(struct drm_device *dev)
231 221
232 obj_size = (aper_size >> NV_CTXDMA_PAGE_SHIFT) * 4; 222 obj_size = (aper_size >> NV_CTXDMA_PAGE_SHIFT) * 4;
233 obj_size += 8; /* ctxdma header */ 223 obj_size += 8; /* ctxdma header */
234 } else {
235 /* 1 entire VM page table */
236 aper_size = (512 * 1024 * 1024);
237 obj_size = (aper_size >> NV_CTXDMA_PAGE_SHIFT) * 8;
238 }
239
240 ret = nouveau_gpuobj_new(dev, NULL, obj_size, 16,
241 NVOBJ_FLAG_ZERO_ALLOC |
242 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
243 if (ret) {
244 NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
245 return ret;
246 }
247
248 dev_priv->gart_info.sg_dummy_page =
249 alloc_page(GFP_KERNEL|__GFP_DMA32|__GFP_ZERO);
250 if (!dev_priv->gart_info.sg_dummy_page) {
251 nouveau_gpuobj_ref(NULL, &gpuobj);
252 return -ENOMEM;
253 }
254 224
255 set_bit(PG_locked, &dev_priv->gart_info.sg_dummy_page->flags); 225 ret = nouveau_gpuobj_new(dev, NULL, obj_size, 16,
256 dev_priv->gart_info.sg_dummy_bus = 226 NVOBJ_FLAG_ZERO_ALLOC |
257 pci_map_page(pdev, dev_priv->gart_info.sg_dummy_page, 0, 227 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
258 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 228 if (ret) {
259 if (pci_dma_mapping_error(pdev, dev_priv->gart_info.sg_dummy_bus)) { 229 NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
260 nouveau_gpuobj_ref(NULL, &gpuobj); 230 return ret;
261 return -EFAULT; 231 }
262 }
263 232
264 if (dev_priv->card_type < NV_50) {
265 /* special case, allocated from global instmem heap so
266 * cinst is invalid, we use it on all channels though so
267 * cinst needs to be valid, set it the same as pinst
268 */
269 gpuobj->cinst = gpuobj->pinst;
270
271 /* Maybe use NV_DMA_TARGET_AGP for PCIE? NVIDIA do this, and
272 * confirmed to work on c51. Perhaps means NV_DMA_TARGET_PCIE
273 * on those cards? */
274 nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY | 233 nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
275 (1 << 12) /* PT present */ | 234 (1 << 12) /* PT present */ |
276 (0 << 13) /* PT *not* linear */ | 235 (0 << 13) /* PT *not* linear */ |
277 (NV_DMA_ACCESS_RW << 14) | 236 (0 << 14) /* RW */ |
278 (NV_DMA_TARGET_PCI << 16)); 237 (2 << 16) /* PCI */);
279 nv_wo32(gpuobj, 4, aper_size - 1); 238 nv_wo32(gpuobj, 4, aper_size - 1);
280 for (i = 2; i < 2 + (aper_size >> 12); i++) { 239 for (i = 2; i < 2 + (aper_size >> 12); i++)
281 nv_wo32(gpuobj, i * 4, 240 nv_wo32(gpuobj, i * 4, 0x00000000);
282 dev_priv->gart_info.sg_dummy_bus | 3); 241
283 } 242 dev_priv->gart_info.sg_ctxdma = gpuobj;
284 } else { 243 dev_priv->gart_info.aper_base = 0;
285 for (i = 0; i < obj_size; i += 8) { 244 dev_priv->gart_info.aper_size = aper_size;
286 nv_wo32(gpuobj, i + 0, 0x00000000); 245 } else
287 nv_wo32(gpuobj, i + 4, 0x00000000); 246 if (dev_priv->chan_vm) {
288 } 247 ret = nouveau_vm_get(dev_priv->chan_vm, 512 * 1024 * 1024,
248 12, NV_MEM_ACCESS_RW,
249 &dev_priv->gart_info.vma);
250 if (ret)
251 return ret;
252
253 dev_priv->gart_info.aper_base = dev_priv->gart_info.vma.offset;
254 dev_priv->gart_info.aper_size = 512 * 1024 * 1024;
289 } 255 }
290 dev_priv->engine.instmem.flush(dev);
291 256
292 dev_priv->gart_info.type = NOUVEAU_GART_SGDMA; 257 dev_priv->gart_info.type = NOUVEAU_GART_SGDMA;
293 dev_priv->gart_info.aper_base = 0;
294 dev_priv->gart_info.aper_size = aper_size;
295 dev_priv->gart_info.sg_ctxdma = gpuobj;
296 return 0; 258 return 0;
297} 259}
298 260
@@ -301,31 +263,19 @@ nouveau_sgdma_takedown(struct drm_device *dev)
301{ 263{
302 struct drm_nouveau_private *dev_priv = dev->dev_private; 264 struct drm_nouveau_private *dev_priv = dev->dev_private;
303 265
304 if (dev_priv->gart_info.sg_dummy_page) {
305 pci_unmap_page(dev->pdev, dev_priv->gart_info.sg_dummy_bus,
306 NV_CTXDMA_PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
307 unlock_page(dev_priv->gart_info.sg_dummy_page);
308 __free_page(dev_priv->gart_info.sg_dummy_page);
309 dev_priv->gart_info.sg_dummy_page = NULL;
310 dev_priv->gart_info.sg_dummy_bus = 0;
311 }
312
313 nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma); 266 nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);
267 nouveau_vm_put(&dev_priv->gart_info.vma);
314} 268}
315 269
316int 270uint32_t
317nouveau_sgdma_get_page(struct drm_device *dev, uint32_t offset, uint32_t *page) 271nouveau_sgdma_get_physical(struct drm_device *dev, uint32_t offset)
318{ 272{
319 struct drm_nouveau_private *dev_priv = dev->dev_private; 273 struct drm_nouveau_private *dev_priv = dev->dev_private;
320 struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma; 274 struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
321 int pte; 275 int pte = (offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
322 276
323 pte = (offset >> NV_CTXDMA_PAGE_SHIFT) << 2; 277 BUG_ON(dev_priv->card_type >= NV_50);
324 if (dev_priv->card_type < NV_50) {
325 *page = nv_ro32(gpuobj, (pte + 8)) & ~NV_CTXDMA_PAGE_MASK;
326 return 0;
327 }
328 278
329 NV_ERROR(dev, "Unimplemented on NV50\n"); 279 return (nv_ro32(gpuobj, 4 * pte) & ~NV_CTXDMA_PAGE_MASK) |
330 return -EINVAL; 280 (offset & NV_CTXDMA_PAGE_MASK);
331} 281}
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
index 049f755567e5..a54fc431fe98 100644
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ b/drivers/gpu/drm/nouveau/nouveau_state.c
@@ -53,10 +53,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
53 engine->instmem.takedown = nv04_instmem_takedown; 53 engine->instmem.takedown = nv04_instmem_takedown;
54 engine->instmem.suspend = nv04_instmem_suspend; 54 engine->instmem.suspend = nv04_instmem_suspend;
55 engine->instmem.resume = nv04_instmem_resume; 55 engine->instmem.resume = nv04_instmem_resume;
56 engine->instmem.populate = nv04_instmem_populate; 56 engine->instmem.get = nv04_instmem_get;
57 engine->instmem.clear = nv04_instmem_clear; 57 engine->instmem.put = nv04_instmem_put;
58 engine->instmem.bind = nv04_instmem_bind; 58 engine->instmem.map = nv04_instmem_map;
59 engine->instmem.unbind = nv04_instmem_unbind; 59 engine->instmem.unmap = nv04_instmem_unmap;
60 engine->instmem.flush = nv04_instmem_flush; 60 engine->instmem.flush = nv04_instmem_flush;
61 engine->mc.init = nv04_mc_init; 61 engine->mc.init = nv04_mc_init;
62 engine->mc.takedown = nv04_mc_takedown; 62 engine->mc.takedown = nv04_mc_takedown;
@@ -65,7 +65,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
65 engine->timer.takedown = nv04_timer_takedown; 65 engine->timer.takedown = nv04_timer_takedown;
66 engine->fb.init = nv04_fb_init; 66 engine->fb.init = nv04_fb_init;
67 engine->fb.takedown = nv04_fb_takedown; 67 engine->fb.takedown = nv04_fb_takedown;
68 engine->graph.grclass = nv04_graph_grclass;
69 engine->graph.init = nv04_graph_init; 68 engine->graph.init = nv04_graph_init;
70 engine->graph.takedown = nv04_graph_takedown; 69 engine->graph.takedown = nv04_graph_takedown;
71 engine->graph.fifo_access = nv04_graph_fifo_access; 70 engine->graph.fifo_access = nv04_graph_fifo_access;
@@ -76,7 +75,7 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
76 engine->graph.unload_context = nv04_graph_unload_context; 75 engine->graph.unload_context = nv04_graph_unload_context;
77 engine->fifo.channels = 16; 76 engine->fifo.channels = 16;
78 engine->fifo.init = nv04_fifo_init; 77 engine->fifo.init = nv04_fifo_init;
79 engine->fifo.takedown = nouveau_stub_takedown; 78 engine->fifo.takedown = nv04_fifo_fini;
80 engine->fifo.disable = nv04_fifo_disable; 79 engine->fifo.disable = nv04_fifo_disable;
81 engine->fifo.enable = nv04_fifo_enable; 80 engine->fifo.enable = nv04_fifo_enable;
82 engine->fifo.reassign = nv04_fifo_reassign; 81 engine->fifo.reassign = nv04_fifo_reassign;
@@ -99,16 +98,20 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
99 engine->pm.clock_get = nv04_pm_clock_get; 98 engine->pm.clock_get = nv04_pm_clock_get;
100 engine->pm.clock_pre = nv04_pm_clock_pre; 99 engine->pm.clock_pre = nv04_pm_clock_pre;
101 engine->pm.clock_set = nv04_pm_clock_set; 100 engine->pm.clock_set = nv04_pm_clock_set;
101 engine->crypt.init = nouveau_stub_init;
102 engine->crypt.takedown = nouveau_stub_takedown;
103 engine->vram.init = nouveau_mem_detect;
104 engine->vram.flags_valid = nouveau_mem_flags_valid;
102 break; 105 break;
103 case 0x10: 106 case 0x10:
104 engine->instmem.init = nv04_instmem_init; 107 engine->instmem.init = nv04_instmem_init;
105 engine->instmem.takedown = nv04_instmem_takedown; 108 engine->instmem.takedown = nv04_instmem_takedown;
106 engine->instmem.suspend = nv04_instmem_suspend; 109 engine->instmem.suspend = nv04_instmem_suspend;
107 engine->instmem.resume = nv04_instmem_resume; 110 engine->instmem.resume = nv04_instmem_resume;
108 engine->instmem.populate = nv04_instmem_populate; 111 engine->instmem.get = nv04_instmem_get;
109 engine->instmem.clear = nv04_instmem_clear; 112 engine->instmem.put = nv04_instmem_put;
110 engine->instmem.bind = nv04_instmem_bind; 113 engine->instmem.map = nv04_instmem_map;
111 engine->instmem.unbind = nv04_instmem_unbind; 114 engine->instmem.unmap = nv04_instmem_unmap;
112 engine->instmem.flush = nv04_instmem_flush; 115 engine->instmem.flush = nv04_instmem_flush;
113 engine->mc.init = nv04_mc_init; 116 engine->mc.init = nv04_mc_init;
114 engine->mc.takedown = nv04_mc_takedown; 117 engine->mc.takedown = nv04_mc_takedown;
@@ -117,8 +120,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
117 engine->timer.takedown = nv04_timer_takedown; 120 engine->timer.takedown = nv04_timer_takedown;
118 engine->fb.init = nv10_fb_init; 121 engine->fb.init = nv10_fb_init;
119 engine->fb.takedown = nv10_fb_takedown; 122 engine->fb.takedown = nv10_fb_takedown;
120 engine->fb.set_region_tiling = nv10_fb_set_region_tiling; 123 engine->fb.init_tile_region = nv10_fb_init_tile_region;
121 engine->graph.grclass = nv10_graph_grclass; 124 engine->fb.set_tile_region = nv10_fb_set_tile_region;
125 engine->fb.free_tile_region = nv10_fb_free_tile_region;
122 engine->graph.init = nv10_graph_init; 126 engine->graph.init = nv10_graph_init;
123 engine->graph.takedown = nv10_graph_takedown; 127 engine->graph.takedown = nv10_graph_takedown;
124 engine->graph.channel = nv10_graph_channel; 128 engine->graph.channel = nv10_graph_channel;
@@ -127,17 +131,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
127 engine->graph.fifo_access = nv04_graph_fifo_access; 131 engine->graph.fifo_access = nv04_graph_fifo_access;
128 engine->graph.load_context = nv10_graph_load_context; 132 engine->graph.load_context = nv10_graph_load_context;
129 engine->graph.unload_context = nv10_graph_unload_context; 133 engine->graph.unload_context = nv10_graph_unload_context;
130 engine->graph.set_region_tiling = nv10_graph_set_region_tiling; 134 engine->graph.set_tile_region = nv10_graph_set_tile_region;
131 engine->fifo.channels = 32; 135 engine->fifo.channels = 32;
132 engine->fifo.init = nv10_fifo_init; 136 engine->fifo.init = nv10_fifo_init;
133 engine->fifo.takedown = nouveau_stub_takedown; 137 engine->fifo.takedown = nv04_fifo_fini;
134 engine->fifo.disable = nv04_fifo_disable; 138 engine->fifo.disable = nv04_fifo_disable;
135 engine->fifo.enable = nv04_fifo_enable; 139 engine->fifo.enable = nv04_fifo_enable;
136 engine->fifo.reassign = nv04_fifo_reassign; 140 engine->fifo.reassign = nv04_fifo_reassign;
137 engine->fifo.cache_pull = nv04_fifo_cache_pull; 141 engine->fifo.cache_pull = nv04_fifo_cache_pull;
138 engine->fifo.channel_id = nv10_fifo_channel_id; 142 engine->fifo.channel_id = nv10_fifo_channel_id;
139 engine->fifo.create_context = nv10_fifo_create_context; 143 engine->fifo.create_context = nv10_fifo_create_context;
140 engine->fifo.destroy_context = nv10_fifo_destroy_context; 144 engine->fifo.destroy_context = nv04_fifo_destroy_context;
141 engine->fifo.load_context = nv10_fifo_load_context; 145 engine->fifo.load_context = nv10_fifo_load_context;
142 engine->fifo.unload_context = nv10_fifo_unload_context; 146 engine->fifo.unload_context = nv10_fifo_unload_context;
143 engine->display.early_init = nv04_display_early_init; 147 engine->display.early_init = nv04_display_early_init;
@@ -153,16 +157,20 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
153 engine->pm.clock_get = nv04_pm_clock_get; 157 engine->pm.clock_get = nv04_pm_clock_get;
154 engine->pm.clock_pre = nv04_pm_clock_pre; 158 engine->pm.clock_pre = nv04_pm_clock_pre;
155 engine->pm.clock_set = nv04_pm_clock_set; 159 engine->pm.clock_set = nv04_pm_clock_set;
160 engine->crypt.init = nouveau_stub_init;
161 engine->crypt.takedown = nouveau_stub_takedown;
162 engine->vram.init = nouveau_mem_detect;
163 engine->vram.flags_valid = nouveau_mem_flags_valid;
156 break; 164 break;
157 case 0x20: 165 case 0x20:
158 engine->instmem.init = nv04_instmem_init; 166 engine->instmem.init = nv04_instmem_init;
159 engine->instmem.takedown = nv04_instmem_takedown; 167 engine->instmem.takedown = nv04_instmem_takedown;
160 engine->instmem.suspend = nv04_instmem_suspend; 168 engine->instmem.suspend = nv04_instmem_suspend;
161 engine->instmem.resume = nv04_instmem_resume; 169 engine->instmem.resume = nv04_instmem_resume;
162 engine->instmem.populate = nv04_instmem_populate; 170 engine->instmem.get = nv04_instmem_get;
163 engine->instmem.clear = nv04_instmem_clear; 171 engine->instmem.put = nv04_instmem_put;
164 engine->instmem.bind = nv04_instmem_bind; 172 engine->instmem.map = nv04_instmem_map;
165 engine->instmem.unbind = nv04_instmem_unbind; 173 engine->instmem.unmap = nv04_instmem_unmap;
166 engine->instmem.flush = nv04_instmem_flush; 174 engine->instmem.flush = nv04_instmem_flush;
167 engine->mc.init = nv04_mc_init; 175 engine->mc.init = nv04_mc_init;
168 engine->mc.takedown = nv04_mc_takedown; 176 engine->mc.takedown = nv04_mc_takedown;
@@ -171,8 +179,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
171 engine->timer.takedown = nv04_timer_takedown; 179 engine->timer.takedown = nv04_timer_takedown;
172 engine->fb.init = nv10_fb_init; 180 engine->fb.init = nv10_fb_init;
173 engine->fb.takedown = nv10_fb_takedown; 181 engine->fb.takedown = nv10_fb_takedown;
174 engine->fb.set_region_tiling = nv10_fb_set_region_tiling; 182 engine->fb.init_tile_region = nv10_fb_init_tile_region;
175 engine->graph.grclass = nv20_graph_grclass; 183 engine->fb.set_tile_region = nv10_fb_set_tile_region;
184 engine->fb.free_tile_region = nv10_fb_free_tile_region;
176 engine->graph.init = nv20_graph_init; 185 engine->graph.init = nv20_graph_init;
177 engine->graph.takedown = nv20_graph_takedown; 186 engine->graph.takedown = nv20_graph_takedown;
178 engine->graph.channel = nv10_graph_channel; 187 engine->graph.channel = nv10_graph_channel;
@@ -181,17 +190,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
181 engine->graph.fifo_access = nv04_graph_fifo_access; 190 engine->graph.fifo_access = nv04_graph_fifo_access;
182 engine->graph.load_context = nv20_graph_load_context; 191 engine->graph.load_context = nv20_graph_load_context;
183 engine->graph.unload_context = nv20_graph_unload_context; 192 engine->graph.unload_context = nv20_graph_unload_context;
184 engine->graph.set_region_tiling = nv20_graph_set_region_tiling; 193 engine->graph.set_tile_region = nv20_graph_set_tile_region;
185 engine->fifo.channels = 32; 194 engine->fifo.channels = 32;
186 engine->fifo.init = nv10_fifo_init; 195 engine->fifo.init = nv10_fifo_init;
187 engine->fifo.takedown = nouveau_stub_takedown; 196 engine->fifo.takedown = nv04_fifo_fini;
188 engine->fifo.disable = nv04_fifo_disable; 197 engine->fifo.disable = nv04_fifo_disable;
189 engine->fifo.enable = nv04_fifo_enable; 198 engine->fifo.enable = nv04_fifo_enable;
190 engine->fifo.reassign = nv04_fifo_reassign; 199 engine->fifo.reassign = nv04_fifo_reassign;
191 engine->fifo.cache_pull = nv04_fifo_cache_pull; 200 engine->fifo.cache_pull = nv04_fifo_cache_pull;
192 engine->fifo.channel_id = nv10_fifo_channel_id; 201 engine->fifo.channel_id = nv10_fifo_channel_id;
193 engine->fifo.create_context = nv10_fifo_create_context; 202 engine->fifo.create_context = nv10_fifo_create_context;
194 engine->fifo.destroy_context = nv10_fifo_destroy_context; 203 engine->fifo.destroy_context = nv04_fifo_destroy_context;
195 engine->fifo.load_context = nv10_fifo_load_context; 204 engine->fifo.load_context = nv10_fifo_load_context;
196 engine->fifo.unload_context = nv10_fifo_unload_context; 205 engine->fifo.unload_context = nv10_fifo_unload_context;
197 engine->display.early_init = nv04_display_early_init; 206 engine->display.early_init = nv04_display_early_init;
@@ -207,16 +216,20 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
207 engine->pm.clock_get = nv04_pm_clock_get; 216 engine->pm.clock_get = nv04_pm_clock_get;
208 engine->pm.clock_pre = nv04_pm_clock_pre; 217 engine->pm.clock_pre = nv04_pm_clock_pre;
209 engine->pm.clock_set = nv04_pm_clock_set; 218 engine->pm.clock_set = nv04_pm_clock_set;
219 engine->crypt.init = nouveau_stub_init;
220 engine->crypt.takedown = nouveau_stub_takedown;
221 engine->vram.init = nouveau_mem_detect;
222 engine->vram.flags_valid = nouveau_mem_flags_valid;
210 break; 223 break;
211 case 0x30: 224 case 0x30:
212 engine->instmem.init = nv04_instmem_init; 225 engine->instmem.init = nv04_instmem_init;
213 engine->instmem.takedown = nv04_instmem_takedown; 226 engine->instmem.takedown = nv04_instmem_takedown;
214 engine->instmem.suspend = nv04_instmem_suspend; 227 engine->instmem.suspend = nv04_instmem_suspend;
215 engine->instmem.resume = nv04_instmem_resume; 228 engine->instmem.resume = nv04_instmem_resume;
216 engine->instmem.populate = nv04_instmem_populate; 229 engine->instmem.get = nv04_instmem_get;
217 engine->instmem.clear = nv04_instmem_clear; 230 engine->instmem.put = nv04_instmem_put;
218 engine->instmem.bind = nv04_instmem_bind; 231 engine->instmem.map = nv04_instmem_map;
219 engine->instmem.unbind = nv04_instmem_unbind; 232 engine->instmem.unmap = nv04_instmem_unmap;
220 engine->instmem.flush = nv04_instmem_flush; 233 engine->instmem.flush = nv04_instmem_flush;
221 engine->mc.init = nv04_mc_init; 234 engine->mc.init = nv04_mc_init;
222 engine->mc.takedown = nv04_mc_takedown; 235 engine->mc.takedown = nv04_mc_takedown;
@@ -225,8 +238,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
225 engine->timer.takedown = nv04_timer_takedown; 238 engine->timer.takedown = nv04_timer_takedown;
226 engine->fb.init = nv30_fb_init; 239 engine->fb.init = nv30_fb_init;
227 engine->fb.takedown = nv30_fb_takedown; 240 engine->fb.takedown = nv30_fb_takedown;
228 engine->fb.set_region_tiling = nv10_fb_set_region_tiling; 241 engine->fb.init_tile_region = nv30_fb_init_tile_region;
229 engine->graph.grclass = nv30_graph_grclass; 242 engine->fb.set_tile_region = nv10_fb_set_tile_region;
243 engine->fb.free_tile_region = nv30_fb_free_tile_region;
230 engine->graph.init = nv30_graph_init; 244 engine->graph.init = nv30_graph_init;
231 engine->graph.takedown = nv20_graph_takedown; 245 engine->graph.takedown = nv20_graph_takedown;
232 engine->graph.fifo_access = nv04_graph_fifo_access; 246 engine->graph.fifo_access = nv04_graph_fifo_access;
@@ -235,17 +249,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
235 engine->graph.destroy_context = nv20_graph_destroy_context; 249 engine->graph.destroy_context = nv20_graph_destroy_context;
236 engine->graph.load_context = nv20_graph_load_context; 250 engine->graph.load_context = nv20_graph_load_context;
237 engine->graph.unload_context = nv20_graph_unload_context; 251 engine->graph.unload_context = nv20_graph_unload_context;
238 engine->graph.set_region_tiling = nv20_graph_set_region_tiling; 252 engine->graph.set_tile_region = nv20_graph_set_tile_region;
239 engine->fifo.channels = 32; 253 engine->fifo.channels = 32;
240 engine->fifo.init = nv10_fifo_init; 254 engine->fifo.init = nv10_fifo_init;
241 engine->fifo.takedown = nouveau_stub_takedown; 255 engine->fifo.takedown = nv04_fifo_fini;
242 engine->fifo.disable = nv04_fifo_disable; 256 engine->fifo.disable = nv04_fifo_disable;
243 engine->fifo.enable = nv04_fifo_enable; 257 engine->fifo.enable = nv04_fifo_enable;
244 engine->fifo.reassign = nv04_fifo_reassign; 258 engine->fifo.reassign = nv04_fifo_reassign;
245 engine->fifo.cache_pull = nv04_fifo_cache_pull; 259 engine->fifo.cache_pull = nv04_fifo_cache_pull;
246 engine->fifo.channel_id = nv10_fifo_channel_id; 260 engine->fifo.channel_id = nv10_fifo_channel_id;
247 engine->fifo.create_context = nv10_fifo_create_context; 261 engine->fifo.create_context = nv10_fifo_create_context;
248 engine->fifo.destroy_context = nv10_fifo_destroy_context; 262 engine->fifo.destroy_context = nv04_fifo_destroy_context;
249 engine->fifo.load_context = nv10_fifo_load_context; 263 engine->fifo.load_context = nv10_fifo_load_context;
250 engine->fifo.unload_context = nv10_fifo_unload_context; 264 engine->fifo.unload_context = nv10_fifo_unload_context;
251 engine->display.early_init = nv04_display_early_init; 265 engine->display.early_init = nv04_display_early_init;
@@ -263,6 +277,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
263 engine->pm.clock_set = nv04_pm_clock_set; 277 engine->pm.clock_set = nv04_pm_clock_set;
264 engine->pm.voltage_get = nouveau_voltage_gpio_get; 278 engine->pm.voltage_get = nouveau_voltage_gpio_get;
265 engine->pm.voltage_set = nouveau_voltage_gpio_set; 279 engine->pm.voltage_set = nouveau_voltage_gpio_set;
280 engine->crypt.init = nouveau_stub_init;
281 engine->crypt.takedown = nouveau_stub_takedown;
282 engine->vram.init = nouveau_mem_detect;
283 engine->vram.flags_valid = nouveau_mem_flags_valid;
266 break; 284 break;
267 case 0x40: 285 case 0x40:
268 case 0x60: 286 case 0x60:
@@ -270,10 +288,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
270 engine->instmem.takedown = nv04_instmem_takedown; 288 engine->instmem.takedown = nv04_instmem_takedown;
271 engine->instmem.suspend = nv04_instmem_suspend; 289 engine->instmem.suspend = nv04_instmem_suspend;
272 engine->instmem.resume = nv04_instmem_resume; 290 engine->instmem.resume = nv04_instmem_resume;
273 engine->instmem.populate = nv04_instmem_populate; 291 engine->instmem.get = nv04_instmem_get;
274 engine->instmem.clear = nv04_instmem_clear; 292 engine->instmem.put = nv04_instmem_put;
275 engine->instmem.bind = nv04_instmem_bind; 293 engine->instmem.map = nv04_instmem_map;
276 engine->instmem.unbind = nv04_instmem_unbind; 294 engine->instmem.unmap = nv04_instmem_unmap;
277 engine->instmem.flush = nv04_instmem_flush; 295 engine->instmem.flush = nv04_instmem_flush;
278 engine->mc.init = nv40_mc_init; 296 engine->mc.init = nv40_mc_init;
279 engine->mc.takedown = nv40_mc_takedown; 297 engine->mc.takedown = nv40_mc_takedown;
@@ -282,8 +300,9 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
282 engine->timer.takedown = nv04_timer_takedown; 300 engine->timer.takedown = nv04_timer_takedown;
283 engine->fb.init = nv40_fb_init; 301 engine->fb.init = nv40_fb_init;
284 engine->fb.takedown = nv40_fb_takedown; 302 engine->fb.takedown = nv40_fb_takedown;
285 engine->fb.set_region_tiling = nv40_fb_set_region_tiling; 303 engine->fb.init_tile_region = nv30_fb_init_tile_region;
286 engine->graph.grclass = nv40_graph_grclass; 304 engine->fb.set_tile_region = nv40_fb_set_tile_region;
305 engine->fb.free_tile_region = nv30_fb_free_tile_region;
287 engine->graph.init = nv40_graph_init; 306 engine->graph.init = nv40_graph_init;
288 engine->graph.takedown = nv40_graph_takedown; 307 engine->graph.takedown = nv40_graph_takedown;
289 engine->graph.fifo_access = nv04_graph_fifo_access; 308 engine->graph.fifo_access = nv04_graph_fifo_access;
@@ -292,17 +311,17 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
292 engine->graph.destroy_context = nv40_graph_destroy_context; 311 engine->graph.destroy_context = nv40_graph_destroy_context;
293 engine->graph.load_context = nv40_graph_load_context; 312 engine->graph.load_context = nv40_graph_load_context;
294 engine->graph.unload_context = nv40_graph_unload_context; 313 engine->graph.unload_context = nv40_graph_unload_context;
295 engine->graph.set_region_tiling = nv40_graph_set_region_tiling; 314 engine->graph.set_tile_region = nv40_graph_set_tile_region;
296 engine->fifo.channels = 32; 315 engine->fifo.channels = 32;
297 engine->fifo.init = nv40_fifo_init; 316 engine->fifo.init = nv40_fifo_init;
298 engine->fifo.takedown = nouveau_stub_takedown; 317 engine->fifo.takedown = nv04_fifo_fini;
299 engine->fifo.disable = nv04_fifo_disable; 318 engine->fifo.disable = nv04_fifo_disable;
300 engine->fifo.enable = nv04_fifo_enable; 319 engine->fifo.enable = nv04_fifo_enable;
301 engine->fifo.reassign = nv04_fifo_reassign; 320 engine->fifo.reassign = nv04_fifo_reassign;
302 engine->fifo.cache_pull = nv04_fifo_cache_pull; 321 engine->fifo.cache_pull = nv04_fifo_cache_pull;
303 engine->fifo.channel_id = nv10_fifo_channel_id; 322 engine->fifo.channel_id = nv10_fifo_channel_id;
304 engine->fifo.create_context = nv40_fifo_create_context; 323 engine->fifo.create_context = nv40_fifo_create_context;
305 engine->fifo.destroy_context = nv40_fifo_destroy_context; 324 engine->fifo.destroy_context = nv04_fifo_destroy_context;
306 engine->fifo.load_context = nv40_fifo_load_context; 325 engine->fifo.load_context = nv40_fifo_load_context;
307 engine->fifo.unload_context = nv40_fifo_unload_context; 326 engine->fifo.unload_context = nv40_fifo_unload_context;
308 engine->display.early_init = nv04_display_early_init; 327 engine->display.early_init = nv04_display_early_init;
@@ -321,6 +340,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
321 engine->pm.voltage_get = nouveau_voltage_gpio_get; 340 engine->pm.voltage_get = nouveau_voltage_gpio_get;
322 engine->pm.voltage_set = nouveau_voltage_gpio_set; 341 engine->pm.voltage_set = nouveau_voltage_gpio_set;
323 engine->pm.temp_get = nv40_temp_get; 342 engine->pm.temp_get = nv40_temp_get;
343 engine->crypt.init = nouveau_stub_init;
344 engine->crypt.takedown = nouveau_stub_takedown;
345 engine->vram.init = nouveau_mem_detect;
346 engine->vram.flags_valid = nouveau_mem_flags_valid;
324 break; 347 break;
325 case 0x50: 348 case 0x50:
326 case 0x80: /* gotta love NVIDIA's consistency.. */ 349 case 0x80: /* gotta love NVIDIA's consistency.. */
@@ -330,10 +353,10 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
330 engine->instmem.takedown = nv50_instmem_takedown; 353 engine->instmem.takedown = nv50_instmem_takedown;
331 engine->instmem.suspend = nv50_instmem_suspend; 354 engine->instmem.suspend = nv50_instmem_suspend;
332 engine->instmem.resume = nv50_instmem_resume; 355 engine->instmem.resume = nv50_instmem_resume;
333 engine->instmem.populate = nv50_instmem_populate; 356 engine->instmem.get = nv50_instmem_get;
334 engine->instmem.clear = nv50_instmem_clear; 357 engine->instmem.put = nv50_instmem_put;
335 engine->instmem.bind = nv50_instmem_bind; 358 engine->instmem.map = nv50_instmem_map;
336 engine->instmem.unbind = nv50_instmem_unbind; 359 engine->instmem.unmap = nv50_instmem_unmap;
337 if (dev_priv->chipset == 0x50) 360 if (dev_priv->chipset == 0x50)
338 engine->instmem.flush = nv50_instmem_flush; 361 engine->instmem.flush = nv50_instmem_flush;
339 else 362 else
@@ -345,7 +368,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
345 engine->timer.takedown = nv04_timer_takedown; 368 engine->timer.takedown = nv04_timer_takedown;
346 engine->fb.init = nv50_fb_init; 369 engine->fb.init = nv50_fb_init;
347 engine->fb.takedown = nv50_fb_takedown; 370 engine->fb.takedown = nv50_fb_takedown;
348 engine->graph.grclass = nv50_graph_grclass;
349 engine->graph.init = nv50_graph_init; 371 engine->graph.init = nv50_graph_init;
350 engine->graph.takedown = nv50_graph_takedown; 372 engine->graph.takedown = nv50_graph_takedown;
351 engine->graph.fifo_access = nv50_graph_fifo_access; 373 engine->graph.fifo_access = nv50_graph_fifo_access;
@@ -381,24 +403,32 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
381 engine->display.init = nv50_display_init; 403 engine->display.init = nv50_display_init;
382 engine->display.destroy = nv50_display_destroy; 404 engine->display.destroy = nv50_display_destroy;
383 engine->gpio.init = nv50_gpio_init; 405 engine->gpio.init = nv50_gpio_init;
384 engine->gpio.takedown = nouveau_stub_takedown; 406 engine->gpio.takedown = nv50_gpio_fini;
385 engine->gpio.get = nv50_gpio_get; 407 engine->gpio.get = nv50_gpio_get;
386 engine->gpio.set = nv50_gpio_set; 408 engine->gpio.set = nv50_gpio_set;
409 engine->gpio.irq_register = nv50_gpio_irq_register;
410 engine->gpio.irq_unregister = nv50_gpio_irq_unregister;
387 engine->gpio.irq_enable = nv50_gpio_irq_enable; 411 engine->gpio.irq_enable = nv50_gpio_irq_enable;
388 switch (dev_priv->chipset) { 412 switch (dev_priv->chipset) {
389 case 0xa3: 413 case 0x84:
390 case 0xa5: 414 case 0x86:
391 case 0xa8: 415 case 0x92:
392 case 0xaf: 416 case 0x94:
393 engine->pm.clock_get = nva3_pm_clock_get; 417 case 0x96:
394 engine->pm.clock_pre = nva3_pm_clock_pre; 418 case 0x98:
395 engine->pm.clock_set = nva3_pm_clock_set; 419 case 0xa0:
396 break; 420 case 0xaa:
397 default: 421 case 0xac:
422 case 0x50:
398 engine->pm.clock_get = nv50_pm_clock_get; 423 engine->pm.clock_get = nv50_pm_clock_get;
399 engine->pm.clock_pre = nv50_pm_clock_pre; 424 engine->pm.clock_pre = nv50_pm_clock_pre;
400 engine->pm.clock_set = nv50_pm_clock_set; 425 engine->pm.clock_set = nv50_pm_clock_set;
401 break; 426 break;
427 default:
428 engine->pm.clock_get = nva3_pm_clock_get;
429 engine->pm.clock_pre = nva3_pm_clock_pre;
430 engine->pm.clock_set = nva3_pm_clock_set;
431 break;
402 } 432 }
403 engine->pm.voltage_get = nouveau_voltage_gpio_get; 433 engine->pm.voltage_get = nouveau_voltage_gpio_get;
404 engine->pm.voltage_set = nouveau_voltage_gpio_set; 434 engine->pm.voltage_set = nouveau_voltage_gpio_set;
@@ -406,17 +436,39 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
406 engine->pm.temp_get = nv84_temp_get; 436 engine->pm.temp_get = nv84_temp_get;
407 else 437 else
408 engine->pm.temp_get = nv40_temp_get; 438 engine->pm.temp_get = nv40_temp_get;
439 switch (dev_priv->chipset) {
440 case 0x84:
441 case 0x86:
442 case 0x92:
443 case 0x94:
444 case 0x96:
445 case 0xa0:
446 engine->crypt.init = nv84_crypt_init;
447 engine->crypt.takedown = nv84_crypt_fini;
448 engine->crypt.create_context = nv84_crypt_create_context;
449 engine->crypt.destroy_context = nv84_crypt_destroy_context;
450 engine->crypt.tlb_flush = nv84_crypt_tlb_flush;
451 break;
452 default:
453 engine->crypt.init = nouveau_stub_init;
454 engine->crypt.takedown = nouveau_stub_takedown;
455 break;
456 }
457 engine->vram.init = nv50_vram_init;
458 engine->vram.get = nv50_vram_new;
459 engine->vram.put = nv50_vram_del;
460 engine->vram.flags_valid = nv50_vram_flags_valid;
409 break; 461 break;
410 case 0xC0: 462 case 0xC0:
411 engine->instmem.init = nvc0_instmem_init; 463 engine->instmem.init = nvc0_instmem_init;
412 engine->instmem.takedown = nvc0_instmem_takedown; 464 engine->instmem.takedown = nvc0_instmem_takedown;
413 engine->instmem.suspend = nvc0_instmem_suspend; 465 engine->instmem.suspend = nvc0_instmem_suspend;
414 engine->instmem.resume = nvc0_instmem_resume; 466 engine->instmem.resume = nvc0_instmem_resume;
415 engine->instmem.populate = nvc0_instmem_populate; 467 engine->instmem.get = nv50_instmem_get;
416 engine->instmem.clear = nvc0_instmem_clear; 468 engine->instmem.put = nv50_instmem_put;
417 engine->instmem.bind = nvc0_instmem_bind; 469 engine->instmem.map = nv50_instmem_map;
418 engine->instmem.unbind = nvc0_instmem_unbind; 470 engine->instmem.unmap = nv50_instmem_unmap;
419 engine->instmem.flush = nvc0_instmem_flush; 471 engine->instmem.flush = nv84_instmem_flush;
420 engine->mc.init = nv50_mc_init; 472 engine->mc.init = nv50_mc_init;
421 engine->mc.takedown = nv50_mc_takedown; 473 engine->mc.takedown = nv50_mc_takedown;
422 engine->timer.init = nv04_timer_init; 474 engine->timer.init = nv04_timer_init;
@@ -424,7 +476,6 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
424 engine->timer.takedown = nv04_timer_takedown; 476 engine->timer.takedown = nv04_timer_takedown;
425 engine->fb.init = nvc0_fb_init; 477 engine->fb.init = nvc0_fb_init;
426 engine->fb.takedown = nvc0_fb_takedown; 478 engine->fb.takedown = nvc0_fb_takedown;
427 engine->graph.grclass = NULL; //nvc0_graph_grclass;
428 engine->graph.init = nvc0_graph_init; 479 engine->graph.init = nvc0_graph_init;
429 engine->graph.takedown = nvc0_graph_takedown; 480 engine->graph.takedown = nvc0_graph_takedown;
430 engine->graph.fifo_access = nvc0_graph_fifo_access; 481 engine->graph.fifo_access = nvc0_graph_fifo_access;
@@ -453,7 +504,15 @@ static int nouveau_init_engine_ptrs(struct drm_device *dev)
453 engine->gpio.takedown = nouveau_stub_takedown; 504 engine->gpio.takedown = nouveau_stub_takedown;
454 engine->gpio.get = nv50_gpio_get; 505 engine->gpio.get = nv50_gpio_get;
455 engine->gpio.set = nv50_gpio_set; 506 engine->gpio.set = nv50_gpio_set;
507 engine->gpio.irq_register = nv50_gpio_irq_register;
508 engine->gpio.irq_unregister = nv50_gpio_irq_unregister;
456 engine->gpio.irq_enable = nv50_gpio_irq_enable; 509 engine->gpio.irq_enable = nv50_gpio_irq_enable;
510 engine->crypt.init = nouveau_stub_init;
511 engine->crypt.takedown = nouveau_stub_takedown;
512 engine->vram.init = nvc0_vram_init;
513 engine->vram.get = nvc0_vram_new;
514 engine->vram.put = nv50_vram_del;
515 engine->vram.flags_valid = nvc0_vram_flags_valid;
457 break; 516 break;
458 default: 517 default:
459 NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset); 518 NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
@@ -493,9 +552,13 @@ nouveau_card_init_channel(struct drm_device *dev)
493 if (ret) 552 if (ret)
494 return ret; 553 return ret;
495 554
555 /* no dma objects on fermi... */
556 if (dev_priv->card_type >= NV_C0)
557 goto out_done;
558
496 ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY, 559 ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY,
497 0, dev_priv->vram_size, 560 0, dev_priv->vram_size,
498 NV_DMA_ACCESS_RW, NV_DMA_TARGET_VIDMEM, 561 NV_MEM_ACCESS_RW, NV_MEM_TARGET_VRAM,
499 &gpuobj); 562 &gpuobj);
500 if (ret) 563 if (ret)
501 goto out_err; 564 goto out_err;
@@ -505,9 +568,10 @@ nouveau_card_init_channel(struct drm_device *dev)
505 if (ret) 568 if (ret)
506 goto out_err; 569 goto out_err;
507 570
508 ret = nouveau_gpuobj_gart_dma_new(dev_priv->channel, 0, 571 ret = nouveau_gpuobj_dma_new(dev_priv->channel, NV_CLASS_DMA_IN_MEMORY,
509 dev_priv->gart_info.aper_size, 572 0, dev_priv->gart_info.aper_size,
510 NV_DMA_ACCESS_RW, &gpuobj, NULL); 573 NV_MEM_ACCESS_RW, NV_MEM_TARGET_GART,
574 &gpuobj);
511 if (ret) 575 if (ret)
512 goto out_err; 576 goto out_err;
513 577
@@ -516,11 +580,12 @@ nouveau_card_init_channel(struct drm_device *dev)
516 if (ret) 580 if (ret)
517 goto out_err; 581 goto out_err;
518 582
583out_done:
584 mutex_unlock(&dev_priv->channel->mutex);
519 return 0; 585 return 0;
520 586
521out_err: 587out_err:
522 nouveau_channel_free(dev_priv->channel); 588 nouveau_channel_put(&dev_priv->channel);
523 dev_priv->channel = NULL;
524 return ret; 589 return ret;
525} 590}
526 591
@@ -531,15 +596,25 @@ static void nouveau_switcheroo_set_state(struct pci_dev *pdev,
531 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 596 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
532 if (state == VGA_SWITCHEROO_ON) { 597 if (state == VGA_SWITCHEROO_ON) {
533 printk(KERN_ERR "VGA switcheroo: switched nouveau on\n"); 598 printk(KERN_ERR "VGA switcheroo: switched nouveau on\n");
599 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
534 nouveau_pci_resume(pdev); 600 nouveau_pci_resume(pdev);
535 drm_kms_helper_poll_enable(dev); 601 drm_kms_helper_poll_enable(dev);
602 dev->switch_power_state = DRM_SWITCH_POWER_ON;
536 } else { 603 } else {
537 printk(KERN_ERR "VGA switcheroo: switched nouveau off\n"); 604 printk(KERN_ERR "VGA switcheroo: switched nouveau off\n");
605 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
538 drm_kms_helper_poll_disable(dev); 606 drm_kms_helper_poll_disable(dev);
539 nouveau_pci_suspend(pdev, pmm); 607 nouveau_pci_suspend(pdev, pmm);
608 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
540 } 609 }
541} 610}
542 611
612static void nouveau_switcheroo_reprobe(struct pci_dev *pdev)
613{
614 struct drm_device *dev = pci_get_drvdata(pdev);
615 nouveau_fbcon_output_poll_changed(dev);
616}
617
543static bool nouveau_switcheroo_can_switch(struct pci_dev *pdev) 618static bool nouveau_switcheroo_can_switch(struct pci_dev *pdev)
544{ 619{
545 struct drm_device *dev = pci_get_drvdata(pdev); 620 struct drm_device *dev = pci_get_drvdata(pdev);
@@ -560,6 +635,7 @@ nouveau_card_init(struct drm_device *dev)
560 635
561 vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode); 636 vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
562 vga_switcheroo_register_client(dev->pdev, nouveau_switcheroo_set_state, 637 vga_switcheroo_register_client(dev->pdev, nouveau_switcheroo_set_state,
638 nouveau_switcheroo_reprobe,
563 nouveau_switcheroo_can_switch); 639 nouveau_switcheroo_can_switch);
564 640
565 /* Initialise internal driver API hooks */ 641 /* Initialise internal driver API hooks */
@@ -567,6 +643,8 @@ nouveau_card_init(struct drm_device *dev)
567 if (ret) 643 if (ret)
568 goto out; 644 goto out;
569 engine = &dev_priv->engine; 645 engine = &dev_priv->engine;
646 spin_lock_init(&dev_priv->channels.lock);
647 spin_lock_init(&dev_priv->tile.lock);
570 spin_lock_init(&dev_priv->context_switch_lock); 648 spin_lock_init(&dev_priv->context_switch_lock);
571 649
572 /* Make the CRTCs and I2C buses accessible */ 650 /* Make the CRTCs and I2C buses accessible */
@@ -625,26 +703,28 @@ nouveau_card_init(struct drm_device *dev)
625 if (ret) 703 if (ret)
626 goto out_fb; 704 goto out_fb;
627 705
706 /* PCRYPT */
707 ret = engine->crypt.init(dev);
708 if (ret)
709 goto out_graph;
710
628 /* PFIFO */ 711 /* PFIFO */
629 ret = engine->fifo.init(dev); 712 ret = engine->fifo.init(dev);
630 if (ret) 713 if (ret)
631 goto out_graph; 714 goto out_crypt;
632 } 715 }
633 716
634 ret = engine->display.create(dev); 717 ret = engine->display.create(dev);
635 if (ret) 718 if (ret)
636 goto out_fifo; 719 goto out_fifo;
637 720
638 /* this call irq_preinstall, register irq handler and 721 ret = drm_vblank_init(dev, nv_two_heads(dev) ? 2 : 1);
639 * call irq_postinstall
640 */
641 ret = drm_irq_install(dev);
642 if (ret) 722 if (ret)
643 goto out_display; 723 goto out_vblank;
644 724
645 ret = drm_vblank_init(dev, 0); 725 ret = nouveau_irq_init(dev);
646 if (ret) 726 if (ret)
647 goto out_irq; 727 goto out_vblank;
648 728
649 /* what about PVIDEO/PCRTC/PRAMDAC etc? */ 729 /* what about PVIDEO/PCRTC/PRAMDAC etc? */
650 730
@@ -669,12 +749,16 @@ nouveau_card_init(struct drm_device *dev)
669out_fence: 749out_fence:
670 nouveau_fence_fini(dev); 750 nouveau_fence_fini(dev);
671out_irq: 751out_irq:
672 drm_irq_uninstall(dev); 752 nouveau_irq_fini(dev);
673out_display: 753out_vblank:
754 drm_vblank_cleanup(dev);
674 engine->display.destroy(dev); 755 engine->display.destroy(dev);
675out_fifo: 756out_fifo:
676 if (!nouveau_noaccel) 757 if (!nouveau_noaccel)
677 engine->fifo.takedown(dev); 758 engine->fifo.takedown(dev);
759out_crypt:
760 if (!nouveau_noaccel)
761 engine->crypt.takedown(dev);
678out_graph: 762out_graph:
679 if (!nouveau_noaccel) 763 if (!nouveau_noaccel)
680 engine->graph.takedown(dev); 764 engine->graph.takedown(dev);
@@ -713,12 +797,12 @@ static void nouveau_card_takedown(struct drm_device *dev)
713 797
714 if (!engine->graph.accel_blocked) { 798 if (!engine->graph.accel_blocked) {
715 nouveau_fence_fini(dev); 799 nouveau_fence_fini(dev);
716 nouveau_channel_free(dev_priv->channel); 800 nouveau_channel_put_unlocked(&dev_priv->channel);
717 dev_priv->channel = NULL;
718 } 801 }
719 802
720 if (!nouveau_noaccel) { 803 if (!nouveau_noaccel) {
721 engine->fifo.takedown(dev); 804 engine->fifo.takedown(dev);
805 engine->crypt.takedown(dev);
722 engine->graph.takedown(dev); 806 engine->graph.takedown(dev);
723 } 807 }
724 engine->fb.takedown(dev); 808 engine->fb.takedown(dev);
@@ -737,7 +821,8 @@ static void nouveau_card_takedown(struct drm_device *dev)
737 nouveau_gpuobj_takedown(dev); 821 nouveau_gpuobj_takedown(dev);
738 nouveau_mem_vram_fini(dev); 822 nouveau_mem_vram_fini(dev);
739 823
740 drm_irq_uninstall(dev); 824 nouveau_irq_fini(dev);
825 drm_vblank_cleanup(dev);
741 826
742 nouveau_pm_fini(dev); 827 nouveau_pm_fini(dev);
743 nouveau_bios_takedown(dev); 828 nouveau_bios_takedown(dev);
@@ -980,6 +1065,7 @@ err_out:
980 1065
981void nouveau_lastclose(struct drm_device *dev) 1066void nouveau_lastclose(struct drm_device *dev)
982{ 1067{
1068 vga_switcheroo_process_delayed_switch();
983} 1069}
984 1070
985int nouveau_unload(struct drm_device *dev) 1071int nouveau_unload(struct drm_device *dev)
@@ -1024,21 +1110,6 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
1024 else 1110 else
1025 getparam->value = NV_PCI; 1111 getparam->value = NV_PCI;
1026 break; 1112 break;
1027 case NOUVEAU_GETPARAM_FB_PHYSICAL:
1028 getparam->value = dev_priv->fb_phys;
1029 break;
1030 case NOUVEAU_GETPARAM_AGP_PHYSICAL:
1031 getparam->value = dev_priv->gart_info.aper_base;
1032 break;
1033 case NOUVEAU_GETPARAM_PCI_PHYSICAL:
1034 if (dev->sg) {
1035 getparam->value = (unsigned long)dev->sg->virtual;
1036 } else {
1037 NV_ERROR(dev, "Requested PCIGART address, "
1038 "while no PCIGART was created\n");
1039 return -EINVAL;
1040 }
1041 break;
1042 case NOUVEAU_GETPARAM_FB_SIZE: 1113 case NOUVEAU_GETPARAM_FB_SIZE:
1043 getparam->value = dev_priv->fb_available_size; 1114 getparam->value = dev_priv->fb_available_size;
1044 break; 1115 break;
@@ -1046,7 +1117,7 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
1046 getparam->value = dev_priv->gart_info.aper_size; 1117 getparam->value = dev_priv->gart_info.aper_size;
1047 break; 1118 break;
1048 case NOUVEAU_GETPARAM_VM_VRAM_BASE: 1119 case NOUVEAU_GETPARAM_VM_VRAM_BASE:
1049 getparam->value = dev_priv->vm_vram_base; 1120 getparam->value = 0; /* deprecated */
1050 break; 1121 break;
1051 case NOUVEAU_GETPARAM_PTIMER_TIME: 1122 case NOUVEAU_GETPARAM_PTIMER_TIME:
1052 getparam->value = dev_priv->engine.timer.read(dev); 1123 getparam->value = dev_priv->engine.timer.read(dev);
@@ -1054,6 +1125,9 @@ int nouveau_ioctl_getparam(struct drm_device *dev, void *data,
1054 case NOUVEAU_GETPARAM_HAS_BO_USAGE: 1125 case NOUVEAU_GETPARAM_HAS_BO_USAGE:
1055 getparam->value = 1; 1126 getparam->value = 1;
1056 break; 1127 break;
1128 case NOUVEAU_GETPARAM_HAS_PAGEFLIP:
1129 getparam->value = (dev_priv->card_type < NV_50);
1130 break;
1057 case NOUVEAU_GETPARAM_GRAPH_UNITS: 1131 case NOUVEAU_GETPARAM_GRAPH_UNITS:
1058 /* NV40 and NV50 versions are quite different, but register 1132 /* NV40 and NV50 versions are quite different, but register
1059 * address is the same. User is supposed to know the card 1133 * address is the same. User is supposed to know the card
@@ -1087,8 +1161,9 @@ nouveau_ioctl_setparam(struct drm_device *dev, void *data,
1087} 1161}
1088 1162
1089/* Wait until (value(reg) & mask) == val, up until timeout has hit */ 1163/* Wait until (value(reg) & mask) == val, up until timeout has hit */
1090bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout, 1164bool
1091 uint32_t reg, uint32_t mask, uint32_t val) 1165nouveau_wait_eq(struct drm_device *dev, uint64_t timeout,
1166 uint32_t reg, uint32_t mask, uint32_t val)
1092{ 1167{
1093 struct drm_nouveau_private *dev_priv = dev->dev_private; 1168 struct drm_nouveau_private *dev_priv = dev->dev_private;
1094 struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer; 1169 struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
@@ -1102,10 +1177,33 @@ bool nouveau_wait_until(struct drm_device *dev, uint64_t timeout,
1102 return false; 1177 return false;
1103} 1178}
1104 1179
1180/* Wait until (value(reg) & mask) != val, up until timeout has hit */
1181bool
1182nouveau_wait_ne(struct drm_device *dev, uint64_t timeout,
1183 uint32_t reg, uint32_t mask, uint32_t val)
1184{
1185 struct drm_nouveau_private *dev_priv = dev->dev_private;
1186 struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
1187 uint64_t start = ptimer->read(dev);
1188
1189 do {
1190 if ((nv_rd32(dev, reg) & mask) != val)
1191 return true;
1192 } while (ptimer->read(dev) - start < timeout);
1193
1194 return false;
1195}
1196
1105/* Waits for PGRAPH to go completely idle */ 1197/* Waits for PGRAPH to go completely idle */
1106bool nouveau_wait_for_idle(struct drm_device *dev) 1198bool nouveau_wait_for_idle(struct drm_device *dev)
1107{ 1199{
1108 if (!nv_wait(dev, NV04_PGRAPH_STATUS, 0xffffffff, 0x00000000)) { 1200 struct drm_nouveau_private *dev_priv = dev->dev_private;
1201 uint32_t mask = ~0;
1202
1203 if (dev_priv->card_type == NV_40)
1204 mask &= ~NV40_PGRAPH_STATUS_SYNC_STALL;
1205
1206 if (!nv_wait(dev, NV04_PGRAPH_STATUS, mask, 0)) {
1109 NV_ERROR(dev, "PGRAPH idle timed out with status 0x%08x\n", 1207 NV_ERROR(dev, "PGRAPH idle timed out with status 0x%08x\n",
1110 nv_rd32(dev, NV04_PGRAPH_STATUS)); 1208 nv_rd32(dev, NV04_PGRAPH_STATUS));
1111 return false; 1209 return false;
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.c b/drivers/gpu/drm/nouveau/nouveau_util.c
new file mode 100644
index 000000000000..fbe0fb13bc1e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_util.c
@@ -0,0 +1,69 @@
1/*
2 * Copyright (C) 2010 Nouveau Project
3 *
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 */
27
28#include <linux/ratelimit.h>
29
30#include "nouveau_util.h"
31
32static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
33
34void
35nouveau_bitfield_print(const struct nouveau_bitfield *bf, u32 value)
36{
37 while (bf->name) {
38 if (value & bf->mask) {
39 printk(" %s", bf->name);
40 value &= ~bf->mask;
41 }
42
43 bf++;
44 }
45
46 if (value)
47 printk(" (unknown bits 0x%08x)", value);
48}
49
50void
51nouveau_enum_print(const struct nouveau_enum *en, u32 value)
52{
53 while (en->name) {
54 if (value == en->value) {
55 printk("%s", en->name);
56 return;
57 }
58
59 en++;
60 }
61
62 printk("(unknown enum 0x%08x)", value);
63}
64
65int
66nouveau_ratelimit(void)
67{
68 return __ratelimit(&nouveau_ratelimit_state);
69}
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.h b/drivers/gpu/drm/nouveau/nouveau_util.h
new file mode 100644
index 000000000000..d9ceaea26f4b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_util.h
@@ -0,0 +1,45 @@
1/*
2 * Copyright (C) 2010 Nouveau Project
3 *
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial
16 * portions of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 */
27
28#ifndef __NOUVEAU_UTIL_H__
29#define __NOUVEAU_UTIL_H__
30
31struct nouveau_bitfield {
32 u32 mask;
33 const char *name;
34};
35
36struct nouveau_enum {
37 u32 value;
38 const char *name;
39};
40
41void nouveau_bitfield_print(const struct nouveau_bitfield *, u32 value);
42void nouveau_enum_print(const struct nouveau_enum *, u32 value);
43int nouveau_ratelimit(void);
44
45#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.c b/drivers/gpu/drm/nouveau/nouveau_vm.c
new file mode 100644
index 000000000000..97d82aedf86b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.c
@@ -0,0 +1,439 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_mm.h"
28#include "nouveau_vm.h"
29
30void
31nouveau_vm_map_at(struct nouveau_vma *vma, u64 delta, struct nouveau_vram *vram)
32{
33 struct nouveau_vm *vm = vma->vm;
34 struct nouveau_mm_node *r;
35 int big = vma->node->type != vm->spg_shift;
36 u32 offset = vma->node->offset + (delta >> 12);
37 u32 bits = vma->node->type - 12;
38 u32 pde = (offset >> vm->pgt_bits) - vm->fpde;
39 u32 pte = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
40 u32 max = 1 << (vm->pgt_bits - bits);
41 u32 end, len;
42
43 list_for_each_entry(r, &vram->regions, rl_entry) {
44 u64 phys = (u64)r->offset << 12;
45 u32 num = r->length >> bits;
46
47 while (num) {
48 struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
49
50 end = (pte + num);
51 if (unlikely(end >= max))
52 end = max;
53 len = end - pte;
54
55 vm->map(vma, pgt, vram, pte, len, phys);
56
57 num -= len;
58 pte += len;
59 if (unlikely(end >= max)) {
60 pde++;
61 pte = 0;
62 }
63 }
64 }
65
66 vm->flush(vm);
67}
68
69void
70nouveau_vm_map(struct nouveau_vma *vma, struct nouveau_vram *vram)
71{
72 nouveau_vm_map_at(vma, 0, vram);
73}
74
75void
76nouveau_vm_map_sg(struct nouveau_vma *vma, u64 delta, u64 length,
77 dma_addr_t *list)
78{
79 struct nouveau_vm *vm = vma->vm;
80 int big = vma->node->type != vm->spg_shift;
81 u32 offset = vma->node->offset + (delta >> 12);
82 u32 bits = vma->node->type - 12;
83 u32 num = length >> vma->node->type;
84 u32 pde = (offset >> vm->pgt_bits) - vm->fpde;
85 u32 pte = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
86 u32 max = 1 << (vm->pgt_bits - bits);
87 u32 end, len;
88
89 while (num) {
90 struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
91
92 end = (pte + num);
93 if (unlikely(end >= max))
94 end = max;
95 len = end - pte;
96
97 vm->map_sg(vma, pgt, pte, list, len);
98
99 num -= len;
100 pte += len;
101 list += len;
102 if (unlikely(end >= max)) {
103 pde++;
104 pte = 0;
105 }
106 }
107
108 vm->flush(vm);
109}
110
111void
112nouveau_vm_unmap_at(struct nouveau_vma *vma, u64 delta, u64 length)
113{
114 struct nouveau_vm *vm = vma->vm;
115 int big = vma->node->type != vm->spg_shift;
116 u32 offset = vma->node->offset + (delta >> 12);
117 u32 bits = vma->node->type - 12;
118 u32 num = length >> vma->node->type;
119 u32 pde = (offset >> vm->pgt_bits) - vm->fpde;
120 u32 pte = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
121 u32 max = 1 << (vm->pgt_bits - bits);
122 u32 end, len;
123
124 while (num) {
125 struct nouveau_gpuobj *pgt = vm->pgt[pde].obj[big];
126
127 end = (pte + num);
128 if (unlikely(end >= max))
129 end = max;
130 len = end - pte;
131
132 vm->unmap(pgt, pte, len);
133
134 num -= len;
135 pte += len;
136 if (unlikely(end >= max)) {
137 pde++;
138 pte = 0;
139 }
140 }
141
142 vm->flush(vm);
143}
144
145void
146nouveau_vm_unmap(struct nouveau_vma *vma)
147{
148 nouveau_vm_unmap_at(vma, 0, (u64)vma->node->length << 12);
149}
150
151static void
152nouveau_vm_unmap_pgt(struct nouveau_vm *vm, int big, u32 fpde, u32 lpde)
153{
154 struct nouveau_vm_pgd *vpgd;
155 struct nouveau_vm_pgt *vpgt;
156 struct nouveau_gpuobj *pgt;
157 u32 pde;
158
159 for (pde = fpde; pde <= lpde; pde++) {
160 vpgt = &vm->pgt[pde - vm->fpde];
161 if (--vpgt->refcount[big])
162 continue;
163
164 pgt = vpgt->obj[big];
165 vpgt->obj[big] = NULL;
166
167 list_for_each_entry(vpgd, &vm->pgd_list, head) {
168 vm->map_pgt(vpgd->obj, pde, vpgt->obj);
169 }
170
171 mutex_unlock(&vm->mm->mutex);
172 nouveau_gpuobj_ref(NULL, &pgt);
173 mutex_lock(&vm->mm->mutex);
174 }
175}
176
177static int
178nouveau_vm_map_pgt(struct nouveau_vm *vm, u32 pde, u32 type)
179{
180 struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
181 struct nouveau_vm_pgd *vpgd;
182 struct nouveau_gpuobj *pgt;
183 int big = (type != vm->spg_shift);
184 u32 pgt_size;
185 int ret;
186
187 pgt_size = (1 << (vm->pgt_bits + 12)) >> type;
188 pgt_size *= 8;
189
190 mutex_unlock(&vm->mm->mutex);
191 ret = nouveau_gpuobj_new(vm->dev, NULL, pgt_size, 0x1000,
192 NVOBJ_FLAG_ZERO_ALLOC, &pgt);
193 mutex_lock(&vm->mm->mutex);
194 if (unlikely(ret))
195 return ret;
196
197 /* someone beat us to filling the PDE while we didn't have the lock */
198 if (unlikely(vpgt->refcount[big]++)) {
199 mutex_unlock(&vm->mm->mutex);
200 nouveau_gpuobj_ref(NULL, &pgt);
201 mutex_lock(&vm->mm->mutex);
202 return 0;
203 }
204
205 vpgt->obj[big] = pgt;
206 list_for_each_entry(vpgd, &vm->pgd_list, head) {
207 vm->map_pgt(vpgd->obj, pde, vpgt->obj);
208 }
209
210 return 0;
211}
212
213int
214nouveau_vm_get(struct nouveau_vm *vm, u64 size, u32 page_shift,
215 u32 access, struct nouveau_vma *vma)
216{
217 u32 align = (1 << page_shift) >> 12;
218 u32 msize = size >> 12;
219 u32 fpde, lpde, pde;
220 int ret;
221
222 mutex_lock(&vm->mm->mutex);
223 ret = nouveau_mm_get(vm->mm, page_shift, msize, 0, align, &vma->node);
224 if (unlikely(ret != 0)) {
225 mutex_unlock(&vm->mm->mutex);
226 return ret;
227 }
228
229 fpde = (vma->node->offset >> vm->pgt_bits);
230 lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
231 for (pde = fpde; pde <= lpde; pde++) {
232 struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
233 int big = (vma->node->type != vm->spg_shift);
234
235 if (likely(vpgt->refcount[big])) {
236 vpgt->refcount[big]++;
237 continue;
238 }
239
240 ret = nouveau_vm_map_pgt(vm, pde, vma->node->type);
241 if (ret) {
242 if (pde != fpde)
243 nouveau_vm_unmap_pgt(vm, big, fpde, pde - 1);
244 nouveau_mm_put(vm->mm, vma->node);
245 mutex_unlock(&vm->mm->mutex);
246 vma->node = NULL;
247 return ret;
248 }
249 }
250 mutex_unlock(&vm->mm->mutex);
251
252 vma->vm = vm;
253 vma->offset = (u64)vma->node->offset << 12;
254 vma->access = access;
255 return 0;
256}
257
258void
259nouveau_vm_put(struct nouveau_vma *vma)
260{
261 struct nouveau_vm *vm = vma->vm;
262 u32 fpde, lpde;
263
264 if (unlikely(vma->node == NULL))
265 return;
266 fpde = (vma->node->offset >> vm->pgt_bits);
267 lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
268
269 mutex_lock(&vm->mm->mutex);
270 nouveau_vm_unmap_pgt(vm, vma->node->type != vm->spg_shift, fpde, lpde);
271 nouveau_mm_put(vm->mm, vma->node);
272 vma->node = NULL;
273 mutex_unlock(&vm->mm->mutex);
274}
275
276int
277nouveau_vm_new(struct drm_device *dev, u64 offset, u64 length, u64 mm_offset,
278 struct nouveau_vm **pvm)
279{
280 struct drm_nouveau_private *dev_priv = dev->dev_private;
281 struct nouveau_vm *vm;
282 u64 mm_length = (offset + length) - mm_offset;
283 u32 block, pgt_bits;
284 int ret;
285
286 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
287 if (!vm)
288 return -ENOMEM;
289
290 if (dev_priv->card_type == NV_50) {
291 vm->map_pgt = nv50_vm_map_pgt;
292 vm->map = nv50_vm_map;
293 vm->map_sg = nv50_vm_map_sg;
294 vm->unmap = nv50_vm_unmap;
295 vm->flush = nv50_vm_flush;
296 vm->spg_shift = 12;
297 vm->lpg_shift = 16;
298
299 pgt_bits = 29;
300 block = (1 << pgt_bits);
301 if (length < block)
302 block = length;
303
304 } else
305 if (dev_priv->card_type == NV_C0) {
306 vm->map_pgt = nvc0_vm_map_pgt;
307 vm->map = nvc0_vm_map;
308 vm->map_sg = nvc0_vm_map_sg;
309 vm->unmap = nvc0_vm_unmap;
310 vm->flush = nvc0_vm_flush;
311 vm->spg_shift = 12;
312 vm->lpg_shift = 17;
313 pgt_bits = 27;
314
315 /* Should be 4096 everywhere, this is a hack that's
316 * currently necessary to avoid an elusive bug that
317 * causes corruption when mixing small/large pages
318 */
319 if (length < (1ULL << 40))
320 block = 4096;
321 else {
322 block = (1 << pgt_bits);
323 if (length < block)
324 block = length;
325 }
326 } else {
327 kfree(vm);
328 return -ENOSYS;
329 }
330
331 vm->fpde = offset >> pgt_bits;
332 vm->lpde = (offset + length - 1) >> pgt_bits;
333 vm->pgt = kcalloc(vm->lpde - vm->fpde + 1, sizeof(*vm->pgt), GFP_KERNEL);
334 if (!vm->pgt) {
335 kfree(vm);
336 return -ENOMEM;
337 }
338
339 INIT_LIST_HEAD(&vm->pgd_list);
340 vm->dev = dev;
341 vm->refcount = 1;
342 vm->pgt_bits = pgt_bits - 12;
343
344 ret = nouveau_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12,
345 block >> 12);
346 if (ret) {
347 kfree(vm);
348 return ret;
349 }
350
351 *pvm = vm;
352 return 0;
353}
354
355static int
356nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
357{
358 struct nouveau_vm_pgd *vpgd;
359 int i;
360
361 if (!pgd)
362 return 0;
363
364 vpgd = kzalloc(sizeof(*vpgd), GFP_KERNEL);
365 if (!vpgd)
366 return -ENOMEM;
367
368 nouveau_gpuobj_ref(pgd, &vpgd->obj);
369
370 mutex_lock(&vm->mm->mutex);
371 for (i = vm->fpde; i <= vm->lpde; i++)
372 vm->map_pgt(pgd, i, vm->pgt[i - vm->fpde].obj);
373 list_add(&vpgd->head, &vm->pgd_list);
374 mutex_unlock(&vm->mm->mutex);
375 return 0;
376}
377
378static void
379nouveau_vm_unlink(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
380{
381 struct nouveau_vm_pgd *vpgd, *tmp;
382
383 if (!pgd)
384 return;
385
386 mutex_lock(&vm->mm->mutex);
387 list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
388 if (vpgd->obj != pgd)
389 continue;
390
391 list_del(&vpgd->head);
392 nouveau_gpuobj_ref(NULL, &vpgd->obj);
393 kfree(vpgd);
394 }
395 mutex_unlock(&vm->mm->mutex);
396}
397
398static void
399nouveau_vm_del(struct nouveau_vm *vm)
400{
401 struct nouveau_vm_pgd *vpgd, *tmp;
402
403 list_for_each_entry_safe(vpgd, tmp, &vm->pgd_list, head) {
404 nouveau_vm_unlink(vm, vpgd->obj);
405 }
406 WARN_ON(nouveau_mm_fini(&vm->mm) != 0);
407
408 kfree(vm->pgt);
409 kfree(vm);
410}
411
412int
413nouveau_vm_ref(struct nouveau_vm *ref, struct nouveau_vm **ptr,
414 struct nouveau_gpuobj *pgd)
415{
416 struct nouveau_vm *vm;
417 int ret;
418
419 vm = ref;
420 if (vm) {
421 ret = nouveau_vm_link(vm, pgd);
422 if (ret)
423 return ret;
424
425 vm->refcount++;
426 }
427
428 vm = *ptr;
429 *ptr = ref;
430
431 if (vm) {
432 nouveau_vm_unlink(vm, pgd);
433
434 if (--vm->refcount == 0)
435 nouveau_vm_del(vm);
436 }
437
438 return 0;
439}
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.h b/drivers/gpu/drm/nouveau/nouveau_vm.h
new file mode 100644
index 000000000000..e1193515771b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vm.h
@@ -0,0 +1,113 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#ifndef __NOUVEAU_VM_H__
26#define __NOUVEAU_VM_H__
27
28#include "drmP.h"
29
30#include "nouveau_drv.h"
31#include "nouveau_mm.h"
32
33struct nouveau_vm_pgt {
34 struct nouveau_gpuobj *obj[2];
35 u32 refcount[2];
36};
37
38struct nouveau_vm_pgd {
39 struct list_head head;
40 struct nouveau_gpuobj *obj;
41};
42
43struct nouveau_vma {
44 struct nouveau_vm *vm;
45 struct nouveau_mm_node *node;
46 u64 offset;
47 u32 access;
48};
49
50struct nouveau_vm {
51 struct drm_device *dev;
52 struct nouveau_mm *mm;
53 int refcount;
54
55 struct list_head pgd_list;
56 atomic_t pgraph_refs;
57 atomic_t pcrypt_refs;
58
59 struct nouveau_vm_pgt *pgt;
60 u32 fpde;
61 u32 lpde;
62
63 u32 pgt_bits;
64 u8 spg_shift;
65 u8 lpg_shift;
66
67 void (*map_pgt)(struct nouveau_gpuobj *pgd, u32 pde,
68 struct nouveau_gpuobj *pgt[2]);
69 void (*map)(struct nouveau_vma *, struct nouveau_gpuobj *,
70 struct nouveau_vram *, u32 pte, u32 cnt, u64 phys);
71 void (*map_sg)(struct nouveau_vma *, struct nouveau_gpuobj *,
72 u32 pte, dma_addr_t *, u32 cnt);
73 void (*unmap)(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt);
74 void (*flush)(struct nouveau_vm *);
75};
76
77/* nouveau_vm.c */
78int nouveau_vm_new(struct drm_device *, u64 offset, u64 length, u64 mm_offset,
79 struct nouveau_vm **);
80int nouveau_vm_ref(struct nouveau_vm *, struct nouveau_vm **,
81 struct nouveau_gpuobj *pgd);
82int nouveau_vm_get(struct nouveau_vm *, u64 size, u32 page_shift,
83 u32 access, struct nouveau_vma *);
84void nouveau_vm_put(struct nouveau_vma *);
85void nouveau_vm_map(struct nouveau_vma *, struct nouveau_vram *);
86void nouveau_vm_map_at(struct nouveau_vma *, u64 offset, struct nouveau_vram *);
87void nouveau_vm_unmap(struct nouveau_vma *);
88void nouveau_vm_unmap_at(struct nouveau_vma *, u64 offset, u64 length);
89void nouveau_vm_map_sg(struct nouveau_vma *, u64 offset, u64 length,
90 dma_addr_t *);
91
92/* nv50_vm.c */
93void nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
94 struct nouveau_gpuobj *pgt[2]);
95void nv50_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
96 struct nouveau_vram *, u32 pte, u32 cnt, u64 phys);
97void nv50_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
98 u32 pte, dma_addr_t *, u32 cnt);
99void nv50_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
100void nv50_vm_flush(struct nouveau_vm *);
101void nv50_vm_flush_engine(struct drm_device *, int engine);
102
103/* nvc0_vm.c */
104void nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
105 struct nouveau_gpuobj *pgt[2]);
106void nvc0_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
107 struct nouveau_vram *, u32 pte, u32 cnt, u64 phys);
108void nvc0_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
109 u32 pte, dma_addr_t *, u32 cnt);
110void nvc0_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
111void nvc0_vm_flush(struct nouveau_vm *);
112
113#endif
diff --git a/drivers/gpu/drm/nouveau/nv04_crtc.c b/drivers/gpu/drm/nouveau/nv04_crtc.c
index 40e180741629..297505eb98d5 100644
--- a/drivers/gpu/drm/nouveau/nv04_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv04_crtc.c
@@ -551,7 +551,10 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
551 if (dev_priv->card_type >= NV_30) 551 if (dev_priv->card_type >= NV_30)
552 regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT); 552 regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT);
553 553
554 regp->crtc_cfg = NV_PCRTC_CONFIG_START_ADDRESS_HSYNC; 554 if (dev_priv->card_type >= NV_10)
555 regp->crtc_cfg = NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC;
556 else
557 regp->crtc_cfg = NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC;
555 558
556 /* Some misc regs */ 559 /* Some misc regs */
557 if (dev_priv->card_type == NV_40) { 560 if (dev_priv->card_type == NV_40) {
@@ -669,6 +672,7 @@ static void nv_crtc_prepare(struct drm_crtc *crtc)
669 if (nv_two_heads(dev)) 672 if (nv_two_heads(dev))
670 NVSetOwner(dev, nv_crtc->index); 673 NVSetOwner(dev, nv_crtc->index);
671 674
675 drm_vblank_pre_modeset(dev, nv_crtc->index);
672 funcs->dpms(crtc, DRM_MODE_DPMS_OFF); 676 funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
673 677
674 NVBlankScreen(dev, nv_crtc->index, true); 678 NVBlankScreen(dev, nv_crtc->index, true);
@@ -701,6 +705,7 @@ static void nv_crtc_commit(struct drm_crtc *crtc)
701#endif 705#endif
702 706
703 funcs->dpms(crtc, DRM_MODE_DPMS_ON); 707 funcs->dpms(crtc, DRM_MODE_DPMS_ON);
708 drm_vblank_post_modeset(dev, nv_crtc->index);
704} 709}
705 710
706static void nv_crtc_destroy(struct drm_crtc *crtc) 711static void nv_crtc_destroy(struct drm_crtc *crtc)
@@ -986,6 +991,7 @@ static const struct drm_crtc_funcs nv04_crtc_funcs = {
986 .cursor_move = nv04_crtc_cursor_move, 991 .cursor_move = nv04_crtc_cursor_move,
987 .gamma_set = nv_crtc_gamma_set, 992 .gamma_set = nv_crtc_gamma_set,
988 .set_config = drm_crtc_helper_set_config, 993 .set_config = drm_crtc_helper_set_config,
994 .page_flip = nouveau_crtc_page_flip,
989 .destroy = nv_crtc_destroy, 995 .destroy = nv_crtc_destroy,
990}; 996};
991 997
diff --git a/drivers/gpu/drm/nouveau/nv04_dac.c b/drivers/gpu/drm/nouveau/nv04_dac.c
index ba6423f2ffcc..e000455e06d0 100644
--- a/drivers/gpu/drm/nouveau/nv04_dac.c
+++ b/drivers/gpu/drm/nouveau/nv04_dac.c
@@ -74,14 +74,14 @@ static int sample_load_twice(struct drm_device *dev, bool sense[2])
74 * use a 10ms timeout (guards against crtc being inactive, in 74 * use a 10ms timeout (guards against crtc being inactive, in
75 * which case blank state would never change) 75 * which case blank state would never change)
76 */ 76 */
77 if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR, 77 if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
78 0x00000001, 0x00000000)) 78 0x00000001, 0x00000000))
79 return -EBUSY; 79 return -EBUSY;
80 if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR, 80 if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
81 0x00000001, 0x00000001)) 81 0x00000001, 0x00000001))
82 return -EBUSY; 82 return -EBUSY;
83 if (!nouveau_wait_until(dev, 10000000, NV_PRMCIO_INP0__COLOR, 83 if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
84 0x00000001, 0x00000000)) 84 0x00000001, 0x00000000))
85 return -EBUSY; 85 return -EBUSY;
86 86
87 udelay(100); 87 udelay(100);
diff --git a/drivers/gpu/drm/nouveau/nv04_display.c b/drivers/gpu/drm/nouveau/nv04_display.c
index 9e28cf772e3c..1715e1464b7d 100644
--- a/drivers/gpu/drm/nouveau/nv04_display.c
+++ b/drivers/gpu/drm/nouveau/nv04_display.c
@@ -32,6 +32,9 @@
32#include "nouveau_encoder.h" 32#include "nouveau_encoder.h"
33#include "nouveau_connector.h" 33#include "nouveau_connector.h"
34 34
35static void nv04_vblank_crtc0_isr(struct drm_device *);
36static void nv04_vblank_crtc1_isr(struct drm_device *);
37
35static void 38static void
36nv04_display_store_initial_head_owner(struct drm_device *dev) 39nv04_display_store_initial_head_owner(struct drm_device *dev)
37{ 40{
@@ -197,6 +200,8 @@ nv04_display_create(struct drm_device *dev)
197 func->save(encoder); 200 func->save(encoder);
198 } 201 }
199 202
203 nouveau_irq_register(dev, 24, nv04_vblank_crtc0_isr);
204 nouveau_irq_register(dev, 25, nv04_vblank_crtc1_isr);
200 return 0; 205 return 0;
201} 206}
202 207
@@ -208,6 +213,9 @@ nv04_display_destroy(struct drm_device *dev)
208 213
209 NV_DEBUG_KMS(dev, "\n"); 214 NV_DEBUG_KMS(dev, "\n");
210 215
216 nouveau_irq_unregister(dev, 24);
217 nouveau_irq_unregister(dev, 25);
218
211 /* Turn every CRTC off. */ 219 /* Turn every CRTC off. */
212 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 220 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
213 struct drm_mode_set modeset = { 221 struct drm_mode_set modeset = {
@@ -258,3 +266,16 @@ nv04_display_init(struct drm_device *dev)
258 return 0; 266 return 0;
259} 267}
260 268
269static void
270nv04_vblank_crtc0_isr(struct drm_device *dev)
271{
272 nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
273 drm_handle_vblank(dev, 0);
274}
275
276static void
277nv04_vblank_crtc1_isr(struct drm_device *dev)
278{
279 nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
280 drm_handle_vblank(dev, 1);
281}
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index 33e4c9388bc1..7a1189371096 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -28,52 +28,39 @@
28#include "nouveau_ramht.h" 28#include "nouveau_ramht.h"
29#include "nouveau_fbcon.h" 29#include "nouveau_fbcon.h"
30 30
31void 31int
32nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) 32nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
33{ 33{
34 struct nouveau_fbdev *nfbdev = info->par; 34 struct nouveau_fbdev *nfbdev = info->par;
35 struct drm_device *dev = nfbdev->dev; 35 struct drm_device *dev = nfbdev->dev;
36 struct drm_nouveau_private *dev_priv = dev->dev_private; 36 struct drm_nouveau_private *dev_priv = dev->dev_private;
37 struct nouveau_channel *chan = dev_priv->channel; 37 struct nouveau_channel *chan = dev_priv->channel;
38 int ret;
38 39
39 if (info->state != FBINFO_STATE_RUNNING) 40 ret = RING_SPACE(chan, 4);
40 return; 41 if (ret)
41 42 return ret;
42 if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 4)) {
43 nouveau_fbcon_gpu_lockup(info);
44 }
45
46 if (info->flags & FBINFO_HWACCEL_DISABLED) {
47 cfb_copyarea(info, region);
48 return;
49 }
50 43
51 BEGIN_RING(chan, NvSubImageBlit, 0x0300, 3); 44 BEGIN_RING(chan, NvSubImageBlit, 0x0300, 3);
52 OUT_RING(chan, (region->sy << 16) | region->sx); 45 OUT_RING(chan, (region->sy << 16) | region->sx);
53 OUT_RING(chan, (region->dy << 16) | region->dx); 46 OUT_RING(chan, (region->dy << 16) | region->dx);
54 OUT_RING(chan, (region->height << 16) | region->width); 47 OUT_RING(chan, (region->height << 16) | region->width);
55 FIRE_RING(chan); 48 FIRE_RING(chan);
49 return 0;
56} 50}
57 51
58void 52int
59nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) 53nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
60{ 54{
61 struct nouveau_fbdev *nfbdev = info->par; 55 struct nouveau_fbdev *nfbdev = info->par;
62 struct drm_device *dev = nfbdev->dev; 56 struct drm_device *dev = nfbdev->dev;
63 struct drm_nouveau_private *dev_priv = dev->dev_private; 57 struct drm_nouveau_private *dev_priv = dev->dev_private;
64 struct nouveau_channel *chan = dev_priv->channel; 58 struct nouveau_channel *chan = dev_priv->channel;
59 int ret;
65 60
66 if (info->state != FBINFO_STATE_RUNNING) 61 ret = RING_SPACE(chan, 7);
67 return; 62 if (ret)
68 63 return ret;
69 if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 7)) {
70 nouveau_fbcon_gpu_lockup(info);
71 }
72
73 if (info->flags & FBINFO_HWACCEL_DISABLED) {
74 cfb_fillrect(info, rect);
75 return;
76 }
77 64
78 BEGIN_RING(chan, NvSubGdiRect, 0x02fc, 1); 65 BEGIN_RING(chan, NvSubGdiRect, 0x02fc, 1);
79 OUT_RING(chan, (rect->rop != ROP_COPY) ? 1 : 3); 66 OUT_RING(chan, (rect->rop != ROP_COPY) ? 1 : 3);
@@ -87,9 +74,10 @@ nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
87 OUT_RING(chan, (rect->dx << 16) | rect->dy); 74 OUT_RING(chan, (rect->dx << 16) | rect->dy);
88 OUT_RING(chan, (rect->width << 16) | rect->height); 75 OUT_RING(chan, (rect->width << 16) | rect->height);
89 FIRE_RING(chan); 76 FIRE_RING(chan);
77 return 0;
90} 78}
91 79
92void 80int
93nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) 81nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
94{ 82{
95 struct nouveau_fbdev *nfbdev = info->par; 83 struct nouveau_fbdev *nfbdev = info->par;
@@ -101,23 +89,14 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
101 uint32_t dsize; 89 uint32_t dsize;
102 uint32_t width; 90 uint32_t width;
103 uint32_t *data = (uint32_t *)image->data; 91 uint32_t *data = (uint32_t *)image->data;
92 int ret;
104 93
105 if (info->state != FBINFO_STATE_RUNNING) 94 if (image->depth != 1)
106 return; 95 return -ENODEV;
107
108 if (image->depth != 1) {
109 cfb_imageblit(info, image);
110 return;
111 }
112
113 if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 8)) {
114 nouveau_fbcon_gpu_lockup(info);
115 }
116 96
117 if (info->flags & FBINFO_HWACCEL_DISABLED) { 97 ret = RING_SPACE(chan, 8);
118 cfb_imageblit(info, image); 98 if (ret)
119 return; 99 return ret;
120 }
121 100
122 width = ALIGN(image->width, 8); 101 width = ALIGN(image->width, 8);
123 dsize = ALIGN(width * image->height, 32) >> 5; 102 dsize = ALIGN(width * image->height, 32) >> 5;
@@ -144,11 +123,9 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
144 while (dsize) { 123 while (dsize) {
145 int iter_len = dsize > 128 ? 128 : dsize; 124 int iter_len = dsize > 128 ? 128 : dsize;
146 125
147 if (RING_SPACE(chan, iter_len + 1)) { 126 ret = RING_SPACE(chan, iter_len + 1);
148 nouveau_fbcon_gpu_lockup(info); 127 if (ret)
149 cfb_imageblit(info, image); 128 return ret;
150 return;
151 }
152 129
153 BEGIN_RING(chan, NvSubGdiRect, 0x0c00, iter_len); 130 BEGIN_RING(chan, NvSubGdiRect, 0x0c00, iter_len);
154 OUT_RINGp(chan, data, iter_len); 131 OUT_RINGp(chan, data, iter_len);
@@ -157,22 +134,7 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
157 } 134 }
158 135
159 FIRE_RING(chan); 136 FIRE_RING(chan);
160} 137 return 0;
161
162static int
163nv04_fbcon_grobj_new(struct drm_device *dev, int class, uint32_t handle)
164{
165 struct drm_nouveau_private *dev_priv = dev->dev_private;
166 struct nouveau_gpuobj *obj = NULL;
167 int ret;
168
169 ret = nouveau_gpuobj_gr_new(dev_priv->channel, class, &obj);
170 if (ret)
171 return ret;
172
173 ret = nouveau_ramht_insert(dev_priv->channel, handle, obj);
174 nouveau_gpuobj_ref(NULL, &obj);
175 return ret;
176} 138}
177 139
178int 140int
@@ -214,29 +176,31 @@ nv04_fbcon_accel_init(struct fb_info *info)
214 return -EINVAL; 176 return -EINVAL;
215 } 177 }
216 178
217 ret = nv04_fbcon_grobj_new(dev, dev_priv->card_type >= NV_10 ? 179 ret = nouveau_gpuobj_gr_new(chan, NvCtxSurf2D,
218 0x0062 : 0x0042, NvCtxSurf2D); 180 dev_priv->card_type >= NV_10 ?
181 0x0062 : 0x0042);
219 if (ret) 182 if (ret)
220 return ret; 183 return ret;
221 184
222 ret = nv04_fbcon_grobj_new(dev, 0x0019, NvClipRect); 185 ret = nouveau_gpuobj_gr_new(chan, NvClipRect, 0x0019);
223 if (ret) 186 if (ret)
224 return ret; 187 return ret;
225 188
226 ret = nv04_fbcon_grobj_new(dev, 0x0043, NvRop); 189 ret = nouveau_gpuobj_gr_new(chan, NvRop, 0x0043);
227 if (ret) 190 if (ret)
228 return ret; 191 return ret;
229 192
230 ret = nv04_fbcon_grobj_new(dev, 0x0044, NvImagePatt); 193 ret = nouveau_gpuobj_gr_new(chan, NvImagePatt, 0x0044);
231 if (ret) 194 if (ret)
232 return ret; 195 return ret;
233 196
234 ret = nv04_fbcon_grobj_new(dev, 0x004a, NvGdiRect); 197 ret = nouveau_gpuobj_gr_new(chan, NvGdiRect, 0x004a);
235 if (ret) 198 if (ret)
236 return ret; 199 return ret;
237 200
238 ret = nv04_fbcon_grobj_new(dev, dev_priv->chipset >= 0x11 ? 201 ret = nouveau_gpuobj_gr_new(chan, NvImageBlit,
239 0x009f : 0x005f, NvImageBlit); 202 dev_priv->chipset >= 0x11 ?
203 0x009f : 0x005f);
240 if (ret) 204 if (ret)
241 return ret; 205 return ret;
242 206
diff --git a/drivers/gpu/drm/nouveau/nv04_fifo.c b/drivers/gpu/drm/nouveau/nv04_fifo.c
index 708293b7ddcd..f89d104698df 100644
--- a/drivers/gpu/drm/nouveau/nv04_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv04_fifo.c
@@ -28,6 +28,7 @@
28#include "drm.h" 28#include "drm.h"
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_ramht.h" 30#include "nouveau_ramht.h"
31#include "nouveau_util.h"
31 32
32#define NV04_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV04_RAMFC__SIZE)) 33#define NV04_RAMFC(c) (dev_priv->ramfc->pinst + ((c) * NV04_RAMFC__SIZE))
33#define NV04_RAMFC__SIZE 32 34#define NV04_RAMFC__SIZE 32
@@ -128,6 +129,11 @@ nv04_fifo_create_context(struct nouveau_channel *chan)
128 if (ret) 129 if (ret)
129 return ret; 130 return ret;
130 131
132 chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
133 NV03_USER(chan->id), PAGE_SIZE);
134 if (!chan->user)
135 return -ENOMEM;
136
131 spin_lock_irqsave(&dev_priv->context_switch_lock, flags); 137 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
132 138
133 /* Setup initial state */ 139 /* Setup initial state */
@@ -151,10 +157,31 @@ void
151nv04_fifo_destroy_context(struct nouveau_channel *chan) 157nv04_fifo_destroy_context(struct nouveau_channel *chan)
152{ 158{
153 struct drm_device *dev = chan->dev; 159 struct drm_device *dev = chan->dev;
160 struct drm_nouveau_private *dev_priv = dev->dev_private;
161 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
162 unsigned long flags;
154 163
155 nv_wr32(dev, NV04_PFIFO_MODE, 164 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
156 nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id)); 165 pfifo->reassign(dev, false);
157 166
167 /* Unload the context if it's the currently active one */
168 if (pfifo->channel_id(dev) == chan->id) {
169 pfifo->disable(dev);
170 pfifo->unload_context(dev);
171 pfifo->enable(dev);
172 }
173
174 /* Keep it from being rescheduled */
175 nv_mask(dev, NV04_PFIFO_MODE, 1 << chan->id, 0);
176
177 pfifo->reassign(dev, true);
178 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
179
180 /* Free the channel resources */
181 if (chan->user) {
182 iounmap(chan->user);
183 chan->user = NULL;
184 }
158 nouveau_gpuobj_ref(NULL, &chan->ramfc); 185 nouveau_gpuobj_ref(NULL, &chan->ramfc);
159} 186}
160 187
@@ -208,7 +235,7 @@ nv04_fifo_unload_context(struct drm_device *dev)
208 if (chid < 0 || chid >= dev_priv->engine.fifo.channels) 235 if (chid < 0 || chid >= dev_priv->engine.fifo.channels)
209 return 0; 236 return 0;
210 237
211 chan = dev_priv->fifos[chid]; 238 chan = dev_priv->channels.ptr[chid];
212 if (!chan) { 239 if (!chan) {
213 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid); 240 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
214 return -EINVAL; 241 return -EINVAL;
@@ -267,6 +294,7 @@ nv04_fifo_init_ramxx(struct drm_device *dev)
267static void 294static void
268nv04_fifo_init_intr(struct drm_device *dev) 295nv04_fifo_init_intr(struct drm_device *dev)
269{ 296{
297 nouveau_irq_register(dev, 8, nv04_fifo_isr);
270 nv_wr32(dev, 0x002100, 0xffffffff); 298 nv_wr32(dev, 0x002100, 0xffffffff);
271 nv_wr32(dev, 0x002140, 0xffffffff); 299 nv_wr32(dev, 0x002140, 0xffffffff);
272} 300}
@@ -289,7 +317,7 @@ nv04_fifo_init(struct drm_device *dev)
289 pfifo->reassign(dev, true); 317 pfifo->reassign(dev, true);
290 318
291 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 319 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
292 if (dev_priv->fifos[i]) { 320 if (dev_priv->channels.ptr[i]) {
293 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE); 321 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
294 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i)); 322 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
295 } 323 }
@@ -298,3 +326,207 @@ nv04_fifo_init(struct drm_device *dev)
298 return 0; 326 return 0;
299} 327}
300 328
329void
330nv04_fifo_fini(struct drm_device *dev)
331{
332 nv_wr32(dev, 0x2140, 0x00000000);
333 nouveau_irq_unregister(dev, 8);
334}
335
336static bool
337nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
338{
339 struct drm_nouveau_private *dev_priv = dev->dev_private;
340 struct nouveau_channel *chan = NULL;
341 struct nouveau_gpuobj *obj;
342 unsigned long flags;
343 const int subc = (addr >> 13) & 0x7;
344 const int mthd = addr & 0x1ffc;
345 bool handled = false;
346 u32 engine;
347
348 spin_lock_irqsave(&dev_priv->channels.lock, flags);
349 if (likely(chid >= 0 && chid < dev_priv->engine.fifo.channels))
350 chan = dev_priv->channels.ptr[chid];
351 if (unlikely(!chan))
352 goto out;
353
354 switch (mthd) {
355 case 0x0000: /* bind object to subchannel */
356 obj = nouveau_ramht_find(chan, data);
357 if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
358 break;
359
360 chan->sw_subchannel[subc] = obj->class;
361 engine = 0x0000000f << (subc * 4);
362
363 nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
364 handled = true;
365 break;
366 default:
367 engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
368 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
369 break;
370
371 if (!nouveau_gpuobj_mthd_call(chan, chan->sw_subchannel[subc],
372 mthd, data))
373 handled = true;
374 break;
375 }
376
377out:
378 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
379 return handled;
380}
381
382void
383nv04_fifo_isr(struct drm_device *dev)
384{
385 struct drm_nouveau_private *dev_priv = dev->dev_private;
386 struct nouveau_engine *engine = &dev_priv->engine;
387 uint32_t status, reassign;
388 int cnt = 0;
389
390 reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
391 while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
392 uint32_t chid, get;
393
394 nv_wr32(dev, NV03_PFIFO_CACHES, 0);
395
396 chid = engine->fifo.channel_id(dev);
397 get = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
398
399 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
400 uint32_t mthd, data;
401 int ptr;
402
403 /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
404 * wrapping on my G80 chips, but CACHE1 isn't big
405 * enough for this much data.. Tests show that it
406 * wraps around to the start at GET=0x800.. No clue
407 * as to why..
408 */
409 ptr = (get & 0x7ff) >> 2;
410
411 if (dev_priv->card_type < NV_40) {
412 mthd = nv_rd32(dev,
413 NV04_PFIFO_CACHE1_METHOD(ptr));
414 data = nv_rd32(dev,
415 NV04_PFIFO_CACHE1_DATA(ptr));
416 } else {
417 mthd = nv_rd32(dev,
418 NV40_PFIFO_CACHE1_METHOD(ptr));
419 data = nv_rd32(dev,
420 NV40_PFIFO_CACHE1_DATA(ptr));
421 }
422
423 if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
424 NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
425 "Mthd 0x%04x Data 0x%08x\n",
426 chid, (mthd >> 13) & 7, mthd & 0x1ffc,
427 data);
428 }
429
430 nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
431 nv_wr32(dev, NV03_PFIFO_INTR_0,
432 NV_PFIFO_INTR_CACHE_ERROR);
433
434 nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
435 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
436 nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
437 nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
438 nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
439 nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
440
441 nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
442 nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
443 nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
444
445 status &= ~NV_PFIFO_INTR_CACHE_ERROR;
446 }
447
448 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
449 u32 dma_get = nv_rd32(dev, 0x003244);
450 u32 dma_put = nv_rd32(dev, 0x003240);
451 u32 push = nv_rd32(dev, 0x003220);
452 u32 state = nv_rd32(dev, 0x003228);
453
454 if (dev_priv->card_type == NV_50) {
455 u32 ho_get = nv_rd32(dev, 0x003328);
456 u32 ho_put = nv_rd32(dev, 0x003320);
457 u32 ib_get = nv_rd32(dev, 0x003334);
458 u32 ib_put = nv_rd32(dev, 0x003330);
459
460 if (nouveau_ratelimit())
461 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
462 "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
463 "State 0x%08x Push 0x%08x\n",
464 chid, ho_get, dma_get, ho_put,
465 dma_put, ib_get, ib_put, state,
466 push);
467
468 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
469 nv_wr32(dev, 0x003364, 0x00000000);
470 if (dma_get != dma_put || ho_get != ho_put) {
471 nv_wr32(dev, 0x003244, dma_put);
472 nv_wr32(dev, 0x003328, ho_put);
473 } else
474 if (ib_get != ib_put) {
475 nv_wr32(dev, 0x003334, ib_put);
476 }
477 } else {
478 NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
479 "Put 0x%08x State 0x%08x Push 0x%08x\n",
480 chid, dma_get, dma_put, state, push);
481
482 if (dma_get != dma_put)
483 nv_wr32(dev, 0x003244, dma_put);
484 }
485
486 nv_wr32(dev, 0x003228, 0x00000000);
487 nv_wr32(dev, 0x003220, 0x00000001);
488 nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
489 status &= ~NV_PFIFO_INTR_DMA_PUSHER;
490 }
491
492 if (status & NV_PFIFO_INTR_SEMAPHORE) {
493 uint32_t sem;
494
495 status &= ~NV_PFIFO_INTR_SEMAPHORE;
496 nv_wr32(dev, NV03_PFIFO_INTR_0,
497 NV_PFIFO_INTR_SEMAPHORE);
498
499 sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
500 nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
501
502 nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
503 nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
504 }
505
506 if (dev_priv->card_type == NV_50) {
507 if (status & 0x00000010) {
508 nv50_fb_vm_trap(dev, 1, "PFIFO_BAR_FAULT");
509 status &= ~0x00000010;
510 nv_wr32(dev, 0x002100, 0x00000010);
511 }
512 }
513
514 if (status) {
515 if (nouveau_ratelimit())
516 NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
517 status, chid);
518 nv_wr32(dev, NV03_PFIFO_INTR_0, status);
519 status = 0;
520 }
521
522 nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
523 }
524
525 if (status) {
526 NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
527 nv_wr32(dev, 0x2140, 0);
528 nv_wr32(dev, 0x140, 0);
529 }
530
531 nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
532}
diff --git a/drivers/gpu/drm/nouveau/nv04_graph.c b/drivers/gpu/drm/nouveau/nv04_graph.c
index c8973421b635..af75015068d6 100644
--- a/drivers/gpu/drm/nouveau/nv04_graph.c
+++ b/drivers/gpu/drm/nouveau/nv04_graph.c
@@ -26,6 +26,11 @@
26#include "drm.h" 26#include "drm.h"
27#include "nouveau_drm.h" 27#include "nouveau_drm.h"
28#include "nouveau_drv.h" 28#include "nouveau_drv.h"
29#include "nouveau_hw.h"
30#include "nouveau_util.h"
31
32static int nv04_graph_register(struct drm_device *dev);
33static void nv04_graph_isr(struct drm_device *dev);
29 34
30static uint32_t nv04_graph_ctx_regs[] = { 35static uint32_t nv04_graph_ctx_regs[] = {
31 0x0040053c, 36 0x0040053c,
@@ -357,10 +362,10 @@ nv04_graph_channel(struct drm_device *dev)
357 if (chid >= dev_priv->engine.fifo.channels) 362 if (chid >= dev_priv->engine.fifo.channels)
358 return NULL; 363 return NULL;
359 364
360 return dev_priv->fifos[chid]; 365 return dev_priv->channels.ptr[chid];
361} 366}
362 367
363void 368static void
364nv04_graph_context_switch(struct drm_device *dev) 369nv04_graph_context_switch(struct drm_device *dev)
365{ 370{
366 struct drm_nouveau_private *dev_priv = dev->dev_private; 371 struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -368,7 +373,6 @@ nv04_graph_context_switch(struct drm_device *dev)
368 struct nouveau_channel *chan = NULL; 373 struct nouveau_channel *chan = NULL;
369 int chid; 374 int chid;
370 375
371 pgraph->fifo_access(dev, false);
372 nouveau_wait_for_idle(dev); 376 nouveau_wait_for_idle(dev);
373 377
374 /* If previous context is valid, we need to save it */ 378 /* If previous context is valid, we need to save it */
@@ -376,11 +380,9 @@ nv04_graph_context_switch(struct drm_device *dev)
376 380
377 /* Load context for next channel */ 381 /* Load context for next channel */
378 chid = dev_priv->engine.fifo.channel_id(dev); 382 chid = dev_priv->engine.fifo.channel_id(dev);
379 chan = dev_priv->fifos[chid]; 383 chan = dev_priv->channels.ptr[chid];
380 if (chan) 384 if (chan)
381 nv04_graph_load_context(chan); 385 nv04_graph_load_context(chan);
382
383 pgraph->fifo_access(dev, true);
384} 386}
385 387
386static uint32_t *ctx_reg(struct graph_state *ctx, uint32_t reg) 388static uint32_t *ctx_reg(struct graph_state *ctx, uint32_t reg)
@@ -412,10 +414,25 @@ int nv04_graph_create_context(struct nouveau_channel *chan)
412 414
413void nv04_graph_destroy_context(struct nouveau_channel *chan) 415void nv04_graph_destroy_context(struct nouveau_channel *chan)
414{ 416{
417 struct drm_device *dev = chan->dev;
418 struct drm_nouveau_private *dev_priv = dev->dev_private;
419 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
415 struct graph_state *pgraph_ctx = chan->pgraph_ctx; 420 struct graph_state *pgraph_ctx = chan->pgraph_ctx;
421 unsigned long flags;
422
423 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
424 pgraph->fifo_access(dev, false);
425
426 /* Unload the context if it's the currently active one */
427 if (pgraph->channel(dev) == chan)
428 pgraph->unload_context(dev);
416 429
430 /* Free the context resources */
417 kfree(pgraph_ctx); 431 kfree(pgraph_ctx);
418 chan->pgraph_ctx = NULL; 432 chan->pgraph_ctx = NULL;
433
434 pgraph->fifo_access(dev, true);
435 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
419} 436}
420 437
421int nv04_graph_load_context(struct nouveau_channel *chan) 438int nv04_graph_load_context(struct nouveau_channel *chan)
@@ -468,13 +485,19 @@ int nv04_graph_init(struct drm_device *dev)
468{ 485{
469 struct drm_nouveau_private *dev_priv = dev->dev_private; 486 struct drm_nouveau_private *dev_priv = dev->dev_private;
470 uint32_t tmp; 487 uint32_t tmp;
488 int ret;
471 489
472 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & 490 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
473 ~NV_PMC_ENABLE_PGRAPH); 491 ~NV_PMC_ENABLE_PGRAPH);
474 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) | 492 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
475 NV_PMC_ENABLE_PGRAPH); 493 NV_PMC_ENABLE_PGRAPH);
476 494
495 ret = nv04_graph_register(dev);
496 if (ret)
497 return ret;
498
477 /* Enable PGRAPH interrupts */ 499 /* Enable PGRAPH interrupts */
500 nouveau_irq_register(dev, 12, nv04_graph_isr);
478 nv_wr32(dev, NV03_PGRAPH_INTR, 0xFFFFFFFF); 501 nv_wr32(dev, NV03_PGRAPH_INTR, 0xFFFFFFFF);
479 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF); 502 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
480 503
@@ -510,6 +533,8 @@ int nv04_graph_init(struct drm_device *dev)
510 533
511void nv04_graph_takedown(struct drm_device *dev) 534void nv04_graph_takedown(struct drm_device *dev)
512{ 535{
536 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
537 nouveau_irq_unregister(dev, 12);
513} 538}
514 539
515void 540void
@@ -524,13 +549,27 @@ nv04_graph_fifo_access(struct drm_device *dev, bool enabled)
524} 549}
525 550
526static int 551static int
527nv04_graph_mthd_set_ref(struct nouveau_channel *chan, int grclass, 552nv04_graph_mthd_set_ref(struct nouveau_channel *chan,
528 int mthd, uint32_t data) 553 u32 class, u32 mthd, u32 data)
529{ 554{
530 atomic_set(&chan->fence.last_sequence_irq, data); 555 atomic_set(&chan->fence.last_sequence_irq, data);
531 return 0; 556 return 0;
532} 557}
533 558
559int
560nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
561 u32 class, u32 mthd, u32 data)
562{
563 struct drm_device *dev = chan->dev;
564 struct nouveau_page_flip_state s;
565
566 if (!nouveau_finish_page_flip(chan, &s))
567 nv_set_crtc_base(dev, s.crtc,
568 s.offset + s.y * s.pitch + s.x * s.bpp / 8);
569
570 return 0;
571}
572
534/* 573/*
535 * Software methods, why they are needed, and how they all work: 574 * Software methods, why they are needed, and how they all work:
536 * 575 *
@@ -606,12 +645,12 @@ nv04_graph_mthd_set_ref(struct nouveau_channel *chan, int grclass,
606 */ 645 */
607 646
608static void 647static void
609nv04_graph_set_ctx1(struct nouveau_channel *chan, uint32_t mask, uint32_t value) 648nv04_graph_set_ctx1(struct nouveau_channel *chan, u32 mask, u32 value)
610{ 649{
611 struct drm_device *dev = chan->dev; 650 struct drm_device *dev = chan->dev;
612 uint32_t instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4; 651 u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
613 int subc = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 13) & 0x7; 652 int subc = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 13) & 0x7;
614 uint32_t tmp; 653 u32 tmp;
615 654
616 tmp = nv_ri32(dev, instance); 655 tmp = nv_ri32(dev, instance);
617 tmp &= ~mask; 656 tmp &= ~mask;
@@ -623,11 +662,11 @@ nv04_graph_set_ctx1(struct nouveau_channel *chan, uint32_t mask, uint32_t value)
623} 662}
624 663
625static void 664static void
626nv04_graph_set_ctx_val(struct nouveau_channel *chan, uint32_t mask, uint32_t value) 665nv04_graph_set_ctx_val(struct nouveau_channel *chan, u32 mask, u32 value)
627{ 666{
628 struct drm_device *dev = chan->dev; 667 struct drm_device *dev = chan->dev;
629 uint32_t instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4; 668 u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
630 uint32_t tmp, ctx1; 669 u32 tmp, ctx1;
631 int class, op, valid = 1; 670 int class, op, valid = 1;
632 671
633 ctx1 = nv_ri32(dev, instance); 672 ctx1 = nv_ri32(dev, instance);
@@ -672,13 +711,13 @@ nv04_graph_set_ctx_val(struct nouveau_channel *chan, uint32_t mask, uint32_t val
672} 711}
673 712
674static int 713static int
675nv04_graph_mthd_set_operation(struct nouveau_channel *chan, int grclass, 714nv04_graph_mthd_set_operation(struct nouveau_channel *chan,
676 int mthd, uint32_t data) 715 u32 class, u32 mthd, u32 data)
677{ 716{
678 if (data > 5) 717 if (data > 5)
679 return 1; 718 return 1;
680 /* Old versions of the objects only accept first three operations. */ 719 /* Old versions of the objects only accept first three operations. */
681 if (data > 2 && grclass < 0x40) 720 if (data > 2 && class < 0x40)
682 return 1; 721 return 1;
683 nv04_graph_set_ctx1(chan, 0x00038000, data << 15); 722 nv04_graph_set_ctx1(chan, 0x00038000, data << 15);
684 /* changing operation changes set of objects needed for validation */ 723 /* changing operation changes set of objects needed for validation */
@@ -687,8 +726,8 @@ nv04_graph_mthd_set_operation(struct nouveau_channel *chan, int grclass,
687} 726}
688 727
689static int 728static int
690nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan, int grclass, 729nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan,
691 int mthd, uint32_t data) 730 u32 class, u32 mthd, u32 data)
692{ 731{
693 uint32_t min = data & 0xffff, max; 732 uint32_t min = data & 0xffff, max;
694 uint32_t w = data >> 16; 733 uint32_t w = data >> 16;
@@ -706,8 +745,8 @@ nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan, int grclass,
706} 745}
707 746
708static int 747static int
709nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan, int grclass, 748nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan,
710 int mthd, uint32_t data) 749 u32 class, u32 mthd, u32 data)
711{ 750{
712 uint32_t min = data & 0xffff, max; 751 uint32_t min = data & 0xffff, max;
713 uint32_t w = data >> 16; 752 uint32_t w = data >> 16;
@@ -725,8 +764,8 @@ nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan, int grclass,
725} 764}
726 765
727static int 766static int
728nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan, int grclass, 767nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan,
729 int mthd, uint32_t data) 768 u32 class, u32 mthd, u32 data)
730{ 769{
731 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 770 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
732 case 0x30: 771 case 0x30:
@@ -742,8 +781,8 @@ nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan, int grclass,
742} 781}
743 782
744static int 783static int
745nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan, int grclass, 784nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan,
746 int mthd, uint32_t data) 785 u32 class, u32 mthd, u32 data)
747{ 786{
748 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 787 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
749 case 0x30: 788 case 0x30:
@@ -763,8 +802,8 @@ nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan, int grclass,
763} 802}
764 803
765static int 804static int
766nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan, int grclass, 805nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan,
767 int mthd, uint32_t data) 806 u32 class, u32 mthd, u32 data)
768{ 807{
769 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 808 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
770 case 0x30: 809 case 0x30:
@@ -778,8 +817,8 @@ nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan, int grclass,
778} 817}
779 818
780static int 819static int
781nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan, int grclass, 820nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan,
782 int mthd, uint32_t data) 821 u32 class, u32 mthd, u32 data)
783{ 822{
784 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 823 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
785 case 0x30: 824 case 0x30:
@@ -793,8 +832,8 @@ nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan, int grclass,
793} 832}
794 833
795static int 834static int
796nv04_graph_mthd_bind_rop(struct nouveau_channel *chan, int grclass, 835nv04_graph_mthd_bind_rop(struct nouveau_channel *chan,
797 int mthd, uint32_t data) 836 u32 class, u32 mthd, u32 data)
798{ 837{
799 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 838 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
800 case 0x30: 839 case 0x30:
@@ -808,8 +847,8 @@ nv04_graph_mthd_bind_rop(struct nouveau_channel *chan, int grclass,
808} 847}
809 848
810static int 849static int
811nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan, int grclass, 850nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan,
812 int mthd, uint32_t data) 851 u32 class, u32 mthd, u32 data)
813{ 852{
814 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 853 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
815 case 0x30: 854 case 0x30:
@@ -823,8 +862,8 @@ nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan, int grclass,
823} 862}
824 863
825static int 864static int
826nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan, int grclass, 865nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan,
827 int mthd, uint32_t data) 866 u32 class, u32 mthd, u32 data)
828{ 867{
829 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 868 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
830 case 0x30: 869 case 0x30:
@@ -838,8 +877,8 @@ nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan, int grclass,
838} 877}
839 878
840static int 879static int
841nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan, int grclass, 880nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan,
842 int mthd, uint32_t data) 881 u32 class, u32 mthd, u32 data)
843{ 882{
844 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 883 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
845 case 0x30: 884 case 0x30:
@@ -853,8 +892,8 @@ nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan, int grclass,
853} 892}
854 893
855static int 894static int
856nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan, int grclass, 895nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan,
857 int mthd, uint32_t data) 896 u32 class, u32 mthd, u32 data)
858{ 897{
859 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 898 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
860 case 0x30: 899 case 0x30:
@@ -868,8 +907,8 @@ nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan, int grclass,
868} 907}
869 908
870static int 909static int
871nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan, int grclass, 910nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan,
872 int mthd, uint32_t data) 911 u32 class, u32 mthd, u32 data)
873{ 912{
874 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 913 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
875 case 0x30: 914 case 0x30:
@@ -883,8 +922,8 @@ nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan, int grclass,
883} 922}
884 923
885static int 924static int
886nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan, int grclass, 925nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan,
887 int mthd, uint32_t data) 926 u32 class, u32 mthd, u32 data)
888{ 927{
889 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 928 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
890 case 0x30: 929 case 0x30:
@@ -898,8 +937,8 @@ nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan, int grclass,
898} 937}
899 938
900static int 939static int
901nv04_graph_mthd_bind_clip(struct nouveau_channel *chan, int grclass, 940nv04_graph_mthd_bind_clip(struct nouveau_channel *chan,
902 int mthd, uint32_t data) 941 u32 class, u32 mthd, u32 data)
903{ 942{
904 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 943 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
905 case 0x30: 944 case 0x30:
@@ -913,8 +952,8 @@ nv04_graph_mthd_bind_clip(struct nouveau_channel *chan, int grclass,
913} 952}
914 953
915static int 954static int
916nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan, int grclass, 955nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan,
917 int mthd, uint32_t data) 956 u32 class, u32 mthd, u32 data)
918{ 957{
919 switch (nv_ri32(chan->dev, data << 4) & 0xff) { 958 switch (nv_ri32(chan->dev, data << 4) & 0xff) {
920 case 0x30: 959 case 0x30:
@@ -930,194 +969,346 @@ nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan, int grclass,
930 return 1; 969 return 1;
931} 970}
932 971
933static struct nouveau_pgraph_object_method nv04_graph_mthds_sw[] = { 972static int
934 { 0x0150, nv04_graph_mthd_set_ref }, 973nv04_graph_register(struct drm_device *dev)
935 {} 974{
936}; 975 struct drm_nouveau_private *dev_priv = dev->dev_private;
937
938static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_gdirect[] = {
939 { 0x0184, nv04_graph_mthd_bind_nv01_patt },
940 { 0x0188, nv04_graph_mthd_bind_rop },
941 { 0x018c, nv04_graph_mthd_bind_beta1 },
942 { 0x0190, nv04_graph_mthd_bind_surf_dst },
943 { 0x02fc, nv04_graph_mthd_set_operation },
944 {},
945};
946
947static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_gdirect[] = {
948 { 0x0188, nv04_graph_mthd_bind_nv04_patt },
949 { 0x018c, nv04_graph_mthd_bind_rop },
950 { 0x0190, nv04_graph_mthd_bind_beta1 },
951 { 0x0194, nv04_graph_mthd_bind_beta4 },
952 { 0x0198, nv04_graph_mthd_bind_surf2d },
953 { 0x02fc, nv04_graph_mthd_set_operation },
954 {},
955};
956
957static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_imageblit[] = {
958 { 0x0184, nv04_graph_mthd_bind_chroma },
959 { 0x0188, nv04_graph_mthd_bind_clip },
960 { 0x018c, nv04_graph_mthd_bind_nv01_patt },
961 { 0x0190, nv04_graph_mthd_bind_rop },
962 { 0x0194, nv04_graph_mthd_bind_beta1 },
963 { 0x0198, nv04_graph_mthd_bind_surf_dst },
964 { 0x019c, nv04_graph_mthd_bind_surf_src },
965 { 0x02fc, nv04_graph_mthd_set_operation },
966 {},
967};
968
969static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_imageblit_ifc[] = {
970 { 0x0184, nv04_graph_mthd_bind_chroma },
971 { 0x0188, nv04_graph_mthd_bind_clip },
972 { 0x018c, nv04_graph_mthd_bind_nv04_patt },
973 { 0x0190, nv04_graph_mthd_bind_rop },
974 { 0x0194, nv04_graph_mthd_bind_beta1 },
975 { 0x0198, nv04_graph_mthd_bind_beta4 },
976 { 0x019c, nv04_graph_mthd_bind_surf2d },
977 { 0x02fc, nv04_graph_mthd_set_operation },
978 {},
979};
980
981static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_iifc[] = {
982 { 0x0188, nv04_graph_mthd_bind_chroma },
983 { 0x018c, nv04_graph_mthd_bind_clip },
984 { 0x0190, nv04_graph_mthd_bind_nv04_patt },
985 { 0x0194, nv04_graph_mthd_bind_rop },
986 { 0x0198, nv04_graph_mthd_bind_beta1 },
987 { 0x019c, nv04_graph_mthd_bind_beta4 },
988 { 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf },
989 { 0x03e4, nv04_graph_mthd_set_operation },
990 {},
991};
992
993static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_ifc[] = {
994 { 0x0184, nv04_graph_mthd_bind_chroma },
995 { 0x0188, nv04_graph_mthd_bind_clip },
996 { 0x018c, nv04_graph_mthd_bind_nv01_patt },
997 { 0x0190, nv04_graph_mthd_bind_rop },
998 { 0x0194, nv04_graph_mthd_bind_beta1 },
999 { 0x0198, nv04_graph_mthd_bind_surf_dst },
1000 { 0x02fc, nv04_graph_mthd_set_operation },
1001 {},
1002};
1003
1004static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_sifc[] = {
1005 { 0x0184, nv04_graph_mthd_bind_chroma },
1006 { 0x0188, nv04_graph_mthd_bind_nv01_patt },
1007 { 0x018c, nv04_graph_mthd_bind_rop },
1008 { 0x0190, nv04_graph_mthd_bind_beta1 },
1009 { 0x0194, nv04_graph_mthd_bind_surf_dst },
1010 { 0x02fc, nv04_graph_mthd_set_operation },
1011 {},
1012};
1013
1014static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_sifc[] = {
1015 { 0x0184, nv04_graph_mthd_bind_chroma },
1016 { 0x0188, nv04_graph_mthd_bind_nv04_patt },
1017 { 0x018c, nv04_graph_mthd_bind_rop },
1018 { 0x0190, nv04_graph_mthd_bind_beta1 },
1019 { 0x0194, nv04_graph_mthd_bind_beta4 },
1020 { 0x0198, nv04_graph_mthd_bind_surf2d },
1021 { 0x02fc, nv04_graph_mthd_set_operation },
1022 {},
1023};
1024
1025static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_sifm[] = {
1026 { 0x0188, nv04_graph_mthd_bind_nv01_patt },
1027 { 0x018c, nv04_graph_mthd_bind_rop },
1028 { 0x0190, nv04_graph_mthd_bind_beta1 },
1029 { 0x0194, nv04_graph_mthd_bind_surf_dst },
1030 { 0x0304, nv04_graph_mthd_set_operation },
1031 {},
1032};
1033
1034static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_sifm[] = {
1035 { 0x0188, nv04_graph_mthd_bind_nv04_patt },
1036 { 0x018c, nv04_graph_mthd_bind_rop },
1037 { 0x0190, nv04_graph_mthd_bind_beta1 },
1038 { 0x0194, nv04_graph_mthd_bind_beta4 },
1039 { 0x0198, nv04_graph_mthd_bind_surf2d_swzsurf },
1040 { 0x0304, nv04_graph_mthd_set_operation },
1041 {},
1042};
1043 976
1044static struct nouveau_pgraph_object_method nv04_graph_mthds_nv01_shape[] = { 977 if (dev_priv->engine.graph.registered)
1045 { 0x0184, nv04_graph_mthd_bind_clip }, 978 return 0;
1046 { 0x0188, nv04_graph_mthd_bind_nv01_patt },
1047 { 0x018c, nv04_graph_mthd_bind_rop },
1048 { 0x0190, nv04_graph_mthd_bind_beta1 },
1049 { 0x0194, nv04_graph_mthd_bind_surf_dst },
1050 { 0x02fc, nv04_graph_mthd_set_operation },
1051 {},
1052};
1053 979
1054static struct nouveau_pgraph_object_method nv04_graph_mthds_nv04_shape[] = { 980 /* dvd subpicture */
1055 { 0x0184, nv04_graph_mthd_bind_clip }, 981 NVOBJ_CLASS(dev, 0x0038, GR);
1056 { 0x0188, nv04_graph_mthd_bind_nv04_patt }, 982
1057 { 0x018c, nv04_graph_mthd_bind_rop }, 983 /* m2mf */
1058 { 0x0190, nv04_graph_mthd_bind_beta1 }, 984 NVOBJ_CLASS(dev, 0x0039, GR);
1059 { 0x0194, nv04_graph_mthd_bind_beta4 }, 985
1060 { 0x0198, nv04_graph_mthd_bind_surf2d }, 986 /* nv03 gdirect */
1061 { 0x02fc, nv04_graph_mthd_set_operation }, 987 NVOBJ_CLASS(dev, 0x004b, GR);
1062 {}, 988 NVOBJ_MTHD (dev, 0x004b, 0x0184, nv04_graph_mthd_bind_nv01_patt);
989 NVOBJ_MTHD (dev, 0x004b, 0x0188, nv04_graph_mthd_bind_rop);
990 NVOBJ_MTHD (dev, 0x004b, 0x018c, nv04_graph_mthd_bind_beta1);
991 NVOBJ_MTHD (dev, 0x004b, 0x0190, nv04_graph_mthd_bind_surf_dst);
992 NVOBJ_MTHD (dev, 0x004b, 0x02fc, nv04_graph_mthd_set_operation);
993
994 /* nv04 gdirect */
995 NVOBJ_CLASS(dev, 0x004a, GR);
996 NVOBJ_MTHD (dev, 0x004a, 0x0188, nv04_graph_mthd_bind_nv04_patt);
997 NVOBJ_MTHD (dev, 0x004a, 0x018c, nv04_graph_mthd_bind_rop);
998 NVOBJ_MTHD (dev, 0x004a, 0x0190, nv04_graph_mthd_bind_beta1);
999 NVOBJ_MTHD (dev, 0x004a, 0x0194, nv04_graph_mthd_bind_beta4);
1000 NVOBJ_MTHD (dev, 0x004a, 0x0198, nv04_graph_mthd_bind_surf2d);
1001 NVOBJ_MTHD (dev, 0x004a, 0x02fc, nv04_graph_mthd_set_operation);
1002
1003 /* nv01 imageblit */
1004 NVOBJ_CLASS(dev, 0x001f, GR);
1005 NVOBJ_MTHD (dev, 0x001f, 0x0184, nv04_graph_mthd_bind_chroma);
1006 NVOBJ_MTHD (dev, 0x001f, 0x0188, nv04_graph_mthd_bind_clip);
1007 NVOBJ_MTHD (dev, 0x001f, 0x018c, nv04_graph_mthd_bind_nv01_patt);
1008 NVOBJ_MTHD (dev, 0x001f, 0x0190, nv04_graph_mthd_bind_rop);
1009 NVOBJ_MTHD (dev, 0x001f, 0x0194, nv04_graph_mthd_bind_beta1);
1010 NVOBJ_MTHD (dev, 0x001f, 0x0198, nv04_graph_mthd_bind_surf_dst);
1011 NVOBJ_MTHD (dev, 0x001f, 0x019c, nv04_graph_mthd_bind_surf_src);
1012 NVOBJ_MTHD (dev, 0x001f, 0x02fc, nv04_graph_mthd_set_operation);
1013
1014 /* nv04 imageblit */
1015 NVOBJ_CLASS(dev, 0x005f, GR);
1016 NVOBJ_MTHD (dev, 0x005f, 0x0184, nv04_graph_mthd_bind_chroma);
1017 NVOBJ_MTHD (dev, 0x005f, 0x0188, nv04_graph_mthd_bind_clip);
1018 NVOBJ_MTHD (dev, 0x005f, 0x018c, nv04_graph_mthd_bind_nv04_patt);
1019 NVOBJ_MTHD (dev, 0x005f, 0x0190, nv04_graph_mthd_bind_rop);
1020 NVOBJ_MTHD (dev, 0x005f, 0x0194, nv04_graph_mthd_bind_beta1);
1021 NVOBJ_MTHD (dev, 0x005f, 0x0198, nv04_graph_mthd_bind_beta4);
1022 NVOBJ_MTHD (dev, 0x005f, 0x019c, nv04_graph_mthd_bind_surf2d);
1023 NVOBJ_MTHD (dev, 0x005f, 0x02fc, nv04_graph_mthd_set_operation);
1024
1025 /* nv04 iifc */
1026 NVOBJ_CLASS(dev, 0x0060, GR);
1027 NVOBJ_MTHD (dev, 0x0060, 0x0188, nv04_graph_mthd_bind_chroma);
1028 NVOBJ_MTHD (dev, 0x0060, 0x018c, nv04_graph_mthd_bind_clip);
1029 NVOBJ_MTHD (dev, 0x0060, 0x0190, nv04_graph_mthd_bind_nv04_patt);
1030 NVOBJ_MTHD (dev, 0x0060, 0x0194, nv04_graph_mthd_bind_rop);
1031 NVOBJ_MTHD (dev, 0x0060, 0x0198, nv04_graph_mthd_bind_beta1);
1032 NVOBJ_MTHD (dev, 0x0060, 0x019c, nv04_graph_mthd_bind_beta4);
1033 NVOBJ_MTHD (dev, 0x0060, 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf);
1034 NVOBJ_MTHD (dev, 0x0060, 0x03e4, nv04_graph_mthd_set_operation);
1035
1036 /* nv05 iifc */
1037 NVOBJ_CLASS(dev, 0x0064, GR);
1038
1039 /* nv01 ifc */
1040 NVOBJ_CLASS(dev, 0x0021, GR);
1041 NVOBJ_MTHD (dev, 0x0021, 0x0184, nv04_graph_mthd_bind_chroma);
1042 NVOBJ_MTHD (dev, 0x0021, 0x0188, nv04_graph_mthd_bind_clip);
1043 NVOBJ_MTHD (dev, 0x0021, 0x018c, nv04_graph_mthd_bind_nv01_patt);
1044 NVOBJ_MTHD (dev, 0x0021, 0x0190, nv04_graph_mthd_bind_rop);
1045 NVOBJ_MTHD (dev, 0x0021, 0x0194, nv04_graph_mthd_bind_beta1);
1046 NVOBJ_MTHD (dev, 0x0021, 0x0198, nv04_graph_mthd_bind_surf_dst);
1047 NVOBJ_MTHD (dev, 0x0021, 0x02fc, nv04_graph_mthd_set_operation);
1048
1049 /* nv04 ifc */
1050 NVOBJ_CLASS(dev, 0x0061, GR);
1051 NVOBJ_MTHD (dev, 0x0061, 0x0184, nv04_graph_mthd_bind_chroma);
1052 NVOBJ_MTHD (dev, 0x0061, 0x0188, nv04_graph_mthd_bind_clip);
1053 NVOBJ_MTHD (dev, 0x0061, 0x018c, nv04_graph_mthd_bind_nv04_patt);
1054 NVOBJ_MTHD (dev, 0x0061, 0x0190, nv04_graph_mthd_bind_rop);
1055 NVOBJ_MTHD (dev, 0x0061, 0x0194, nv04_graph_mthd_bind_beta1);
1056 NVOBJ_MTHD (dev, 0x0061, 0x0198, nv04_graph_mthd_bind_beta4);
1057 NVOBJ_MTHD (dev, 0x0061, 0x019c, nv04_graph_mthd_bind_surf2d);
1058 NVOBJ_MTHD (dev, 0x0061, 0x02fc, nv04_graph_mthd_set_operation);
1059
1060 /* nv05 ifc */
1061 NVOBJ_CLASS(dev, 0x0065, GR);
1062
1063 /* nv03 sifc */
1064 NVOBJ_CLASS(dev, 0x0036, GR);
1065 NVOBJ_MTHD (dev, 0x0036, 0x0184, nv04_graph_mthd_bind_chroma);
1066 NVOBJ_MTHD (dev, 0x0036, 0x0188, nv04_graph_mthd_bind_nv01_patt);
1067 NVOBJ_MTHD (dev, 0x0036, 0x018c, nv04_graph_mthd_bind_rop);
1068 NVOBJ_MTHD (dev, 0x0036, 0x0190, nv04_graph_mthd_bind_beta1);
1069 NVOBJ_MTHD (dev, 0x0036, 0x0194, nv04_graph_mthd_bind_surf_dst);
1070 NVOBJ_MTHD (dev, 0x0036, 0x02fc, nv04_graph_mthd_set_operation);
1071
1072 /* nv04 sifc */
1073 NVOBJ_CLASS(dev, 0x0076, GR);
1074 NVOBJ_MTHD (dev, 0x0076, 0x0184, nv04_graph_mthd_bind_chroma);
1075 NVOBJ_MTHD (dev, 0x0076, 0x0188, nv04_graph_mthd_bind_nv04_patt);
1076 NVOBJ_MTHD (dev, 0x0076, 0x018c, nv04_graph_mthd_bind_rop);
1077 NVOBJ_MTHD (dev, 0x0076, 0x0190, nv04_graph_mthd_bind_beta1);
1078 NVOBJ_MTHD (dev, 0x0076, 0x0194, nv04_graph_mthd_bind_beta4);
1079 NVOBJ_MTHD (dev, 0x0076, 0x0198, nv04_graph_mthd_bind_surf2d);
1080 NVOBJ_MTHD (dev, 0x0076, 0x02fc, nv04_graph_mthd_set_operation);
1081
1082 /* nv05 sifc */
1083 NVOBJ_CLASS(dev, 0x0066, GR);
1084
1085 /* nv03 sifm */
1086 NVOBJ_CLASS(dev, 0x0037, GR);
1087 NVOBJ_MTHD (dev, 0x0037, 0x0188, nv04_graph_mthd_bind_nv01_patt);
1088 NVOBJ_MTHD (dev, 0x0037, 0x018c, nv04_graph_mthd_bind_rop);
1089 NVOBJ_MTHD (dev, 0x0037, 0x0190, nv04_graph_mthd_bind_beta1);
1090 NVOBJ_MTHD (dev, 0x0037, 0x0194, nv04_graph_mthd_bind_surf_dst);
1091 NVOBJ_MTHD (dev, 0x0037, 0x0304, nv04_graph_mthd_set_operation);
1092
1093 /* nv04 sifm */
1094 NVOBJ_CLASS(dev, 0x0077, GR);
1095 NVOBJ_MTHD (dev, 0x0077, 0x0188, nv04_graph_mthd_bind_nv04_patt);
1096 NVOBJ_MTHD (dev, 0x0077, 0x018c, nv04_graph_mthd_bind_rop);
1097 NVOBJ_MTHD (dev, 0x0077, 0x0190, nv04_graph_mthd_bind_beta1);
1098 NVOBJ_MTHD (dev, 0x0077, 0x0194, nv04_graph_mthd_bind_beta4);
1099 NVOBJ_MTHD (dev, 0x0077, 0x0198, nv04_graph_mthd_bind_surf2d_swzsurf);
1100 NVOBJ_MTHD (dev, 0x0077, 0x0304, nv04_graph_mthd_set_operation);
1101
1102 /* null */
1103 NVOBJ_CLASS(dev, 0x0030, GR);
1104
1105 /* surf2d */
1106 NVOBJ_CLASS(dev, 0x0042, GR);
1107
1108 /* rop */
1109 NVOBJ_CLASS(dev, 0x0043, GR);
1110
1111 /* beta1 */
1112 NVOBJ_CLASS(dev, 0x0012, GR);
1113
1114 /* beta4 */
1115 NVOBJ_CLASS(dev, 0x0072, GR);
1116
1117 /* cliprect */
1118 NVOBJ_CLASS(dev, 0x0019, GR);
1119
1120 /* nv01 pattern */
1121 NVOBJ_CLASS(dev, 0x0018, GR);
1122
1123 /* nv04 pattern */
1124 NVOBJ_CLASS(dev, 0x0044, GR);
1125
1126 /* swzsurf */
1127 NVOBJ_CLASS(dev, 0x0052, GR);
1128
1129 /* surf3d */
1130 NVOBJ_CLASS(dev, 0x0053, GR);
1131 NVOBJ_MTHD (dev, 0x0053, 0x02f8, nv04_graph_mthd_surf3d_clip_h);
1132 NVOBJ_MTHD (dev, 0x0053, 0x02fc, nv04_graph_mthd_surf3d_clip_v);
1133
1134 /* nv03 tex_tri */
1135 NVOBJ_CLASS(dev, 0x0048, GR);
1136 NVOBJ_MTHD (dev, 0x0048, 0x0188, nv04_graph_mthd_bind_clip);
1137 NVOBJ_MTHD (dev, 0x0048, 0x018c, nv04_graph_mthd_bind_surf_color);
1138 NVOBJ_MTHD (dev, 0x0048, 0x0190, nv04_graph_mthd_bind_surf_zeta);
1139
1140 /* tex_tri */
1141 NVOBJ_CLASS(dev, 0x0054, GR);
1142
1143 /* multitex_tri */
1144 NVOBJ_CLASS(dev, 0x0055, GR);
1145
1146 /* nv01 chroma */
1147 NVOBJ_CLASS(dev, 0x0017, GR);
1148
1149 /* nv04 chroma */
1150 NVOBJ_CLASS(dev, 0x0057, GR);
1151
1152 /* surf_dst */
1153 NVOBJ_CLASS(dev, 0x0058, GR);
1154
1155 /* surf_src */
1156 NVOBJ_CLASS(dev, 0x0059, GR);
1157
1158 /* surf_color */
1159 NVOBJ_CLASS(dev, 0x005a, GR);
1160
1161 /* surf_zeta */
1162 NVOBJ_CLASS(dev, 0x005b, GR);
1163
1164 /* nv01 line */
1165 NVOBJ_CLASS(dev, 0x001c, GR);
1166 NVOBJ_MTHD (dev, 0x001c, 0x0184, nv04_graph_mthd_bind_clip);
1167 NVOBJ_MTHD (dev, 0x001c, 0x0188, nv04_graph_mthd_bind_nv01_patt);
1168 NVOBJ_MTHD (dev, 0x001c, 0x018c, nv04_graph_mthd_bind_rop);
1169 NVOBJ_MTHD (dev, 0x001c, 0x0190, nv04_graph_mthd_bind_beta1);
1170 NVOBJ_MTHD (dev, 0x001c, 0x0194, nv04_graph_mthd_bind_surf_dst);
1171 NVOBJ_MTHD (dev, 0x001c, 0x02fc, nv04_graph_mthd_set_operation);
1172
1173 /* nv04 line */
1174 NVOBJ_CLASS(dev, 0x005c, GR);
1175 NVOBJ_MTHD (dev, 0x005c, 0x0184, nv04_graph_mthd_bind_clip);
1176 NVOBJ_MTHD (dev, 0x005c, 0x0188, nv04_graph_mthd_bind_nv04_patt);
1177 NVOBJ_MTHD (dev, 0x005c, 0x018c, nv04_graph_mthd_bind_rop);
1178 NVOBJ_MTHD (dev, 0x005c, 0x0190, nv04_graph_mthd_bind_beta1);
1179 NVOBJ_MTHD (dev, 0x005c, 0x0194, nv04_graph_mthd_bind_beta4);
1180 NVOBJ_MTHD (dev, 0x005c, 0x0198, nv04_graph_mthd_bind_surf2d);
1181 NVOBJ_MTHD (dev, 0x005c, 0x02fc, nv04_graph_mthd_set_operation);
1182
1183 /* nv01 tri */
1184 NVOBJ_CLASS(dev, 0x001d, GR);
1185 NVOBJ_MTHD (dev, 0x001d, 0x0184, nv04_graph_mthd_bind_clip);
1186 NVOBJ_MTHD (dev, 0x001d, 0x0188, nv04_graph_mthd_bind_nv01_patt);
1187 NVOBJ_MTHD (dev, 0x001d, 0x018c, nv04_graph_mthd_bind_rop);
1188 NVOBJ_MTHD (dev, 0x001d, 0x0190, nv04_graph_mthd_bind_beta1);
1189 NVOBJ_MTHD (dev, 0x001d, 0x0194, nv04_graph_mthd_bind_surf_dst);
1190 NVOBJ_MTHD (dev, 0x001d, 0x02fc, nv04_graph_mthd_set_operation);
1191
1192 /* nv04 tri */
1193 NVOBJ_CLASS(dev, 0x005d, GR);
1194 NVOBJ_MTHD (dev, 0x005d, 0x0184, nv04_graph_mthd_bind_clip);
1195 NVOBJ_MTHD (dev, 0x005d, 0x0188, nv04_graph_mthd_bind_nv04_patt);
1196 NVOBJ_MTHD (dev, 0x005d, 0x018c, nv04_graph_mthd_bind_rop);
1197 NVOBJ_MTHD (dev, 0x005d, 0x0190, nv04_graph_mthd_bind_beta1);
1198 NVOBJ_MTHD (dev, 0x005d, 0x0194, nv04_graph_mthd_bind_beta4);
1199 NVOBJ_MTHD (dev, 0x005d, 0x0198, nv04_graph_mthd_bind_surf2d);
1200 NVOBJ_MTHD (dev, 0x005d, 0x02fc, nv04_graph_mthd_set_operation);
1201
1202 /* nv01 rect */
1203 NVOBJ_CLASS(dev, 0x001e, GR);
1204 NVOBJ_MTHD (dev, 0x001e, 0x0184, nv04_graph_mthd_bind_clip);
1205 NVOBJ_MTHD (dev, 0x001e, 0x0188, nv04_graph_mthd_bind_nv01_patt);
1206 NVOBJ_MTHD (dev, 0x001e, 0x018c, nv04_graph_mthd_bind_rop);
1207 NVOBJ_MTHD (dev, 0x001e, 0x0190, nv04_graph_mthd_bind_beta1);
1208 NVOBJ_MTHD (dev, 0x001e, 0x0194, nv04_graph_mthd_bind_surf_dst);
1209 NVOBJ_MTHD (dev, 0x001e, 0x02fc, nv04_graph_mthd_set_operation);
1210
1211 /* nv04 rect */
1212 NVOBJ_CLASS(dev, 0x005e, GR);
1213 NVOBJ_MTHD (dev, 0x005e, 0x0184, nv04_graph_mthd_bind_clip);
1214 NVOBJ_MTHD (dev, 0x005e, 0x0188, nv04_graph_mthd_bind_nv04_patt);
1215 NVOBJ_MTHD (dev, 0x005e, 0x018c, nv04_graph_mthd_bind_rop);
1216 NVOBJ_MTHD (dev, 0x005e, 0x0190, nv04_graph_mthd_bind_beta1);
1217 NVOBJ_MTHD (dev, 0x005e, 0x0194, nv04_graph_mthd_bind_beta4);
1218 NVOBJ_MTHD (dev, 0x005e, 0x0198, nv04_graph_mthd_bind_surf2d);
1219 NVOBJ_MTHD (dev, 0x005e, 0x02fc, nv04_graph_mthd_set_operation);
1220
1221 /* nvsw */
1222 NVOBJ_CLASS(dev, 0x506e, SW);
1223 NVOBJ_MTHD (dev, 0x506e, 0x0150, nv04_graph_mthd_set_ref);
1224 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
1225
1226 dev_priv->engine.graph.registered = true;
1227 return 0;
1063}; 1228};
1064 1229
1065static struct nouveau_pgraph_object_method nv04_graph_mthds_nv03_tex_tri[] = { 1230static struct nouveau_bitfield nv04_graph_intr[] = {
1066 { 0x0188, nv04_graph_mthd_bind_clip }, 1231 { NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
1067 { 0x018c, nv04_graph_mthd_bind_surf_color }, 1232 {}
1068 { 0x0190, nv04_graph_mthd_bind_surf_zeta },
1069 {},
1070}; 1233};
1071 1234
1072static struct nouveau_pgraph_object_method nv04_graph_mthds_surf3d[] = { 1235static struct nouveau_bitfield nv04_graph_nstatus[] =
1073 { 0x02f8, nv04_graph_mthd_surf3d_clip_h }, 1236{
1074 { 0x02fc, nv04_graph_mthd_surf3d_clip_v }, 1237 { NV04_PGRAPH_NSTATUS_STATE_IN_USE, "STATE_IN_USE" },
1075 {}, 1238 { NV04_PGRAPH_NSTATUS_INVALID_STATE, "INVALID_STATE" },
1239 { NV04_PGRAPH_NSTATUS_BAD_ARGUMENT, "BAD_ARGUMENT" },
1240 { NV04_PGRAPH_NSTATUS_PROTECTION_FAULT, "PROTECTION_FAULT" },
1241 {}
1076}; 1242};
1077 1243
1078struct nouveau_pgraph_object_class nv04_graph_grclass[] = { 1244struct nouveau_bitfield nv04_graph_nsource[] =
1079 { 0x0038, false, NULL }, /* dvd subpicture */ 1245{
1080 { 0x0039, false, NULL }, /* m2mf */ 1246 { NV03_PGRAPH_NSOURCE_NOTIFICATION, "NOTIFICATION" },
1081 { 0x004b, false, nv04_graph_mthds_nv03_gdirect }, /* nv03 gdirect */ 1247 { NV03_PGRAPH_NSOURCE_DATA_ERROR, "DATA_ERROR" },
1082 { 0x004a, false, nv04_graph_mthds_nv04_gdirect }, /* nv04 gdirect */ 1248 { NV03_PGRAPH_NSOURCE_PROTECTION_ERROR, "PROTECTION_ERROR" },
1083 { 0x001f, false, nv04_graph_mthds_nv01_imageblit }, /* nv01 imageblit */ 1249 { NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION, "RANGE_EXCEPTION" },
1084 { 0x005f, false, nv04_graph_mthds_nv04_imageblit_ifc }, /* nv04 imageblit */ 1250 { NV03_PGRAPH_NSOURCE_LIMIT_COLOR, "LIMIT_COLOR" },
1085 { 0x0060, false, nv04_graph_mthds_nv04_iifc }, /* nv04 iifc */ 1251 { NV03_PGRAPH_NSOURCE_LIMIT_ZETA, "LIMIT_ZETA" },
1086 { 0x0064, false, NULL }, /* nv05 iifc */ 1252 { NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD, "ILLEGAL_MTHD" },
1087 { 0x0021, false, nv04_graph_mthds_nv01_ifc }, /* nv01 ifc */ 1253 { NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION, "DMA_R_PROTECTION" },
1088 { 0x0061, false, nv04_graph_mthds_nv04_imageblit_ifc }, /* nv04 ifc */ 1254 { NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION, "DMA_W_PROTECTION" },
1089 { 0x0065, false, NULL }, /* nv05 ifc */ 1255 { NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION, "FORMAT_EXCEPTION" },
1090 { 0x0036, false, nv04_graph_mthds_nv03_sifc }, /* nv03 sifc */ 1256 { NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION, "PATCH_EXCEPTION" },
1091 { 0x0076, false, nv04_graph_mthds_nv04_sifc }, /* nv04 sifc */ 1257 { NV03_PGRAPH_NSOURCE_STATE_INVALID, "STATE_INVALID" },
1092 { 0x0066, false, NULL }, /* nv05 sifc */ 1258 { NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY, "DOUBLE_NOTIFY" },
1093 { 0x0037, false, nv04_graph_mthds_nv03_sifm }, /* nv03 sifm */ 1259 { NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE, "NOTIFY_IN_USE" },
1094 { 0x0077, false, nv04_graph_mthds_nv04_sifm }, /* nv04 sifm */ 1260 { NV03_PGRAPH_NSOURCE_METHOD_CNT, "METHOD_CNT" },
1095 { 0x0030, false, NULL }, /* null */ 1261 { NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION, "BFR_NOTIFICATION" },
1096 { 0x0042, false, NULL }, /* surf2d */ 1262 { NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
1097 { 0x0043, false, NULL }, /* rop */ 1263 { NV03_PGRAPH_NSOURCE_DMA_WIDTH_A, "DMA_WIDTH_A" },
1098 { 0x0012, false, NULL }, /* beta1 */ 1264 { NV03_PGRAPH_NSOURCE_DMA_WIDTH_B, "DMA_WIDTH_B" },
1099 { 0x0072, false, NULL }, /* beta4 */
1100 { 0x0019, false, NULL }, /* cliprect */
1101 { 0x0018, false, NULL }, /* nv01 pattern */
1102 { 0x0044, false, NULL }, /* nv04 pattern */
1103 { 0x0052, false, NULL }, /* swzsurf */
1104 { 0x0053, false, nv04_graph_mthds_surf3d }, /* surf3d */
1105 { 0x0048, false, nv04_graph_mthds_nv03_tex_tri }, /* nv03 tex_tri */
1106 { 0x0054, false, NULL }, /* tex_tri */
1107 { 0x0055, false, NULL }, /* multitex_tri */
1108 { 0x0017, false, NULL }, /* nv01 chroma */
1109 { 0x0057, false, NULL }, /* nv04 chroma */
1110 { 0x0058, false, NULL }, /* surf_dst */
1111 { 0x0059, false, NULL }, /* surf_src */
1112 { 0x005a, false, NULL }, /* surf_color */
1113 { 0x005b, false, NULL }, /* surf_zeta */
1114 { 0x001c, false, nv04_graph_mthds_nv01_shape }, /* nv01 line */
1115 { 0x005c, false, nv04_graph_mthds_nv04_shape }, /* nv04 line */
1116 { 0x001d, false, nv04_graph_mthds_nv01_shape }, /* nv01 tri */
1117 { 0x005d, false, nv04_graph_mthds_nv04_shape }, /* nv04 tri */
1118 { 0x001e, false, nv04_graph_mthds_nv01_shape }, /* nv01 rect */
1119 { 0x005e, false, nv04_graph_mthds_nv04_shape }, /* nv04 rect */
1120 { 0x506e, true, nv04_graph_mthds_sw },
1121 {} 1265 {}
1122}; 1266};
1123 1267
1268static void
1269nv04_graph_isr(struct drm_device *dev)
1270{
1271 u32 stat;
1272
1273 while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1274 u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
1275 u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
1276 u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
1277 u32 chid = (addr & 0x0f000000) >> 24;
1278 u32 subc = (addr & 0x0000e000) >> 13;
1279 u32 mthd = (addr & 0x00001ffc);
1280 u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
1281 u32 class = nv_rd32(dev, 0x400180 + subc * 4) & 0xff;
1282 u32 show = stat;
1283
1284 if (stat & NV_PGRAPH_INTR_NOTIFY) {
1285 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
1286 if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
1287 show &= ~NV_PGRAPH_INTR_NOTIFY;
1288 }
1289 }
1290
1291 if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
1292 nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
1293 stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1294 show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1295 nv04_graph_context_switch(dev);
1296 }
1297
1298 nv_wr32(dev, NV03_PGRAPH_INTR, stat);
1299 nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
1300
1301 if (show && nouveau_ratelimit()) {
1302 NV_INFO(dev, "PGRAPH -");
1303 nouveau_bitfield_print(nv04_graph_intr, show);
1304 printk(" nsource:");
1305 nouveau_bitfield_print(nv04_graph_nsource, nsource);
1306 printk(" nstatus:");
1307 nouveau_bitfield_print(nv04_graph_nstatus, nstatus);
1308 printk("\n");
1309 NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
1310 "mthd 0x%04x data 0x%08x\n",
1311 chid, subc, class, mthd, data);
1312 }
1313 }
1314}
diff --git a/drivers/gpu/drm/nouveau/nv04_instmem.c b/drivers/gpu/drm/nouveau/nv04_instmem.c
index 0b5ae297abde..b8e3edb5c063 100644
--- a/drivers/gpu/drm/nouveau/nv04_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv04_instmem.c
@@ -98,42 +98,66 @@ nv04_instmem_takedown(struct drm_device *dev)
98} 98}
99 99
100int 100int
101nv04_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, 101nv04_instmem_suspend(struct drm_device *dev)
102 uint32_t *sz)
103{ 102{
104 return 0; 103 return 0;
105} 104}
106 105
107void 106void
108nv04_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 107nv04_instmem_resume(struct drm_device *dev)
109{
110}
111
112int
113nv04_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
114{ 108{
115 return 0;
116} 109}
117 110
118int 111int
119nv04_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 112nv04_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
120{ 113{
114 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
115 struct drm_mm_node *ramin = NULL;
116
117 do {
118 if (drm_mm_pre_get(&dev_priv->ramin_heap))
119 return -ENOMEM;
120
121 spin_lock(&dev_priv->ramin_lock);
122 ramin = drm_mm_search_free(&dev_priv->ramin_heap, size, align, 0);
123 if (ramin == NULL) {
124 spin_unlock(&dev_priv->ramin_lock);
125 return -ENOMEM;
126 }
127
128 ramin = drm_mm_get_block_atomic(ramin, size, align);
129 spin_unlock(&dev_priv->ramin_lock);
130 } while (ramin == NULL);
131
132 gpuobj->node = ramin;
133 gpuobj->vinst = ramin->start;
121 return 0; 134 return 0;
122} 135}
123 136
124void 137void
125nv04_instmem_flush(struct drm_device *dev) 138nv04_instmem_put(struct nouveau_gpuobj *gpuobj)
126{ 139{
140 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
141
142 spin_lock(&dev_priv->ramin_lock);
143 drm_mm_put_block(gpuobj->node);
144 gpuobj->node = NULL;
145 spin_unlock(&dev_priv->ramin_lock);
127} 146}
128 147
129int 148int
130nv04_instmem_suspend(struct drm_device *dev) 149nv04_instmem_map(struct nouveau_gpuobj *gpuobj)
131{ 150{
151 gpuobj->pinst = gpuobj->vinst;
132 return 0; 152 return 0;
133} 153}
134 154
135void 155void
136nv04_instmem_resume(struct drm_device *dev) 156nv04_instmem_unmap(struct nouveau_gpuobj *gpuobj)
137{ 157{
138} 158}
139 159
160void
161nv04_instmem_flush(struct drm_device *dev)
162{
163}
diff --git a/drivers/gpu/drm/nouveau/nv10_fb.c b/drivers/gpu/drm/nouveau/nv10_fb.c
index cc5cda44e501..f78181a59b4a 100644
--- a/drivers/gpu/drm/nouveau/nv10_fb.c
+++ b/drivers/gpu/drm/nouveau/nv10_fb.c
@@ -3,23 +3,109 @@
3#include "nouveau_drv.h" 3#include "nouveau_drv.h"
4#include "nouveau_drm.h" 4#include "nouveau_drm.h"
5 5
6static struct drm_mm_node *
7nv20_fb_alloc_tag(struct drm_device *dev, uint32_t size)
8{
9 struct drm_nouveau_private *dev_priv = dev->dev_private;
10 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
11 struct drm_mm_node *mem;
12 int ret;
13
14 ret = drm_mm_pre_get(&pfb->tag_heap);
15 if (ret)
16 return NULL;
17
18 spin_lock(&dev_priv->tile.lock);
19 mem = drm_mm_search_free(&pfb->tag_heap, size, 0, 0);
20 if (mem)
21 mem = drm_mm_get_block_atomic(mem, size, 0);
22 spin_unlock(&dev_priv->tile.lock);
23
24 return mem;
25}
26
27static void
28nv20_fb_free_tag(struct drm_device *dev, struct drm_mm_node *mem)
29{
30 struct drm_nouveau_private *dev_priv = dev->dev_private;
31
32 spin_lock(&dev_priv->tile.lock);
33 drm_mm_put_block(mem);
34 spin_unlock(&dev_priv->tile.lock);
35}
36
37void
38nv10_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
39 uint32_t size, uint32_t pitch, uint32_t flags)
40{
41 struct drm_nouveau_private *dev_priv = dev->dev_private;
42 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
43 int bpp = (flags & NOUVEAU_GEM_TILE_32BPP ? 32 : 16);
44
45 tile->addr = addr;
46 tile->limit = max(1u, addr + size) - 1;
47 tile->pitch = pitch;
48
49 if (dev_priv->card_type == NV_20) {
50 if (flags & NOUVEAU_GEM_TILE_ZETA) {
51 /*
52 * Allocate some of the on-die tag memory,
53 * used to store Z compression meta-data (most
54 * likely just a bitmap determining if a given
55 * tile is compressed or not).
56 */
57 tile->tag_mem = nv20_fb_alloc_tag(dev, size / 256);
58
59 if (tile->tag_mem) {
60 /* Enable Z compression */
61 if (dev_priv->chipset >= 0x25)
62 tile->zcomp = tile->tag_mem->start |
63 (bpp == 16 ?
64 NV25_PFB_ZCOMP_MODE_16 :
65 NV25_PFB_ZCOMP_MODE_32);
66 else
67 tile->zcomp = tile->tag_mem->start |
68 NV20_PFB_ZCOMP_EN |
69 (bpp == 16 ? 0 :
70 NV20_PFB_ZCOMP_MODE_32);
71 }
72
73 tile->addr |= 3;
74 } else {
75 tile->addr |= 1;
76 }
77
78 } else {
79 tile->addr |= 1 << 31;
80 }
81}
82
6void 83void
7nv10_fb_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 84nv10_fb_free_tile_region(struct drm_device *dev, int i)
8 uint32_t size, uint32_t pitch)
9{ 85{
10 struct drm_nouveau_private *dev_priv = dev->dev_private; 86 struct drm_nouveau_private *dev_priv = dev->dev_private;
11 uint32_t limit = max(1u, addr + size) - 1; 87 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
12 88
13 if (pitch) { 89 if (tile->tag_mem) {
14 if (dev_priv->card_type >= NV_20) 90 nv20_fb_free_tag(dev, tile->tag_mem);
15 addr |= 1; 91 tile->tag_mem = NULL;
16 else
17 addr |= 1 << 31;
18 } 92 }
19 93
20 nv_wr32(dev, NV10_PFB_TLIMIT(i), limit); 94 tile->addr = tile->limit = tile->pitch = tile->zcomp = 0;
21 nv_wr32(dev, NV10_PFB_TSIZE(i), pitch); 95}
22 nv_wr32(dev, NV10_PFB_TILE(i), addr); 96
97void
98nv10_fb_set_tile_region(struct drm_device *dev, int i)
99{
100 struct drm_nouveau_private *dev_priv = dev->dev_private;
101 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
102
103 nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
104 nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
105 nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
106
107 if (dev_priv->card_type == NV_20)
108 nv_wr32(dev, NV20_PFB_ZCOMP(i), tile->zcomp);
23} 109}
24 110
25int 111int
@@ -31,9 +117,14 @@ nv10_fb_init(struct drm_device *dev)
31 117
32 pfb->num_tiles = NV10_PFB_TILE__SIZE; 118 pfb->num_tiles = NV10_PFB_TILE__SIZE;
33 119
120 if (dev_priv->card_type == NV_20)
121 drm_mm_init(&pfb->tag_heap, 0,
122 (dev_priv->chipset >= 0x25 ?
123 64 * 1024 : 32 * 1024));
124
34 /* Turn all the tiling regions off. */ 125 /* Turn all the tiling regions off. */
35 for (i = 0; i < pfb->num_tiles; i++) 126 for (i = 0; i < pfb->num_tiles; i++)
36 pfb->set_region_tiling(dev, i, 0, 0, 0); 127 pfb->set_tile_region(dev, i);
37 128
38 return 0; 129 return 0;
39} 130}
@@ -41,4 +132,13 @@ nv10_fb_init(struct drm_device *dev)
41void 132void
42nv10_fb_takedown(struct drm_device *dev) 133nv10_fb_takedown(struct drm_device *dev)
43{ 134{
135 struct drm_nouveau_private *dev_priv = dev->dev_private;
136 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
137 int i;
138
139 for (i = 0; i < pfb->num_tiles; i++)
140 pfb->free_tile_region(dev, i);
141
142 if (dev_priv->card_type == NV_20)
143 drm_mm_takedown(&pfb->tag_heap);
44} 144}
diff --git a/drivers/gpu/drm/nouveau/nv10_fifo.c b/drivers/gpu/drm/nouveau/nv10_fifo.c
index f1b03ad58fd5..d2ecbff4bee1 100644
--- a/drivers/gpu/drm/nouveau/nv10_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv10_fifo.c
@@ -53,6 +53,11 @@ nv10_fifo_create_context(struct nouveau_channel *chan)
53 if (ret) 53 if (ret)
54 return ret; 54 return ret;
55 55
56 chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
57 NV03_USER(chan->id), PAGE_SIZE);
58 if (!chan->user)
59 return -ENOMEM;
60
56 /* Fill entries that are seen filled in dumps of nvidia driver just 61 /* Fill entries that are seen filled in dumps of nvidia driver just
57 * after channel's is put into DMA mode 62 * after channel's is put into DMA mode
58 */ 63 */
@@ -73,17 +78,6 @@ nv10_fifo_create_context(struct nouveau_channel *chan)
73 return 0; 78 return 0;
74} 79}
75 80
76void
77nv10_fifo_destroy_context(struct nouveau_channel *chan)
78{
79 struct drm_device *dev = chan->dev;
80
81 nv_wr32(dev, NV04_PFIFO_MODE,
82 nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id));
83
84 nouveau_gpuobj_ref(NULL, &chan->ramfc);
85}
86
87static void 81static void
88nv10_fifo_do_load_context(struct drm_device *dev, int chid) 82nv10_fifo_do_load_context(struct drm_device *dev, int chid)
89{ 83{
@@ -219,6 +213,7 @@ nv10_fifo_init_ramxx(struct drm_device *dev)
219static void 213static void
220nv10_fifo_init_intr(struct drm_device *dev) 214nv10_fifo_init_intr(struct drm_device *dev)
221{ 215{
216 nouveau_irq_register(dev, 8, nv04_fifo_isr);
222 nv_wr32(dev, 0x002100, 0xffffffff); 217 nv_wr32(dev, 0x002100, 0xffffffff);
223 nv_wr32(dev, 0x002140, 0xffffffff); 218 nv_wr32(dev, 0x002140, 0xffffffff);
224} 219}
@@ -241,7 +236,7 @@ nv10_fifo_init(struct drm_device *dev)
241 pfifo->reassign(dev, true); 236 pfifo->reassign(dev, true);
242 237
243 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 238 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
244 if (dev_priv->fifos[i]) { 239 if (dev_priv->channels.ptr[i]) {
245 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE); 240 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
246 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i)); 241 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
247 } 242 }
diff --git a/drivers/gpu/drm/nouveau/nv10_graph.c b/drivers/gpu/drm/nouveau/nv10_graph.c
index 8e68c9731159..8c92edb7bbcd 100644
--- a/drivers/gpu/drm/nouveau/nv10_graph.c
+++ b/drivers/gpu/drm/nouveau/nv10_graph.c
@@ -26,6 +26,10 @@
26#include "drm.h" 26#include "drm.h"
27#include "nouveau_drm.h" 27#include "nouveau_drm.h"
28#include "nouveau_drv.h" 28#include "nouveau_drv.h"
29#include "nouveau_util.h"
30
31static int nv10_graph_register(struct drm_device *);
32static void nv10_graph_isr(struct drm_device *);
29 33
30#define NV10_FIFO_NUMBER 32 34#define NV10_FIFO_NUMBER 32
31 35
@@ -786,15 +790,13 @@ nv10_graph_unload_context(struct drm_device *dev)
786 return 0; 790 return 0;
787} 791}
788 792
789void 793static void
790nv10_graph_context_switch(struct drm_device *dev) 794nv10_graph_context_switch(struct drm_device *dev)
791{ 795{
792 struct drm_nouveau_private *dev_priv = dev->dev_private; 796 struct drm_nouveau_private *dev_priv = dev->dev_private;
793 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
794 struct nouveau_channel *chan = NULL; 797 struct nouveau_channel *chan = NULL;
795 int chid; 798 int chid;
796 799
797 pgraph->fifo_access(dev, false);
798 nouveau_wait_for_idle(dev); 800 nouveau_wait_for_idle(dev);
799 801
800 /* If previous context is valid, we need to save it */ 802 /* If previous context is valid, we need to save it */
@@ -802,11 +804,9 @@ nv10_graph_context_switch(struct drm_device *dev)
802 804
803 /* Load context for next channel */ 805 /* Load context for next channel */
804 chid = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f; 806 chid = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
805 chan = dev_priv->fifos[chid]; 807 chan = dev_priv->channels.ptr[chid];
806 if (chan && chan->pgraph_ctx) 808 if (chan && chan->pgraph_ctx)
807 nv10_graph_load_context(chan); 809 nv10_graph_load_context(chan);
808
809 pgraph->fifo_access(dev, true);
810} 810}
811 811
812#define NV_WRITE_CTX(reg, val) do { \ 812#define NV_WRITE_CTX(reg, val) do { \
@@ -833,7 +833,7 @@ nv10_graph_channel(struct drm_device *dev)
833 if (chid >= dev_priv->engine.fifo.channels) 833 if (chid >= dev_priv->engine.fifo.channels)
834 return NULL; 834 return NULL;
835 835
836 return dev_priv->fifos[chid]; 836 return dev_priv->channels.ptr[chid];
837} 837}
838 838
839int nv10_graph_create_context(struct nouveau_channel *chan) 839int nv10_graph_create_context(struct nouveau_channel *chan)
@@ -875,37 +875,54 @@ int nv10_graph_create_context(struct nouveau_channel *chan)
875 875
876void nv10_graph_destroy_context(struct nouveau_channel *chan) 876void nv10_graph_destroy_context(struct nouveau_channel *chan)
877{ 877{
878 struct drm_device *dev = chan->dev;
879 struct drm_nouveau_private *dev_priv = dev->dev_private;
880 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
878 struct graph_state *pgraph_ctx = chan->pgraph_ctx; 881 struct graph_state *pgraph_ctx = chan->pgraph_ctx;
882 unsigned long flags;
883
884 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
885 pgraph->fifo_access(dev, false);
886
887 /* Unload the context if it's the currently active one */
888 if (pgraph->channel(dev) == chan)
889 pgraph->unload_context(dev);
879 890
891 /* Free the context resources */
880 kfree(pgraph_ctx); 892 kfree(pgraph_ctx);
881 chan->pgraph_ctx = NULL; 893 chan->pgraph_ctx = NULL;
894
895 pgraph->fifo_access(dev, true);
896 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
882} 897}
883 898
884void 899void
885nv10_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 900nv10_graph_set_tile_region(struct drm_device *dev, int i)
886 uint32_t size, uint32_t pitch)
887{ 901{
888 uint32_t limit = max(1u, addr + size) - 1; 902 struct drm_nouveau_private *dev_priv = dev->dev_private;
889 903 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
890 if (pitch)
891 addr |= 1 << 31;
892 904
893 nv_wr32(dev, NV10_PGRAPH_TLIMIT(i), limit); 905 nv_wr32(dev, NV10_PGRAPH_TLIMIT(i), tile->limit);
894 nv_wr32(dev, NV10_PGRAPH_TSIZE(i), pitch); 906 nv_wr32(dev, NV10_PGRAPH_TSIZE(i), tile->pitch);
895 nv_wr32(dev, NV10_PGRAPH_TILE(i), addr); 907 nv_wr32(dev, NV10_PGRAPH_TILE(i), tile->addr);
896} 908}
897 909
898int nv10_graph_init(struct drm_device *dev) 910int nv10_graph_init(struct drm_device *dev)
899{ 911{
900 struct drm_nouveau_private *dev_priv = dev->dev_private; 912 struct drm_nouveau_private *dev_priv = dev->dev_private;
901 uint32_t tmp; 913 uint32_t tmp;
902 int i; 914 int ret, i;
903 915
904 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & 916 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
905 ~NV_PMC_ENABLE_PGRAPH); 917 ~NV_PMC_ENABLE_PGRAPH);
906 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) | 918 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
907 NV_PMC_ENABLE_PGRAPH); 919 NV_PMC_ENABLE_PGRAPH);
908 920
921 ret = nv10_graph_register(dev);
922 if (ret)
923 return ret;
924
925 nouveau_irq_register(dev, 12, nv10_graph_isr);
909 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF); 926 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF);
910 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF); 927 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
911 928
@@ -928,7 +945,7 @@ int nv10_graph_init(struct drm_device *dev)
928 945
929 /* Turn all the tiling regions off. */ 946 /* Turn all the tiling regions off. */
930 for (i = 0; i < NV10_PFB_TILE__SIZE; i++) 947 for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
931 nv10_graph_set_region_tiling(dev, i, 0, 0, 0); 948 nv10_graph_set_tile_region(dev, i);
932 949
933 nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000); 950 nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000);
934 nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000); 951 nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000);
@@ -948,17 +965,17 @@ int nv10_graph_init(struct drm_device *dev)
948 965
949void nv10_graph_takedown(struct drm_device *dev) 966void nv10_graph_takedown(struct drm_device *dev)
950{ 967{
968 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
969 nouveau_irq_unregister(dev, 12);
951} 970}
952 971
953static int 972static int
954nv17_graph_mthd_lma_window(struct nouveau_channel *chan, int grclass, 973nv17_graph_mthd_lma_window(struct nouveau_channel *chan,
955 int mthd, uint32_t data) 974 u32 class, u32 mthd, u32 data)
956{ 975{
957 struct drm_device *dev = chan->dev; 976 struct drm_device *dev = chan->dev;
958 struct graph_state *ctx = chan->pgraph_ctx; 977 struct graph_state *ctx = chan->pgraph_ctx;
959 struct pipe_state *pipe = &ctx->pipe_state; 978 struct pipe_state *pipe = &ctx->pipe_state;
960 struct drm_nouveau_private *dev_priv = dev->dev_private;
961 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
962 uint32_t pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3]; 979 uint32_t pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3];
963 uint32_t xfmode0, xfmode1; 980 uint32_t xfmode0, xfmode1;
964 int i; 981 int i;
@@ -1025,18 +1042,14 @@ nv17_graph_mthd_lma_window(struct nouveau_channel *chan, int grclass,
1025 1042
1026 nouveau_wait_for_idle(dev); 1043 nouveau_wait_for_idle(dev);
1027 1044
1028 pgraph->fifo_access(dev, true);
1029
1030 return 0; 1045 return 0;
1031} 1046}
1032 1047
1033static int 1048static int
1034nv17_graph_mthd_lma_enable(struct nouveau_channel *chan, int grclass, 1049nv17_graph_mthd_lma_enable(struct nouveau_channel *chan,
1035 int mthd, uint32_t data) 1050 u32 class, u32 mthd, u32 data)
1036{ 1051{
1037 struct drm_device *dev = chan->dev; 1052 struct drm_device *dev = chan->dev;
1038 struct drm_nouveau_private *dev_priv = dev->dev_private;
1039 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
1040 1053
1041 nouveau_wait_for_idle(dev); 1054 nouveau_wait_for_idle(dev);
1042 1055
@@ -1045,40 +1058,118 @@ nv17_graph_mthd_lma_enable(struct nouveau_channel *chan, int grclass,
1045 nv_wr32(dev, 0x004006b0, 1058 nv_wr32(dev, 0x004006b0,
1046 nv_rd32(dev, 0x004006b0) | 0x8 << 24); 1059 nv_rd32(dev, 0x004006b0) | 0x8 << 24);
1047 1060
1048 pgraph->fifo_access(dev, true); 1061 return 0;
1062}
1063
1064static int
1065nv10_graph_register(struct drm_device *dev)
1066{
1067 struct drm_nouveau_private *dev_priv = dev->dev_private;
1068
1069 if (dev_priv->engine.graph.registered)
1070 return 0;
1071
1072 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
1073 NVOBJ_CLASS(dev, 0x0030, GR); /* null */
1074 NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
1075 NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
1076 NVOBJ_CLASS(dev, 0x005f, GR); /* imageblit */
1077 NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
1078 NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
1079 NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
1080 NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
1081 NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
1082 NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
1083 NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
1084 NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
1085 NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
1086 NVOBJ_CLASS(dev, 0x0052, GR); /* swzsurf */
1087 NVOBJ_CLASS(dev, 0x0093, GR); /* surf3d */
1088 NVOBJ_CLASS(dev, 0x0094, GR); /* tex_tri */
1089 NVOBJ_CLASS(dev, 0x0095, GR); /* multitex_tri */
1090
1091 /* celcius */
1092 if (dev_priv->chipset <= 0x10) {
1093 NVOBJ_CLASS(dev, 0x0056, GR);
1094 } else
1095 if (dev_priv->chipset < 0x17 || dev_priv->chipset == 0x1a) {
1096 NVOBJ_CLASS(dev, 0x0096, GR);
1097 } else {
1098 NVOBJ_CLASS(dev, 0x0099, GR);
1099 NVOBJ_MTHD (dev, 0x0099, 0x1638, nv17_graph_mthd_lma_window);
1100 NVOBJ_MTHD (dev, 0x0099, 0x163c, nv17_graph_mthd_lma_window);
1101 NVOBJ_MTHD (dev, 0x0099, 0x1640, nv17_graph_mthd_lma_window);
1102 NVOBJ_MTHD (dev, 0x0099, 0x1644, nv17_graph_mthd_lma_window);
1103 NVOBJ_MTHD (dev, 0x0099, 0x1658, nv17_graph_mthd_lma_enable);
1104 }
1049 1105
1106 /* nvsw */
1107 NVOBJ_CLASS(dev, 0x506e, SW);
1108 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
1109
1110 dev_priv->engine.graph.registered = true;
1050 return 0; 1111 return 0;
1051} 1112}
1052 1113
1053static struct nouveau_pgraph_object_method nv17_graph_celsius_mthds[] = { 1114struct nouveau_bitfield nv10_graph_intr[] = {
1054 { 0x1638, nv17_graph_mthd_lma_window }, 1115 { NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
1055 { 0x163c, nv17_graph_mthd_lma_window }, 1116 { NV_PGRAPH_INTR_ERROR, "ERROR" },
1056 { 0x1640, nv17_graph_mthd_lma_window },
1057 { 0x1644, nv17_graph_mthd_lma_window },
1058 { 0x1658, nv17_graph_mthd_lma_enable },
1059 {} 1117 {}
1060}; 1118};
1061 1119
1062struct nouveau_pgraph_object_class nv10_graph_grclass[] = { 1120struct nouveau_bitfield nv10_graph_nstatus[] =
1063 { 0x0030, false, NULL }, /* null */ 1121{
1064 { 0x0039, false, NULL }, /* m2mf */ 1122 { NV10_PGRAPH_NSTATUS_STATE_IN_USE, "STATE_IN_USE" },
1065 { 0x004a, false, NULL }, /* gdirect */ 1123 { NV10_PGRAPH_NSTATUS_INVALID_STATE, "INVALID_STATE" },
1066 { 0x005f, false, NULL }, /* imageblit */ 1124 { NV10_PGRAPH_NSTATUS_BAD_ARGUMENT, "BAD_ARGUMENT" },
1067 { 0x009f, false, NULL }, /* imageblit (nv12) */ 1125 { NV10_PGRAPH_NSTATUS_PROTECTION_FAULT, "PROTECTION_FAULT" },
1068 { 0x008a, false, NULL }, /* ifc */
1069 { 0x0089, false, NULL }, /* sifm */
1070 { 0x0062, false, NULL }, /* surf2d */
1071 { 0x0043, false, NULL }, /* rop */
1072 { 0x0012, false, NULL }, /* beta1 */
1073 { 0x0072, false, NULL }, /* beta4 */
1074 { 0x0019, false, NULL }, /* cliprect */
1075 { 0x0044, false, NULL }, /* pattern */
1076 { 0x0052, false, NULL }, /* swzsurf */
1077 { 0x0093, false, NULL }, /* surf3d */
1078 { 0x0094, false, NULL }, /* tex_tri */
1079 { 0x0095, false, NULL }, /* multitex_tri */
1080 { 0x0056, false, NULL }, /* celcius (nv10) */
1081 { 0x0096, false, NULL }, /* celcius (nv11) */
1082 { 0x0099, false, nv17_graph_celsius_mthds }, /* celcius (nv17) */
1083 {} 1126 {}
1084}; 1127};
1128
1129static void
1130nv10_graph_isr(struct drm_device *dev)
1131{
1132 u32 stat;
1133
1134 while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
1135 u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
1136 u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
1137 u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
1138 u32 chid = (addr & 0x01f00000) >> 20;
1139 u32 subc = (addr & 0x00070000) >> 16;
1140 u32 mthd = (addr & 0x00001ffc);
1141 u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
1142 u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
1143 u32 show = stat;
1144
1145 if (stat & NV_PGRAPH_INTR_ERROR) {
1146 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
1147 if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
1148 show &= ~NV_PGRAPH_INTR_ERROR;
1149 }
1150 }
1151
1152 if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
1153 nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
1154 stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1155 show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
1156 nv10_graph_context_switch(dev);
1157 }
1158
1159 nv_wr32(dev, NV03_PGRAPH_INTR, stat);
1160 nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
1161
1162 if (show && nouveau_ratelimit()) {
1163 NV_INFO(dev, "PGRAPH -");
1164 nouveau_bitfield_print(nv10_graph_intr, show);
1165 printk(" nsource:");
1166 nouveau_bitfield_print(nv04_graph_nsource, nsource);
1167 printk(" nstatus:");
1168 nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
1169 printk("\n");
1170 NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
1171 "mthd 0x%04x data 0x%08x\n",
1172 chid, subc, class, mthd, data);
1173 }
1174 }
1175}
diff --git a/drivers/gpu/drm/nouveau/nv20_graph.c b/drivers/gpu/drm/nouveau/nv20_graph.c
index 12ab9cd56eca..8464b76798d5 100644
--- a/drivers/gpu/drm/nouveau/nv20_graph.c
+++ b/drivers/gpu/drm/nouveau/nv20_graph.c
@@ -32,6 +32,10 @@
32#define NV34_GRCTX_SIZE (18140) 32#define NV34_GRCTX_SIZE (18140)
33#define NV35_36_GRCTX_SIZE (22396) 33#define NV35_36_GRCTX_SIZE (22396)
34 34
35static int nv20_graph_register(struct drm_device *);
36static int nv30_graph_register(struct drm_device *);
37static void nv20_graph_isr(struct drm_device *);
38
35static void 39static void
36nv20_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx) 40nv20_graph_context_init(struct drm_device *dev, struct nouveau_gpuobj *ctx)
37{ 41{
@@ -425,9 +429,21 @@ nv20_graph_destroy_context(struct nouveau_channel *chan)
425 struct drm_device *dev = chan->dev; 429 struct drm_device *dev = chan->dev;
426 struct drm_nouveau_private *dev_priv = dev->dev_private; 430 struct drm_nouveau_private *dev_priv = dev->dev_private;
427 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph; 431 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
432 unsigned long flags;
428 433
429 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx); 434 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
435 pgraph->fifo_access(dev, false);
436
437 /* Unload the context if it's the currently active one */
438 if (pgraph->channel(dev) == chan)
439 pgraph->unload_context(dev);
440
441 pgraph->fifo_access(dev, true);
442 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
443
444 /* Free the context resources */
430 nv_wo32(pgraph->ctx_table, chan->id * 4, 0); 445 nv_wo32(pgraph->ctx_table, chan->id * 4, 0);
446 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx);
431} 447}
432 448
433int 449int
@@ -496,24 +512,27 @@ nv20_graph_rdi(struct drm_device *dev)
496} 512}
497 513
498void 514void
499nv20_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 515nv20_graph_set_tile_region(struct drm_device *dev, int i)
500 uint32_t size, uint32_t pitch)
501{ 516{
502 uint32_t limit = max(1u, addr + size) - 1; 517 struct drm_nouveau_private *dev_priv = dev->dev_private;
503 518 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
504 if (pitch)
505 addr |= 1;
506 519
507 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit); 520 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
508 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch); 521 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
509 nv_wr32(dev, NV20_PGRAPH_TILE(i), addr); 522 nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
510 523
511 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0030 + 4 * i); 524 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0030 + 4 * i);
512 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, limit); 525 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->limit);
513 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0050 + 4 * i); 526 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0050 + 4 * i);
514 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, pitch); 527 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->pitch);
515 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0010 + 4 * i); 528 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0010 + 4 * i);
516 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, addr); 529 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->addr);
530
531 if (dev_priv->card_type == NV_20) {
532 nv_wr32(dev, NV20_PGRAPH_ZCOMP(i), tile->zcomp);
533 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00ea0090 + 4 * i);
534 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->zcomp);
535 }
517} 536}
518 537
519int 538int
@@ -560,6 +579,13 @@ nv20_graph_init(struct drm_device *dev)
560 579
561 nv20_graph_rdi(dev); 580 nv20_graph_rdi(dev);
562 581
582 ret = nv20_graph_register(dev);
583 if (ret) {
584 nouveau_gpuobj_ref(NULL, &pgraph->ctx_table);
585 return ret;
586 }
587
588 nouveau_irq_register(dev, 12, nv20_graph_isr);
563 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF); 589 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF);
564 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF); 590 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
565 591
@@ -571,16 +597,17 @@ nv20_graph_init(struct drm_device *dev)
571 nv_wr32(dev, 0x40009C , 0x00000040); 597 nv_wr32(dev, 0x40009C , 0x00000040);
572 598
573 if (dev_priv->chipset >= 0x25) { 599 if (dev_priv->chipset >= 0x25) {
574 nv_wr32(dev, 0x400890, 0x00080000); 600 nv_wr32(dev, 0x400890, 0x00a8cfff);
575 nv_wr32(dev, 0x400610, 0x304B1FB6); 601 nv_wr32(dev, 0x400610, 0x304B1FB6);
576 nv_wr32(dev, 0x400B80, 0x18B82880); 602 nv_wr32(dev, 0x400B80, 0x1cbd3883);
577 nv_wr32(dev, 0x400B84, 0x44000000); 603 nv_wr32(dev, 0x400B84, 0x44000000);
578 nv_wr32(dev, 0x400098, 0x40000080); 604 nv_wr32(dev, 0x400098, 0x40000080);
579 nv_wr32(dev, 0x400B88, 0x000000ff); 605 nv_wr32(dev, 0x400B88, 0x000000ff);
606
580 } else { 607 } else {
581 nv_wr32(dev, 0x400880, 0x00080000); /* 0x0008c7df */ 608 nv_wr32(dev, 0x400880, 0x0008c7df);
582 nv_wr32(dev, 0x400094, 0x00000005); 609 nv_wr32(dev, 0x400094, 0x00000005);
583 nv_wr32(dev, 0x400B80, 0x45CAA208); /* 0x45eae20e */ 610 nv_wr32(dev, 0x400B80, 0x45eae20e);
584 nv_wr32(dev, 0x400B84, 0x24000000); 611 nv_wr32(dev, 0x400B84, 0x24000000);
585 nv_wr32(dev, 0x400098, 0x00000040); 612 nv_wr32(dev, 0x400098, 0x00000040);
586 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E00038); 613 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E00038);
@@ -591,14 +618,8 @@ nv20_graph_init(struct drm_device *dev)
591 618
592 /* Turn all the tiling regions off. */ 619 /* Turn all the tiling regions off. */
593 for (i = 0; i < NV10_PFB_TILE__SIZE; i++) 620 for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
594 nv20_graph_set_region_tiling(dev, i, 0, 0, 0); 621 nv20_graph_set_tile_region(dev, i);
595 622
596 for (i = 0; i < 8; i++) {
597 nv_wr32(dev, 0x400980 + i * 4, nv_rd32(dev, 0x100300 + i * 4));
598 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0090 + i * 4);
599 nv_wr32(dev, NV10_PGRAPH_RDI_DATA,
600 nv_rd32(dev, 0x100300 + i * 4));
601 }
602 nv_wr32(dev, 0x4009a0, nv_rd32(dev, 0x100324)); 623 nv_wr32(dev, 0x4009a0, nv_rd32(dev, 0x100324));
603 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA000C); 624 nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA000C);
604 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, nv_rd32(dev, 0x100324)); 625 nv_wr32(dev, NV10_PGRAPH_RDI_DATA, nv_rd32(dev, 0x100324));
@@ -642,6 +663,9 @@ nv20_graph_takedown(struct drm_device *dev)
642 struct drm_nouveau_private *dev_priv = dev->dev_private; 663 struct drm_nouveau_private *dev_priv = dev->dev_private;
643 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph; 664 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
644 665
666 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
667 nouveau_irq_unregister(dev, 12);
668
645 nouveau_gpuobj_ref(NULL, &pgraph->ctx_table); 669 nouveau_gpuobj_ref(NULL, &pgraph->ctx_table);
646} 670}
647 671
@@ -684,9 +708,16 @@ nv30_graph_init(struct drm_device *dev)
684 return ret; 708 return ret;
685 } 709 }
686 710
711 ret = nv30_graph_register(dev);
712 if (ret) {
713 nouveau_gpuobj_ref(NULL, &pgraph->ctx_table);
714 return ret;
715 }
716
687 nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE, 717 nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE,
688 pgraph->ctx_table->pinst >> 4); 718 pgraph->ctx_table->pinst >> 4);
689 719
720 nouveau_irq_register(dev, 12, nv20_graph_isr);
690 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF); 721 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF);
691 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF); 722 nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
692 723
@@ -724,7 +755,7 @@ nv30_graph_init(struct drm_device *dev)
724 755
725 /* Turn all the tiling regions off. */ 756 /* Turn all the tiling regions off. */
726 for (i = 0; i < NV10_PFB_TILE__SIZE; i++) 757 for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
727 nv20_graph_set_region_tiling(dev, i, 0, 0, 0); 758 nv20_graph_set_tile_region(dev, i);
728 759
729 nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100); 760 nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
730 nv_wr32(dev, NV10_PGRAPH_STATE , 0xFFFFFFFF); 761 nv_wr32(dev, NV10_PGRAPH_STATE , 0xFFFFFFFF);
@@ -744,46 +775,125 @@ nv30_graph_init(struct drm_device *dev)
744 return 0; 775 return 0;
745} 776}
746 777
747struct nouveau_pgraph_object_class nv20_graph_grclass[] = { 778static int
748 { 0x0030, false, NULL }, /* null */ 779nv20_graph_register(struct drm_device *dev)
749 { 0x0039, false, NULL }, /* m2mf */ 780{
750 { 0x004a, false, NULL }, /* gdirect */ 781 struct drm_nouveau_private *dev_priv = dev->dev_private;
751 { 0x009f, false, NULL }, /* imageblit (nv12) */ 782
752 { 0x008a, false, NULL }, /* ifc */ 783 if (dev_priv->engine.graph.registered)
753 { 0x0089, false, NULL }, /* sifm */ 784 return 0;
754 { 0x0062, false, NULL }, /* surf2d */ 785
755 { 0x0043, false, NULL }, /* rop */ 786 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
756 { 0x0012, false, NULL }, /* beta1 */ 787 NVOBJ_CLASS(dev, 0x0030, GR); /* null */
757 { 0x0072, false, NULL }, /* beta4 */ 788 NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
758 { 0x0019, false, NULL }, /* cliprect */ 789 NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
759 { 0x0044, false, NULL }, /* pattern */ 790 NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
760 { 0x009e, false, NULL }, /* swzsurf */ 791 NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
761 { 0x0096, false, NULL }, /* celcius */ 792 NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
762 { 0x0097, false, NULL }, /* kelvin (nv20) */ 793 NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
763 { 0x0597, false, NULL }, /* kelvin (nv25) */ 794 NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
764 {} 795 NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
765}; 796 NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
766 797 NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
767struct nouveau_pgraph_object_class nv30_graph_grclass[] = { 798 NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
768 { 0x0030, false, NULL }, /* null */ 799 NVOBJ_CLASS(dev, 0x009e, GR); /* swzsurf */
769 { 0x0039, false, NULL }, /* m2mf */ 800 NVOBJ_CLASS(dev, 0x0096, GR); /* celcius */
770 { 0x004a, false, NULL }, /* gdirect */ 801
771 { 0x009f, false, NULL }, /* imageblit (nv12) */ 802 /* kelvin */
772 { 0x008a, false, NULL }, /* ifc */ 803 if (dev_priv->chipset < 0x25)
773 { 0x038a, false, NULL }, /* ifc (nv30) */ 804 NVOBJ_CLASS(dev, 0x0097, GR);
774 { 0x0089, false, NULL }, /* sifm */ 805 else
775 { 0x0389, false, NULL }, /* sifm (nv30) */ 806 NVOBJ_CLASS(dev, 0x0597, GR);
776 { 0x0062, false, NULL }, /* surf2d */ 807
777 { 0x0362, false, NULL }, /* surf2d (nv30) */ 808 /* nvsw */
778 { 0x0043, false, NULL }, /* rop */ 809 NVOBJ_CLASS(dev, 0x506e, SW);
779 { 0x0012, false, NULL }, /* beta1 */ 810 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
780 { 0x0072, false, NULL }, /* beta4 */ 811
781 { 0x0019, false, NULL }, /* cliprect */ 812 dev_priv->engine.graph.registered = true;
782 { 0x0044, false, NULL }, /* pattern */ 813 return 0;
783 { 0x039e, false, NULL }, /* swzsurf */ 814}
784 { 0x0397, false, NULL }, /* rankine (nv30) */ 815
785 { 0x0497, false, NULL }, /* rankine (nv35) */ 816static int
786 { 0x0697, false, NULL }, /* rankine (nv34) */ 817nv30_graph_register(struct drm_device *dev)
787 {} 818{
788}; 819 struct drm_nouveau_private *dev_priv = dev->dev_private;
789 820
821 if (dev_priv->engine.graph.registered)
822 return 0;
823
824 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
825 NVOBJ_CLASS(dev, 0x0030, GR); /* null */
826 NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
827 NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
828 NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
829 NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
830 NVOBJ_CLASS(dev, 0x038a, GR); /* ifc (nv30) */
831 NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
832 NVOBJ_CLASS(dev, 0x0389, GR); /* sifm (nv30) */
833 NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
834 NVOBJ_CLASS(dev, 0x0362, GR); /* surf2d (nv30) */
835 NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
836 NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
837 NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
838 NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
839 NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
840 NVOBJ_CLASS(dev, 0x039e, GR); /* swzsurf */
841
842 /* rankine */
843 if (0x00000003 & (1 << (dev_priv->chipset & 0x0f)))
844 NVOBJ_CLASS(dev, 0x0397, GR);
845 else
846 if (0x00000010 & (1 << (dev_priv->chipset & 0x0f)))
847 NVOBJ_CLASS(dev, 0x0697, GR);
848 else
849 if (0x000001e0 & (1 << (dev_priv->chipset & 0x0f)))
850 NVOBJ_CLASS(dev, 0x0497, GR);
851
852 /* nvsw */
853 NVOBJ_CLASS(dev, 0x506e, SW);
854 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
855
856 dev_priv->engine.graph.registered = true;
857 return 0;
858}
859
860static void
861nv20_graph_isr(struct drm_device *dev)
862{
863 u32 stat;
864
865 while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
866 u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
867 u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
868 u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
869 u32 chid = (addr & 0x01f00000) >> 20;
870 u32 subc = (addr & 0x00070000) >> 16;
871 u32 mthd = (addr & 0x00001ffc);
872 u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
873 u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
874 u32 show = stat;
875
876 if (stat & NV_PGRAPH_INTR_ERROR) {
877 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
878 if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
879 show &= ~NV_PGRAPH_INTR_ERROR;
880 }
881 }
882
883 nv_wr32(dev, NV03_PGRAPH_INTR, stat);
884 nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
885
886 if (show && nouveau_ratelimit()) {
887 NV_INFO(dev, "PGRAPH -");
888 nouveau_bitfield_print(nv10_graph_intr, show);
889 printk(" nsource:");
890 nouveau_bitfield_print(nv04_graph_nsource, nsource);
891 printk(" nstatus:");
892 nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
893 printk("\n");
894 NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
895 "mthd 0x%04x data 0x%08x\n",
896 chid, subc, class, mthd, data);
897 }
898 }
899}
diff --git a/drivers/gpu/drm/nouveau/nv30_fb.c b/drivers/gpu/drm/nouveau/nv30_fb.c
index 4a3f2f095128..e0135f0e2144 100644
--- a/drivers/gpu/drm/nouveau/nv30_fb.c
+++ b/drivers/gpu/drm/nouveau/nv30_fb.c
@@ -29,6 +29,27 @@
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_drm.h" 30#include "nouveau_drm.h"
31 31
32void
33nv30_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
34 uint32_t size, uint32_t pitch, uint32_t flags)
35{
36 struct drm_nouveau_private *dev_priv = dev->dev_private;
37 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
38
39 tile->addr = addr | 1;
40 tile->limit = max(1u, addr + size) - 1;
41 tile->pitch = pitch;
42}
43
44void
45nv30_fb_free_tile_region(struct drm_device *dev, int i)
46{
47 struct drm_nouveau_private *dev_priv = dev->dev_private;
48 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
49
50 tile->addr = tile->limit = tile->pitch = 0;
51}
52
32static int 53static int
33calc_bias(struct drm_device *dev, int k, int i, int j) 54calc_bias(struct drm_device *dev, int k, int i, int j)
34{ 55{
@@ -65,7 +86,7 @@ nv30_fb_init(struct drm_device *dev)
65 86
66 /* Turn all the tiling regions off. */ 87 /* Turn all the tiling regions off. */
67 for (i = 0; i < pfb->num_tiles; i++) 88 for (i = 0; i < pfb->num_tiles; i++)
68 pfb->set_region_tiling(dev, i, 0, 0, 0); 89 pfb->set_tile_region(dev, i);
69 90
70 /* Init the memory timing regs at 0x10037c/0x1003ac */ 91 /* Init the memory timing regs at 0x10037c/0x1003ac */
71 if (dev_priv->chipset == 0x30 || 92 if (dev_priv->chipset == 0x30 ||
diff --git a/drivers/gpu/drm/nouveau/nv40_fb.c b/drivers/gpu/drm/nouveau/nv40_fb.c
index 3cd07d8d5bd7..f3d9c0505f7b 100644
--- a/drivers/gpu/drm/nouveau/nv40_fb.c
+++ b/drivers/gpu/drm/nouveau/nv40_fb.c
@@ -4,26 +4,22 @@
4#include "nouveau_drm.h" 4#include "nouveau_drm.h"
5 5
6void 6void
7nv40_fb_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 7nv40_fb_set_tile_region(struct drm_device *dev, int i)
8 uint32_t size, uint32_t pitch)
9{ 8{
10 struct drm_nouveau_private *dev_priv = dev->dev_private; 9 struct drm_nouveau_private *dev_priv = dev->dev_private;
11 uint32_t limit = max(1u, addr + size) - 1; 10 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
12
13 if (pitch)
14 addr |= 1;
15 11
16 switch (dev_priv->chipset) { 12 switch (dev_priv->chipset) {
17 case 0x40: 13 case 0x40:
18 nv_wr32(dev, NV10_PFB_TLIMIT(i), limit); 14 nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
19 nv_wr32(dev, NV10_PFB_TSIZE(i), pitch); 15 nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
20 nv_wr32(dev, NV10_PFB_TILE(i), addr); 16 nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
21 break; 17 break;
22 18
23 default: 19 default:
24 nv_wr32(dev, NV40_PFB_TLIMIT(i), limit); 20 nv_wr32(dev, NV40_PFB_TLIMIT(i), tile->limit);
25 nv_wr32(dev, NV40_PFB_TSIZE(i), pitch); 21 nv_wr32(dev, NV40_PFB_TSIZE(i), tile->pitch);
26 nv_wr32(dev, NV40_PFB_TILE(i), addr); 22 nv_wr32(dev, NV40_PFB_TILE(i), tile->addr);
27 break; 23 break;
28 } 24 }
29} 25}
@@ -64,7 +60,7 @@ nv40_fb_init(struct drm_device *dev)
64 60
65 /* Turn all the tiling regions off. */ 61 /* Turn all the tiling regions off. */
66 for (i = 0; i < pfb->num_tiles; i++) 62 for (i = 0; i < pfb->num_tiles; i++)
67 pfb->set_region_tiling(dev, i, 0, 0, 0); 63 pfb->set_tile_region(dev, i);
68 64
69 return 0; 65 return 0;
70} 66}
diff --git a/drivers/gpu/drm/nouveau/nv40_fifo.c b/drivers/gpu/drm/nouveau/nv40_fifo.c
index d337b8b28cdd..49b9a35a9cd6 100644
--- a/drivers/gpu/drm/nouveau/nv40_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv40_fifo.c
@@ -47,6 +47,11 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
47 if (ret) 47 if (ret)
48 return ret; 48 return ret;
49 49
50 chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
51 NV40_USER(chan->id), PAGE_SIZE);
52 if (!chan->user)
53 return -ENOMEM;
54
50 spin_lock_irqsave(&dev_priv->context_switch_lock, flags); 55 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
51 56
52 nv_wi32(dev, fc + 0, chan->pushbuf_base); 57 nv_wi32(dev, fc + 0, chan->pushbuf_base);
@@ -59,7 +64,6 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
59 NV_PFIFO_CACHE1_BIG_ENDIAN | 64 NV_PFIFO_CACHE1_BIG_ENDIAN |
60#endif 65#endif
61 0x30000000 /* no idea.. */); 66 0x30000000 /* no idea.. */);
62 nv_wi32(dev, fc + 56, chan->ramin_grctx->pinst >> 4);
63 nv_wi32(dev, fc + 60, 0x0001FFFF); 67 nv_wi32(dev, fc + 60, 0x0001FFFF);
64 68
65 /* enable the fifo dma operation */ 69 /* enable the fifo dma operation */
@@ -70,17 +74,6 @@ nv40_fifo_create_context(struct nouveau_channel *chan)
70 return 0; 74 return 0;
71} 75}
72 76
73void
74nv40_fifo_destroy_context(struct nouveau_channel *chan)
75{
76 struct drm_device *dev = chan->dev;
77
78 nv_wr32(dev, NV04_PFIFO_MODE,
79 nv_rd32(dev, NV04_PFIFO_MODE) & ~(1 << chan->id));
80
81 nouveau_gpuobj_ref(NULL, &chan->ramfc);
82}
83
84static void 77static void
85nv40_fifo_do_load_context(struct drm_device *dev, int chid) 78nv40_fifo_do_load_context(struct drm_device *dev, int chid)
86{ 79{
@@ -279,6 +272,7 @@ nv40_fifo_init_ramxx(struct drm_device *dev)
279static void 272static void
280nv40_fifo_init_intr(struct drm_device *dev) 273nv40_fifo_init_intr(struct drm_device *dev)
281{ 274{
275 nouveau_irq_register(dev, 8, nv04_fifo_isr);
282 nv_wr32(dev, 0x002100, 0xffffffff); 276 nv_wr32(dev, 0x002100, 0xffffffff);
283 nv_wr32(dev, 0x002140, 0xffffffff); 277 nv_wr32(dev, 0x002140, 0xffffffff);
284} 278}
@@ -301,7 +295,7 @@ nv40_fifo_init(struct drm_device *dev)
301 pfifo->reassign(dev, true); 295 pfifo->reassign(dev, true);
302 296
303 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 297 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
304 if (dev_priv->fifos[i]) { 298 if (dev_priv->channels.ptr[i]) {
305 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE); 299 uint32_t mode = nv_rd32(dev, NV04_PFIFO_MODE);
306 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i)); 300 nv_wr32(dev, NV04_PFIFO_MODE, mode | (1 << i));
307 } 301 }
diff --git a/drivers/gpu/drm/nouveau/nv40_graph.c b/drivers/gpu/drm/nouveau/nv40_graph.c
index 7ee1b91569b8..19ef92a0375a 100644
--- a/drivers/gpu/drm/nouveau/nv40_graph.c
+++ b/drivers/gpu/drm/nouveau/nv40_graph.c
@@ -29,6 +29,9 @@
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_grctx.h" 30#include "nouveau_grctx.h"
31 31
32static int nv40_graph_register(struct drm_device *);
33static void nv40_graph_isr(struct drm_device *);
34
32struct nouveau_channel * 35struct nouveau_channel *
33nv40_graph_channel(struct drm_device *dev) 36nv40_graph_channel(struct drm_device *dev)
34{ 37{
@@ -42,7 +45,7 @@ nv40_graph_channel(struct drm_device *dev)
42 inst = (inst & NV40_PGRAPH_CTXCTL_CUR_INSTANCE) << 4; 45 inst = (inst & NV40_PGRAPH_CTXCTL_CUR_INSTANCE) << 4;
43 46
44 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 47 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
45 struct nouveau_channel *chan = dev_priv->fifos[i]; 48 struct nouveau_channel *chan = dev_priv->channels.ptr[i];
46 49
47 if (chan && chan->ramin_grctx && 50 if (chan && chan->ramin_grctx &&
48 chan->ramin_grctx->pinst == inst) 51 chan->ramin_grctx->pinst == inst)
@@ -59,6 +62,7 @@ nv40_graph_create_context(struct nouveau_channel *chan)
59 struct drm_nouveau_private *dev_priv = dev->dev_private; 62 struct drm_nouveau_private *dev_priv = dev->dev_private;
60 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph; 63 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
61 struct nouveau_grctx ctx = {}; 64 struct nouveau_grctx ctx = {};
65 unsigned long flags;
62 int ret; 66 int ret;
63 67
64 ret = nouveau_gpuobj_new(dev, chan, pgraph->grctx_size, 16, 68 ret = nouveau_gpuobj_new(dev, chan, pgraph->grctx_size, 16,
@@ -73,12 +77,39 @@ nv40_graph_create_context(struct nouveau_channel *chan)
73 nv40_grctx_init(&ctx); 77 nv40_grctx_init(&ctx);
74 78
75 nv_wo32(chan->ramin_grctx, 0, chan->ramin_grctx->pinst); 79 nv_wo32(chan->ramin_grctx, 0, chan->ramin_grctx->pinst);
80
81 /* init grctx pointer in ramfc, and on PFIFO if channel is
82 * already active there
83 */
84 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
85 nv_wo32(chan->ramfc, 0x38, chan->ramin_grctx->pinst >> 4);
86 nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
87 if ((nv_rd32(dev, 0x003204) & 0x0000001f) == chan->id)
88 nv_wr32(dev, 0x0032e0, chan->ramin_grctx->pinst >> 4);
89 nv_mask(dev, 0x002500, 0x00000001, 0x00000001);
90 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
76 return 0; 91 return 0;
77} 92}
78 93
79void 94void
80nv40_graph_destroy_context(struct nouveau_channel *chan) 95nv40_graph_destroy_context(struct nouveau_channel *chan)
81{ 96{
97 struct drm_device *dev = chan->dev;
98 struct drm_nouveau_private *dev_priv = dev->dev_private;
99 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
100 unsigned long flags;
101
102 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
103 pgraph->fifo_access(dev, false);
104
105 /* Unload the context if it's the currently active one */
106 if (pgraph->channel(dev) == chan)
107 pgraph->unload_context(dev);
108
109 pgraph->fifo_access(dev, true);
110 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
111
112 /* Free the context resources */
82 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx); 113 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx);
83} 114}
84 115
@@ -174,43 +205,39 @@ nv40_graph_unload_context(struct drm_device *dev)
174} 205}
175 206
176void 207void
177nv40_graph_set_region_tiling(struct drm_device *dev, int i, uint32_t addr, 208nv40_graph_set_tile_region(struct drm_device *dev, int i)
178 uint32_t size, uint32_t pitch)
179{ 209{
180 struct drm_nouveau_private *dev_priv = dev->dev_private; 210 struct drm_nouveau_private *dev_priv = dev->dev_private;
181 uint32_t limit = max(1u, addr + size) - 1; 211 struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
182
183 if (pitch)
184 addr |= 1;
185 212
186 switch (dev_priv->chipset) { 213 switch (dev_priv->chipset) {
187 case 0x44: 214 case 0x44:
188 case 0x4a: 215 case 0x4a:
189 case 0x4e: 216 case 0x4e:
190 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch); 217 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
191 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit); 218 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
192 nv_wr32(dev, NV20_PGRAPH_TILE(i), addr); 219 nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
193 break; 220 break;
194 221
195 case 0x46: 222 case 0x46:
196 case 0x47: 223 case 0x47:
197 case 0x49: 224 case 0x49:
198 case 0x4b: 225 case 0x4b:
199 nv_wr32(dev, NV47_PGRAPH_TSIZE(i), pitch); 226 nv_wr32(dev, NV47_PGRAPH_TSIZE(i), tile->pitch);
200 nv_wr32(dev, NV47_PGRAPH_TLIMIT(i), limit); 227 nv_wr32(dev, NV47_PGRAPH_TLIMIT(i), tile->limit);
201 nv_wr32(dev, NV47_PGRAPH_TILE(i), addr); 228 nv_wr32(dev, NV47_PGRAPH_TILE(i), tile->addr);
202 nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), pitch); 229 nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
203 nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), limit); 230 nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
204 nv_wr32(dev, NV40_PGRAPH_TILE1(i), addr); 231 nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
205 break; 232 break;
206 233
207 default: 234 default:
208 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), pitch); 235 nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
209 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), limit); 236 nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
210 nv_wr32(dev, NV20_PGRAPH_TILE(i), addr); 237 nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
211 nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), pitch); 238 nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
212 nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), limit); 239 nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
213 nv_wr32(dev, NV40_PGRAPH_TILE1(i), addr); 240 nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
214 break; 241 break;
215 } 242 }
216} 243}
@@ -232,7 +259,7 @@ nv40_graph_init(struct drm_device *dev)
232 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb; 259 struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
233 struct nouveau_grctx ctx = {}; 260 struct nouveau_grctx ctx = {};
234 uint32_t vramsz, *cp; 261 uint32_t vramsz, *cp;
235 int i, j; 262 int ret, i, j;
236 263
237 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) & 264 nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
238 ~NV_PMC_ENABLE_PGRAPH); 265 ~NV_PMC_ENABLE_PGRAPH);
@@ -256,9 +283,14 @@ nv40_graph_init(struct drm_device *dev)
256 283
257 kfree(cp); 284 kfree(cp);
258 285
286 ret = nv40_graph_register(dev);
287 if (ret)
288 return ret;
289
259 /* No context present currently */ 290 /* No context present currently */
260 nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, 0x00000000); 291 nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, 0x00000000);
261 292
293 nouveau_irq_register(dev, 12, nv40_graph_isr);
262 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF); 294 nv_wr32(dev, NV03_PGRAPH_INTR , 0xFFFFFFFF);
263 nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xFFFFFFFF); 295 nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xFFFFFFFF);
264 296
@@ -347,7 +379,7 @@ nv40_graph_init(struct drm_device *dev)
347 379
348 /* Turn all the tiling regions off. */ 380 /* Turn all the tiling regions off. */
349 for (i = 0; i < pfb->num_tiles; i++) 381 for (i = 0; i < pfb->num_tiles; i++)
350 nv40_graph_set_region_tiling(dev, i, 0, 0, 0); 382 nv40_graph_set_tile_region(dev, i);
351 383
352 /* begin RAM config */ 384 /* begin RAM config */
353 vramsz = pci_resource_len(dev->pdev, 0) - 1; 385 vramsz = pci_resource_len(dev->pdev, 0) - 1;
@@ -390,26 +422,111 @@ nv40_graph_init(struct drm_device *dev)
390 422
391void nv40_graph_takedown(struct drm_device *dev) 423void nv40_graph_takedown(struct drm_device *dev)
392{ 424{
425 nouveau_irq_unregister(dev, 12);
393} 426}
394 427
395struct nouveau_pgraph_object_class nv40_graph_grclass[] = { 428static int
396 { 0x0030, false, NULL }, /* null */ 429nv40_graph_register(struct drm_device *dev)
397 { 0x0039, false, NULL }, /* m2mf */ 430{
398 { 0x004a, false, NULL }, /* gdirect */ 431 struct drm_nouveau_private *dev_priv = dev->dev_private;
399 { 0x009f, false, NULL }, /* imageblit (nv12) */ 432
400 { 0x008a, false, NULL }, /* ifc */ 433 if (dev_priv->engine.graph.registered)
401 { 0x0089, false, NULL }, /* sifm */ 434 return 0;
402 { 0x3089, false, NULL }, /* sifm (nv40) */
403 { 0x0062, false, NULL }, /* surf2d */
404 { 0x3062, false, NULL }, /* surf2d (nv40) */
405 { 0x0043, false, NULL }, /* rop */
406 { 0x0012, false, NULL }, /* beta1 */
407 { 0x0072, false, NULL }, /* beta4 */
408 { 0x0019, false, NULL }, /* cliprect */
409 { 0x0044, false, NULL }, /* pattern */
410 { 0x309e, false, NULL }, /* swzsurf */
411 { 0x4097, false, NULL }, /* curie (nv40) */
412 { 0x4497, false, NULL }, /* curie (nv44) */
413 {}
414};
415 435
436 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
437 NVOBJ_CLASS(dev, 0x0030, GR); /* null */
438 NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
439 NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
440 NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
441 NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
442 NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
443 NVOBJ_CLASS(dev, 0x3089, GR); /* sifm (nv40) */
444 NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
445 NVOBJ_CLASS(dev, 0x3062, GR); /* surf2d (nv40) */
446 NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
447 NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
448 NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
449 NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
450 NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
451 NVOBJ_CLASS(dev, 0x309e, GR); /* swzsurf */
452
453 /* curie */
454 if (dev_priv->chipset >= 0x60 ||
455 0x00005450 & (1 << (dev_priv->chipset & 0x0f)))
456 NVOBJ_CLASS(dev, 0x4497, GR);
457 else
458 NVOBJ_CLASS(dev, 0x4097, GR);
459
460 /* nvsw */
461 NVOBJ_CLASS(dev, 0x506e, SW);
462 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv04_graph_mthd_page_flip);
463
464 dev_priv->engine.graph.registered = true;
465 return 0;
466}
467
468static int
469nv40_graph_isr_chid(struct drm_device *dev, u32 inst)
470{
471 struct drm_nouveau_private *dev_priv = dev->dev_private;
472 struct nouveau_channel *chan;
473 unsigned long flags;
474 int i;
475
476 spin_lock_irqsave(&dev_priv->channels.lock, flags);
477 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
478 chan = dev_priv->channels.ptr[i];
479 if (!chan || !chan->ramin_grctx)
480 continue;
481
482 if (inst == chan->ramin_grctx->pinst)
483 break;
484 }
485 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
486 return i;
487}
488
489static void
490nv40_graph_isr(struct drm_device *dev)
491{
492 u32 stat;
493
494 while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
495 u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
496 u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
497 u32 inst = (nv_rd32(dev, 0x40032c) & 0x000fffff) << 4;
498 u32 chid = nv40_graph_isr_chid(dev, inst);
499 u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
500 u32 subc = (addr & 0x00070000) >> 16;
501 u32 mthd = (addr & 0x00001ffc);
502 u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
503 u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xffff;
504 u32 show = stat;
505
506 if (stat & NV_PGRAPH_INTR_ERROR) {
507 if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
508 if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
509 show &= ~NV_PGRAPH_INTR_ERROR;
510 } else
511 if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
512 nv_mask(dev, 0x402000, 0, 0);
513 }
514 }
515
516 nv_wr32(dev, NV03_PGRAPH_INTR, stat);
517 nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
518
519 if (show && nouveau_ratelimit()) {
520 NV_INFO(dev, "PGRAPH -");
521 nouveau_bitfield_print(nv10_graph_intr, show);
522 printk(" nsource:");
523 nouveau_bitfield_print(nv04_graph_nsource, nsource);
524 printk(" nstatus:");
525 nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
526 printk("\n");
527 NV_INFO(dev, "PGRAPH - ch %d (0x%08x) subc %d "
528 "class 0x%04x mthd 0x%04x data 0x%08x\n",
529 chid, inst, subc, class, mthd, data);
530 }
531 }
532}
diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c
index 56476d0c6de8..9023c4dbb449 100644
--- a/drivers/gpu/drm/nouveau/nv50_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv50_crtc.c
@@ -115,15 +115,16 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
115 OUT_RING(evo, 0); 115 OUT_RING(evo, 0);
116 BEGIN_RING(evo, 0, NV50_EVO_CRTC(index, FB_DMA), 1); 116 BEGIN_RING(evo, 0, NV50_EVO_CRTC(index, FB_DMA), 1);
117 if (dev_priv->chipset != 0x50) 117 if (dev_priv->chipset != 0x50)
118 if (nv_crtc->fb.tile_flags == 0x7a00) 118 if (nv_crtc->fb.tile_flags == 0x7a00 ||
119 nv_crtc->fb.tile_flags == 0xfe00)
119 OUT_RING(evo, NvEvoFB32); 120 OUT_RING(evo, NvEvoFB32);
120 else 121 else
121 if (nv_crtc->fb.tile_flags == 0x7000) 122 if (nv_crtc->fb.tile_flags == 0x7000)
122 OUT_RING(evo, NvEvoFB16); 123 OUT_RING(evo, NvEvoFB16);
123 else 124 else
124 OUT_RING(evo, NvEvoVRAM); 125 OUT_RING(evo, NvEvoVRAM_LP);
125 else 126 else
126 OUT_RING(evo, NvEvoVRAM); 127 OUT_RING(evo, NvEvoVRAM_LP);
127 } 128 }
128 129
129 nv_crtc->fb.blanked = blanked; 130 nv_crtc->fb.blanked = blanked;
@@ -345,7 +346,6 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
345 uint32_t buffer_handle, uint32_t width, uint32_t height) 346 uint32_t buffer_handle, uint32_t width, uint32_t height)
346{ 347{
347 struct drm_device *dev = crtc->dev; 348 struct drm_device *dev = crtc->dev;
348 struct drm_nouveau_private *dev_priv = dev->dev_private;
349 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); 349 struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
350 struct nouveau_bo *cursor = NULL; 350 struct nouveau_bo *cursor = NULL;
351 struct drm_gem_object *gem; 351 struct drm_gem_object *gem;
@@ -374,8 +374,7 @@ nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
374 374
375 nouveau_bo_unmap(cursor); 375 nouveau_bo_unmap(cursor);
376 376
377 nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.offset - 377 nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.nvbo->bo.mem.start << PAGE_SHIFT);
378 dev_priv->vm_vram_base);
379 nv_crtc->cursor.show(nv_crtc, true); 378 nv_crtc->cursor.show(nv_crtc, true);
380 379
381out: 380out:
@@ -437,6 +436,7 @@ static const struct drm_crtc_funcs nv50_crtc_funcs = {
437 .cursor_move = nv50_crtc_cursor_move, 436 .cursor_move = nv50_crtc_cursor_move,
438 .gamma_set = nv50_crtc_gamma_set, 437 .gamma_set = nv50_crtc_gamma_set,
439 .set_config = drm_crtc_helper_set_config, 438 .set_config = drm_crtc_helper_set_config,
439 .page_flip = nouveau_crtc_page_flip,
440 .destroy = nv50_crtc_destroy, 440 .destroy = nv50_crtc_destroy,
441}; 441};
442 442
@@ -453,6 +453,7 @@ nv50_crtc_prepare(struct drm_crtc *crtc)
453 453
454 NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index); 454 NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
455 455
456 drm_vblank_pre_modeset(dev, nv_crtc->index);
456 nv50_crtc_blank(nv_crtc, true); 457 nv50_crtc_blank(nv_crtc, true);
457} 458}
458 459
@@ -468,6 +469,7 @@ nv50_crtc_commit(struct drm_crtc *crtc)
468 NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index); 469 NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
469 470
470 nv50_crtc_blank(nv_crtc, false); 471 nv50_crtc_blank(nv_crtc, false);
472 drm_vblank_post_modeset(dev, nv_crtc->index);
471 473
472 ret = RING_SPACE(evo, 2); 474 ret = RING_SPACE(evo, 2);
473 if (ret) { 475 if (ret) {
@@ -545,7 +547,7 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
545 return -EINVAL; 547 return -EINVAL;
546 } 548 }
547 549
548 nv_crtc->fb.offset = fb->nvbo->bo.offset - dev_priv->vm_vram_base; 550 nv_crtc->fb.offset = fb->nvbo->bo.mem.start << PAGE_SHIFT;
549 nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo); 551 nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo);
550 nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8; 552 nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8;
551 if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) { 553 if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) {
@@ -554,13 +556,14 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
554 return ret; 556 return ret;
555 557
556 BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, FB_DMA), 1); 558 BEGIN_RING(evo, 0, NV50_EVO_CRTC(nv_crtc->index, FB_DMA), 1);
557 if (nv_crtc->fb.tile_flags == 0x7a00) 559 if (nv_crtc->fb.tile_flags == 0x7a00 ||
560 nv_crtc->fb.tile_flags == 0xfe00)
558 OUT_RING(evo, NvEvoFB32); 561 OUT_RING(evo, NvEvoFB32);
559 else 562 else
560 if (nv_crtc->fb.tile_flags == 0x7000) 563 if (nv_crtc->fb.tile_flags == 0x7000)
561 OUT_RING(evo, NvEvoFB16); 564 OUT_RING(evo, NvEvoFB16);
562 else 565 else
563 OUT_RING(evo, NvEvoVRAM); 566 OUT_RING(evo, NvEvoVRAM_LP);
564 } 567 }
565 568
566 ret = RING_SPACE(evo, 12); 569 ret = RING_SPACE(evo, 12);
@@ -574,8 +577,10 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
574 if (!nv_crtc->fb.tile_flags) { 577 if (!nv_crtc->fb.tile_flags) {
575 OUT_RING(evo, drm_fb->pitch | (1 << 20)); 578 OUT_RING(evo, drm_fb->pitch | (1 << 20));
576 } else { 579 } else {
577 OUT_RING(evo, ((drm_fb->pitch / 4) << 4) | 580 u32 tile_mode = fb->nvbo->tile_mode;
578 fb->nvbo->tile_mode); 581 if (dev_priv->card_type >= NV_C0)
582 tile_mode >>= 4;
583 OUT_RING(evo, ((drm_fb->pitch / 4) << 4) | tile_mode);
579 } 584 }
580 if (dev_priv->chipset == 0x50) 585 if (dev_priv->chipset == 0x50)
581 OUT_RING(evo, (nv_crtc->fb.tile_flags << 8) | format); 586 OUT_RING(evo, (nv_crtc->fb.tile_flags << 8) | format);
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index f624c611ddea..7cc94ed9ed95 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -33,6 +33,8 @@
33#include "nouveau_ramht.h" 33#include "nouveau_ramht.h"
34#include "drm_crtc_helper.h" 34#include "drm_crtc_helper.h"
35 35
36static void nv50_display_isr(struct drm_device *);
37
36static inline int 38static inline int
37nv50_sor_nr(struct drm_device *dev) 39nv50_sor_nr(struct drm_device *dev)
38{ 40{
@@ -46,159 +48,6 @@ nv50_sor_nr(struct drm_device *dev)
46 return 4; 48 return 4;
47} 49}
48 50
49static void
50nv50_evo_channel_del(struct nouveau_channel **pchan)
51{
52 struct nouveau_channel *chan = *pchan;
53
54 if (!chan)
55 return;
56 *pchan = NULL;
57
58 nouveau_gpuobj_channel_takedown(chan);
59 nouveau_bo_unmap(chan->pushbuf_bo);
60 nouveau_bo_ref(NULL, &chan->pushbuf_bo);
61
62 if (chan->user)
63 iounmap(chan->user);
64
65 kfree(chan);
66}
67
68static int
69nv50_evo_dmaobj_new(struct nouveau_channel *evo, uint32_t class, uint32_t name,
70 uint32_t tile_flags, uint32_t magic_flags,
71 uint32_t offset, uint32_t limit)
72{
73 struct drm_nouveau_private *dev_priv = evo->dev->dev_private;
74 struct drm_device *dev = evo->dev;
75 struct nouveau_gpuobj *obj = NULL;
76 int ret;
77
78 ret = nouveau_gpuobj_new(dev, evo, 6*4, 32, 0, &obj);
79 if (ret)
80 return ret;
81 obj->engine = NVOBJ_ENGINE_DISPLAY;
82
83 nv_wo32(obj, 0, (tile_flags << 22) | (magic_flags << 16) | class);
84 nv_wo32(obj, 4, limit);
85 nv_wo32(obj, 8, offset);
86 nv_wo32(obj, 12, 0x00000000);
87 nv_wo32(obj, 16, 0x00000000);
88 if (dev_priv->card_type < NV_C0)
89 nv_wo32(obj, 20, 0x00010000);
90 else
91 nv_wo32(obj, 20, 0x00020000);
92 dev_priv->engine.instmem.flush(dev);
93
94 ret = nouveau_ramht_insert(evo, name, obj);
95 nouveau_gpuobj_ref(NULL, &obj);
96 if (ret) {
97 return ret;
98 }
99
100 return 0;
101}
102
103static int
104nv50_evo_channel_new(struct drm_device *dev, struct nouveau_channel **pchan)
105{
106 struct drm_nouveau_private *dev_priv = dev->dev_private;
107 struct nouveau_gpuobj *ramht = NULL;
108 struct nouveau_channel *chan;
109 int ret;
110
111 chan = kzalloc(sizeof(struct nouveau_channel), GFP_KERNEL);
112 if (!chan)
113 return -ENOMEM;
114 *pchan = chan;
115
116 chan->id = -1;
117 chan->dev = dev;
118 chan->user_get = 4;
119 chan->user_put = 0;
120
121 ret = nouveau_gpuobj_new(dev, NULL, 32768, 0x1000,
122 NVOBJ_FLAG_ZERO_ALLOC, &chan->ramin);
123 if (ret) {
124 NV_ERROR(dev, "Error allocating EVO channel memory: %d\n", ret);
125 nv50_evo_channel_del(pchan);
126 return ret;
127 }
128
129 ret = drm_mm_init(&chan->ramin_heap, 0, 32768);
130 if (ret) {
131 NV_ERROR(dev, "Error initialising EVO PRAMIN heap: %d\n", ret);
132 nv50_evo_channel_del(pchan);
133 return ret;
134 }
135
136 ret = nouveau_gpuobj_new(dev, chan, 4096, 16, 0, &ramht);
137 if (ret) {
138 NV_ERROR(dev, "Unable to allocate EVO RAMHT: %d\n", ret);
139 nv50_evo_channel_del(pchan);
140 return ret;
141 }
142
143 ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
144 nouveau_gpuobj_ref(NULL, &ramht);
145 if (ret) {
146 nv50_evo_channel_del(pchan);
147 return ret;
148 }
149
150 if (dev_priv->chipset != 0x50) {
151 ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoFB16, 0x70, 0x19,
152 0, 0xffffffff);
153 if (ret) {
154 nv50_evo_channel_del(pchan);
155 return ret;
156 }
157
158
159 ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoFB32, 0x7a, 0x19,
160 0, 0xffffffff);
161 if (ret) {
162 nv50_evo_channel_del(pchan);
163 return ret;
164 }
165 }
166
167 ret = nv50_evo_dmaobj_new(chan, 0x3d, NvEvoVRAM, 0, 0x19,
168 0, dev_priv->vram_size);
169 if (ret) {
170 nv50_evo_channel_del(pchan);
171 return ret;
172 }
173
174 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
175 false, true, &chan->pushbuf_bo);
176 if (ret == 0)
177 ret = nouveau_bo_pin(chan->pushbuf_bo, TTM_PL_FLAG_VRAM);
178 if (ret) {
179 NV_ERROR(dev, "Error creating EVO DMA push buffer: %d\n", ret);
180 nv50_evo_channel_del(pchan);
181 return ret;
182 }
183
184 ret = nouveau_bo_map(chan->pushbuf_bo);
185 if (ret) {
186 NV_ERROR(dev, "Error mapping EVO DMA push buffer: %d\n", ret);
187 nv50_evo_channel_del(pchan);
188 return ret;
189 }
190
191 chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
192 NV50_PDISPLAY_USER(0), PAGE_SIZE);
193 if (!chan->user) {
194 NV_ERROR(dev, "Error mapping EVO control regs.\n");
195 nv50_evo_channel_del(pchan);
196 return -ENOMEM;
197 }
198
199 return 0;
200}
201
202int 51int
203nv50_display_early_init(struct drm_device *dev) 52nv50_display_early_init(struct drm_device *dev)
204{ 53{
@@ -214,17 +63,16 @@ int
214nv50_display_init(struct drm_device *dev) 63nv50_display_init(struct drm_device *dev)
215{ 64{
216 struct drm_nouveau_private *dev_priv = dev->dev_private; 65 struct drm_nouveau_private *dev_priv = dev->dev_private;
217 struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
218 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio; 66 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
219 struct nouveau_channel *evo = dev_priv->evo;
220 struct drm_connector *connector; 67 struct drm_connector *connector;
221 uint32_t val, ram_amount; 68 struct nouveau_channel *evo;
222 uint64_t start;
223 int ret, i; 69 int ret, i;
70 u32 val;
224 71
225 NV_DEBUG_KMS(dev, "\n"); 72 NV_DEBUG_KMS(dev, "\n");
226 73
227 nv_wr32(dev, 0x00610184, nv_rd32(dev, 0x00614004)); 74 nv_wr32(dev, 0x00610184, nv_rd32(dev, 0x00614004));
75
228 /* 76 /*
229 * I think the 0x006101XX range is some kind of main control area 77 * I think the 0x006101XX range is some kind of main control area
230 * that enables things. 78 * that enables things.
@@ -240,16 +88,19 @@ nv50_display_init(struct drm_device *dev)
240 val = nv_rd32(dev, 0x0061610c + (i * 0x800)); 88 val = nv_rd32(dev, 0x0061610c + (i * 0x800));
241 nv_wr32(dev, 0x0061019c + (i * 0x10), val); 89 nv_wr32(dev, 0x0061019c + (i * 0x10), val);
242 } 90 }
91
243 /* DAC */ 92 /* DAC */
244 for (i = 0; i < 3; i++) { 93 for (i = 0; i < 3; i++) {
245 val = nv_rd32(dev, 0x0061a000 + (i * 0x800)); 94 val = nv_rd32(dev, 0x0061a000 + (i * 0x800));
246 nv_wr32(dev, 0x006101d0 + (i * 0x04), val); 95 nv_wr32(dev, 0x006101d0 + (i * 0x04), val);
247 } 96 }
97
248 /* SOR */ 98 /* SOR */
249 for (i = 0; i < nv50_sor_nr(dev); i++) { 99 for (i = 0; i < nv50_sor_nr(dev); i++) {
250 val = nv_rd32(dev, 0x0061c000 + (i * 0x800)); 100 val = nv_rd32(dev, 0x0061c000 + (i * 0x800));
251 nv_wr32(dev, 0x006101e0 + (i * 0x04), val); 101 nv_wr32(dev, 0x006101e0 + (i * 0x04), val);
252 } 102 }
103
253 /* EXT */ 104 /* EXT */
254 for (i = 0; i < 3; i++) { 105 for (i = 0; i < 3; i++) {
255 val = nv_rd32(dev, 0x0061e000 + (i * 0x800)); 106 val = nv_rd32(dev, 0x0061e000 + (i * 0x800));
@@ -262,17 +113,6 @@ nv50_display_init(struct drm_device *dev)
262 nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL1(i), 0x00000001); 113 nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL1(i), 0x00000001);
263 } 114 }
264 115
265 /* This used to be in crtc unblank, but seems out of place there. */
266 nv_wr32(dev, NV50_PDISPLAY_UNK_380, 0);
267 /* RAM is clamped to 256 MiB. */
268 ram_amount = dev_priv->vram_size;
269 NV_DEBUG_KMS(dev, "ram_amount %d\n", ram_amount);
270 if (ram_amount > 256*1024*1024)
271 ram_amount = 256*1024*1024;
272 nv_wr32(dev, NV50_PDISPLAY_RAM_AMOUNT, ram_amount - 1);
273 nv_wr32(dev, NV50_PDISPLAY_UNK_388, 0x150000);
274 nv_wr32(dev, NV50_PDISPLAY_UNK_38C, 0);
275
276 /* The precise purpose is unknown, i suspect it has something to do 116 /* The precise purpose is unknown, i suspect it has something to do
277 * with text mode. 117 * with text mode.
278 */ 118 */
@@ -287,37 +127,6 @@ nv50_display_init(struct drm_device *dev)
287 } 127 }
288 } 128 }
289 129
290 /* taken from nv bug #12637, attempts to un-wedge the hw if it's
291 * stuck in some unspecified state
292 */
293 start = ptimer->read(dev);
294 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x2b00);
295 while ((val = nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0))) & 0x1e0000) {
296 if ((val & 0x9f0000) == 0x20000)
297 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
298 val | 0x800000);
299
300 if ((val & 0x3f0000) == 0x30000)
301 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
302 val | 0x200000);
303
304 if (ptimer->read(dev) - start > 1000000000ULL) {
305 NV_ERROR(dev, "timeout: (0x610200 & 0x1e0000) != 0\n");
306 NV_ERROR(dev, "0x610200 = 0x%08x\n", val);
307 return -EBUSY;
308 }
309 }
310
311 nv_wr32(dev, NV50_PDISPLAY_CTRL_STATE, NV50_PDISPLAY_CTRL_STATE_ENABLE);
312 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x1000b03);
313 if (!nv_wait(dev, NV50_PDISPLAY_CHANNEL_STAT(0),
314 0x40000000, 0x40000000)) {
315 NV_ERROR(dev, "timeout: (0x610200 & 0x40000000) == 0x40000000\n");
316 NV_ERROR(dev, "0x610200 = 0x%08x\n",
317 nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)));
318 return -EBUSY;
319 }
320
321 for (i = 0; i < 2; i++) { 130 for (i = 0; i < 2; i++) {
322 nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0x2000); 131 nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0x2000);
323 if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 132 if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
@@ -341,39 +150,31 @@ nv50_display_init(struct drm_device *dev)
341 } 150 }
342 } 151 }
343 152
344 nv_wr32(dev, NV50_PDISPLAY_OBJECTS, (evo->ramin->vinst >> 8) | 9); 153 nv_wr32(dev, NV50_PDISPLAY_PIO_CTRL, 0x00000000);
154 nv_mask(dev, NV50_PDISPLAY_INTR_0, 0x00000000, 0x00000000);
155 nv_wr32(dev, NV50_PDISPLAY_INTR_EN_0, 0x00000000);
156 nv_mask(dev, NV50_PDISPLAY_INTR_1, 0x00000000, 0x00000000);
157 nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1,
158 NV50_PDISPLAY_INTR_EN_1_CLK_UNK10 |
159 NV50_PDISPLAY_INTR_EN_1_CLK_UNK20 |
160 NV50_PDISPLAY_INTR_EN_1_CLK_UNK40);
161
162 /* enable hotplug interrupts */
163 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
164 struct nouveau_connector *conn = nouveau_connector(connector);
345 165
346 /* initialise fifo */ 166 if (conn->dcb->gpio_tag == 0xff)
347 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_DMA_CB(0), 167 continue;
348 ((evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT) >> 8) | 168
349 NV50_PDISPLAY_CHANNEL_DMA_CB_LOCATION_VRAM | 169 pgpio->irq_enable(dev, conn->dcb->gpio_tag, true);
350 NV50_PDISPLAY_CHANNEL_DMA_CB_VALID);
351 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_UNK2(0), 0x00010000);
352 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_UNK3(0), 0x00000002);
353 if (!nv_wait(dev, 0x610200, 0x80000000, 0x00000000)) {
354 NV_ERROR(dev, "timeout: (0x610200 & 0x80000000) == 0\n");
355 NV_ERROR(dev, "0x610200 = 0x%08x\n", nv_rd32(dev, 0x610200));
356 return -EBUSY;
357 } 170 }
358 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 171
359 (nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)) & ~0x00000003) | 172 ret = nv50_evo_init(dev);
360 NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED);
361 nv_wr32(dev, NV50_PDISPLAY_USER_PUT(0), 0);
362 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x01000003 |
363 NV50_PDISPLAY_CHANNEL_STAT_DMA_ENABLED);
364 nv_wr32(dev, 0x610300, nv_rd32(dev, 0x610300) & ~1);
365
366 evo->dma.max = (4096/4) - 2;
367 evo->dma.put = 0;
368 evo->dma.cur = evo->dma.put;
369 evo->dma.free = evo->dma.max - evo->dma.cur;
370
371 ret = RING_SPACE(evo, NOUVEAU_DMA_SKIPS);
372 if (ret) 173 if (ret)
373 return ret; 174 return ret;
175 evo = dev_priv->evo;
374 176
375 for (i = 0; i < NOUVEAU_DMA_SKIPS; i++) 177 nv_wr32(dev, NV50_PDISPLAY_OBJECTS, (evo->ramin->vinst >> 8) | 9);
376 OUT_RING(evo, 0);
377 178
378 ret = RING_SPACE(evo, 11); 179 ret = RING_SPACE(evo, 11);
379 if (ret) 180 if (ret)
@@ -393,21 +194,6 @@ nv50_display_init(struct drm_device *dev)
393 if (!nv_wait(dev, 0x640004, 0xffffffff, evo->dma.put << 2)) 194 if (!nv_wait(dev, 0x640004, 0xffffffff, evo->dma.put << 2))
394 NV_ERROR(dev, "evo pushbuf stalled\n"); 195 NV_ERROR(dev, "evo pushbuf stalled\n");
395 196
396 /* enable clock change interrupts. */
397 nv_wr32(dev, 0x610028, 0x00010001);
398 nv_wr32(dev, NV50_PDISPLAY_INTR_EN, (NV50_PDISPLAY_INTR_EN_CLK_UNK10 |
399 NV50_PDISPLAY_INTR_EN_CLK_UNK20 |
400 NV50_PDISPLAY_INTR_EN_CLK_UNK40));
401
402 /* enable hotplug interrupts */
403 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
404 struct nouveau_connector *conn = nouveau_connector(connector);
405
406 if (conn->dcb->gpio_tag == 0xff)
407 continue;
408
409 pgpio->irq_enable(dev, conn->dcb->gpio_tag, true);
410 }
411 197
412 return 0; 198 return 0;
413} 199}
@@ -452,13 +238,7 @@ static int nv50_display_disable(struct drm_device *dev)
452 } 238 }
453 } 239 }
454 240
455 nv_wr32(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0); 241 nv50_evo_fini(dev);
456 nv_wr32(dev, NV50_PDISPLAY_CTRL_STATE, 0);
457 if (!nv_wait(dev, NV50_PDISPLAY_CHANNEL_STAT(0), 0x1e0000, 0)) {
458 NV_ERROR(dev, "timeout: (0x610200 & 0x1e0000) == 0\n");
459 NV_ERROR(dev, "0x610200 = 0x%08x\n",
460 nv_rd32(dev, NV50_PDISPLAY_CHANNEL_STAT(0)));
461 }
462 242
463 for (i = 0; i < 3; i++) { 243 for (i = 0; i < 3; i++) {
464 if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(i), 244 if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(i),
@@ -470,7 +250,7 @@ static int nv50_display_disable(struct drm_device *dev)
470 } 250 }
471 251
472 /* disable interrupts. */ 252 /* disable interrupts. */
473 nv_wr32(dev, NV50_PDISPLAY_INTR_EN, 0x00000000); 253 nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1, 0x00000000);
474 254
475 /* disable hotplug interrupts */ 255 /* disable hotplug interrupts */
476 nv_wr32(dev, 0xe054, 0xffffffff); 256 nv_wr32(dev, 0xe054, 0xffffffff);
@@ -508,13 +288,6 @@ int nv50_display_create(struct drm_device *dev)
508 288
509 dev->mode_config.fb_base = dev_priv->fb_phys; 289 dev->mode_config.fb_base = dev_priv->fb_phys;
510 290
511 /* Create EVO channel */
512 ret = nv50_evo_channel_new(dev, &dev_priv->evo);
513 if (ret) {
514 NV_ERROR(dev, "Error creating EVO channel: %d\n", ret);
515 return ret;
516 }
517
518 /* Create CRTC objects */ 291 /* Create CRTC objects */
519 for (i = 0; i < 2; i++) 292 for (i = 0; i < 2; i++)
520 nv50_crtc_create(dev, i); 293 nv50_crtc_create(dev, i);
@@ -557,6 +330,9 @@ int nv50_display_create(struct drm_device *dev)
557 } 330 }
558 } 331 }
559 332
333 INIT_WORK(&dev_priv->irq_work, nv50_display_irq_handler_bh);
334 nouveau_irq_register(dev, 26, nv50_display_isr);
335
560 ret = nv50_display_init(dev); 336 ret = nv50_display_init(dev);
561 if (ret) { 337 if (ret) {
562 nv50_display_destroy(dev); 338 nv50_display_destroy(dev);
@@ -569,14 +345,12 @@ int nv50_display_create(struct drm_device *dev)
569void 345void
570nv50_display_destroy(struct drm_device *dev) 346nv50_display_destroy(struct drm_device *dev)
571{ 347{
572 struct drm_nouveau_private *dev_priv = dev->dev_private;
573
574 NV_DEBUG_KMS(dev, "\n"); 348 NV_DEBUG_KMS(dev, "\n");
575 349
576 drm_mode_config_cleanup(dev); 350 drm_mode_config_cleanup(dev);
577 351
578 nv50_display_disable(dev); 352 nv50_display_disable(dev);
579 nv50_evo_channel_del(&dev_priv->evo); 353 nouveau_irq_unregister(dev, 26);
580} 354}
581 355
582static u16 356static u16
@@ -660,32 +434,32 @@ static void
660nv50_display_vblank_crtc_handler(struct drm_device *dev, int crtc) 434nv50_display_vblank_crtc_handler(struct drm_device *dev, int crtc)
661{ 435{
662 struct drm_nouveau_private *dev_priv = dev->dev_private; 436 struct drm_nouveau_private *dev_priv = dev->dev_private;
663 struct nouveau_channel *chan; 437 struct nouveau_channel *chan, *tmp;
664 struct list_head *entry, *tmp;
665 438
666 list_for_each_safe(entry, tmp, &dev_priv->vbl_waiting) { 439 list_for_each_entry_safe(chan, tmp, &dev_priv->vbl_waiting,
667 chan = list_entry(entry, struct nouveau_channel, nvsw.vbl_wait); 440 nvsw.vbl_wait) {
441 if (chan->nvsw.vblsem_head != crtc)
442 continue;
668 443
669 nouveau_bo_wr32(chan->notifier_bo, chan->nvsw.vblsem_offset, 444 nouveau_bo_wr32(chan->notifier_bo, chan->nvsw.vblsem_offset,
670 chan->nvsw.vblsem_rval); 445 chan->nvsw.vblsem_rval);
671 list_del(&chan->nvsw.vbl_wait); 446 list_del(&chan->nvsw.vbl_wait);
447 drm_vblank_put(dev, crtc);
672 } 448 }
449
450 drm_handle_vblank(dev, crtc);
673} 451}
674 452
675static void 453static void
676nv50_display_vblank_handler(struct drm_device *dev, uint32_t intr) 454nv50_display_vblank_handler(struct drm_device *dev, uint32_t intr)
677{ 455{
678 intr &= NV50_PDISPLAY_INTR_1_VBLANK_CRTC;
679
680 if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_0) 456 if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_0)
681 nv50_display_vblank_crtc_handler(dev, 0); 457 nv50_display_vblank_crtc_handler(dev, 0);
682 458
683 if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_1) 459 if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_1)
684 nv50_display_vblank_crtc_handler(dev, 1); 460 nv50_display_vblank_crtc_handler(dev, 1);
685 461
686 nv_wr32(dev, NV50_PDISPLAY_INTR_EN, nv_rd32(dev, 462 nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_VBLANK_CRTC);
687 NV50_PDISPLAY_INTR_EN) & ~intr);
688 nv_wr32(dev, NV50_PDISPLAY_INTR_1, intr);
689} 463}
690 464
691static void 465static void
@@ -1011,108 +785,31 @@ nv50_display_irq_handler_bh(struct work_struct *work)
1011static void 785static void
1012nv50_display_error_handler(struct drm_device *dev) 786nv50_display_error_handler(struct drm_device *dev)
1013{ 787{
1014 uint32_t addr, data; 788 u32 channels = (nv_rd32(dev, NV50_PDISPLAY_INTR_0) & 0x001f0000) >> 16;
1015 789 u32 addr, data;
1016 nv_wr32(dev, NV50_PDISPLAY_INTR_0, 0x00010000); 790 int chid;
1017 addr = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_ADDR);
1018 data = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_DATA);
1019
1020 NV_ERROR(dev, "EvoCh %d Mthd 0x%04x Data 0x%08x (0x%04x 0x%02x)\n",
1021 0, addr & 0xffc, data, addr >> 16, (addr >> 12) & 0xf);
1022
1023 nv_wr32(dev, NV50_PDISPLAY_TRAPPED_ADDR, 0x90000000);
1024}
1025
1026void
1027nv50_display_irq_hotplug_bh(struct work_struct *work)
1028{
1029 struct drm_nouveau_private *dev_priv =
1030 container_of(work, struct drm_nouveau_private, hpd_work);
1031 struct drm_device *dev = dev_priv->dev;
1032 struct drm_connector *connector;
1033 const uint32_t gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
1034 uint32_t unplug_mask, plug_mask, change_mask;
1035 uint32_t hpd0, hpd1;
1036
1037 spin_lock_irq(&dev_priv->hpd_state.lock);
1038 hpd0 = dev_priv->hpd_state.hpd0_bits;
1039 dev_priv->hpd_state.hpd0_bits = 0;
1040 hpd1 = dev_priv->hpd_state.hpd1_bits;
1041 dev_priv->hpd_state.hpd1_bits = 0;
1042 spin_unlock_irq(&dev_priv->hpd_state.lock);
1043
1044 hpd0 &= nv_rd32(dev, 0xe050);
1045 if (dev_priv->chipset >= 0x90)
1046 hpd1 &= nv_rd32(dev, 0xe070);
1047
1048 plug_mask = (hpd0 & 0x0000ffff) | (hpd1 << 16);
1049 unplug_mask = (hpd0 >> 16) | (hpd1 & 0xffff0000);
1050 change_mask = plug_mask | unplug_mask;
1051
1052 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1053 struct drm_encoder_helper_funcs *helper;
1054 struct nouveau_connector *nv_connector =
1055 nouveau_connector(connector);
1056 struct nouveau_encoder *nv_encoder;
1057 struct dcb_gpio_entry *gpio;
1058 uint32_t reg;
1059 bool plugged;
1060
1061 if (!nv_connector->dcb)
1062 continue;
1063 791
1064 gpio = nouveau_bios_gpio_entry(dev, nv_connector->dcb->gpio_tag); 792 for (chid = 0; chid < 5; chid++) {
1065 if (!gpio || !(change_mask & (1 << gpio->line))) 793 if (!(channels & (1 << chid)))
1066 continue; 794 continue;
1067 795
1068 reg = nv_rd32(dev, gpio_reg[gpio->line >> 3]); 796 nv_wr32(dev, NV50_PDISPLAY_INTR_0, 0x00010000 << chid);
1069 plugged = !!(reg & (4 << ((gpio->line & 7) << 2))); 797 addr = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid));
1070 NV_INFO(dev, "%splugged %s\n", plugged ? "" : "un", 798 data = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_DATA(chid));
1071 drm_get_connector_name(connector)) ; 799 NV_ERROR(dev, "EvoCh %d Mthd 0x%04x Data 0x%08x "
1072 800 "(0x%04x 0x%02x)\n", chid,
1073 if (!connector->encoder || !connector->encoder->crtc || 801 addr & 0xffc, data, addr >> 16, (addr >> 12) & 0xf);
1074 !connector->encoder->crtc->enabled)
1075 continue;
1076 nv_encoder = nouveau_encoder(connector->encoder);
1077 helper = connector->encoder->helper_private;
1078
1079 if (nv_encoder->dcb->type != OUTPUT_DP)
1080 continue;
1081 802
1082 if (plugged) 803 nv_wr32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid), 0x90000000);
1083 helper->dpms(connector->encoder, DRM_MODE_DPMS_ON);
1084 else
1085 helper->dpms(connector->encoder, DRM_MODE_DPMS_OFF);
1086 } 804 }
1087
1088 drm_helper_hpd_irq_event(dev);
1089} 805}
1090 806
1091void 807static void
1092nv50_display_irq_handler(struct drm_device *dev) 808nv50_display_isr(struct drm_device *dev)
1093{ 809{
1094 struct drm_nouveau_private *dev_priv = dev->dev_private; 810 struct drm_nouveau_private *dev_priv = dev->dev_private;
1095 uint32_t delayed = 0; 811 uint32_t delayed = 0;
1096 812
1097 if (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_HOTPLUG) {
1098 uint32_t hpd0_bits, hpd1_bits = 0;
1099
1100 hpd0_bits = nv_rd32(dev, 0xe054);
1101 nv_wr32(dev, 0xe054, hpd0_bits);
1102
1103 if (dev_priv->chipset >= 0x90) {
1104 hpd1_bits = nv_rd32(dev, 0xe074);
1105 nv_wr32(dev, 0xe074, hpd1_bits);
1106 }
1107
1108 spin_lock(&dev_priv->hpd_state.lock);
1109 dev_priv->hpd_state.hpd0_bits |= hpd0_bits;
1110 dev_priv->hpd_state.hpd1_bits |= hpd1_bits;
1111 spin_unlock(&dev_priv->hpd_state.lock);
1112
1113 queue_work(dev_priv->wq, &dev_priv->hpd_work);
1114 }
1115
1116 while (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_DISPLAY) { 813 while (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_DISPLAY) {
1117 uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0); 814 uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0);
1118 uint32_t intr1 = nv_rd32(dev, NV50_PDISPLAY_INTR_1); 815 uint32_t intr1 = nv_rd32(dev, NV50_PDISPLAY_INTR_1);
@@ -1123,9 +820,9 @@ nv50_display_irq_handler(struct drm_device *dev)
1123 if (!intr0 && !(intr1 & ~delayed)) 820 if (!intr0 && !(intr1 & ~delayed))
1124 break; 821 break;
1125 822
1126 if (intr0 & 0x00010000) { 823 if (intr0 & 0x001f0000) {
1127 nv50_display_error_handler(dev); 824 nv50_display_error_handler(dev);
1128 intr0 &= ~0x00010000; 825 intr0 &= ~0x001f0000;
1129 } 826 }
1130 827
1131 if (intr1 & NV50_PDISPLAY_INTR_1_VBLANK_CRTC) { 828 if (intr1 & NV50_PDISPLAY_INTR_1_VBLANK_CRTC) {
@@ -1156,4 +853,3 @@ nv50_display_irq_handler(struct drm_device *dev)
1156 } 853 }
1157 } 854 }
1158} 855}
1159
diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h
index c551f0b85ee0..f0e30b78ef6b 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.h
+++ b/drivers/gpu/drm/nouveau/nv50_display.h
@@ -35,9 +35,7 @@
35#include "nouveau_crtc.h" 35#include "nouveau_crtc.h"
36#include "nv50_evo.h" 36#include "nv50_evo.h"
37 37
38void nv50_display_irq_handler(struct drm_device *dev);
39void nv50_display_irq_handler_bh(struct work_struct *work); 38void nv50_display_irq_handler_bh(struct work_struct *work);
40void nv50_display_irq_hotplug_bh(struct work_struct *work);
41int nv50_display_early_init(struct drm_device *dev); 39int nv50_display_early_init(struct drm_device *dev);
42void nv50_display_late_takedown(struct drm_device *dev); 40void nv50_display_late_takedown(struct drm_device *dev);
43int nv50_display_create(struct drm_device *dev); 41int nv50_display_create(struct drm_device *dev);
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.c b/drivers/gpu/drm/nouveau/nv50_evo.c
new file mode 100644
index 000000000000..14e24e906ee8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_evo.c
@@ -0,0 +1,345 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26
27#include "nouveau_drv.h"
28#include "nouveau_dma.h"
29#include "nouveau_ramht.h"
30
31static void
32nv50_evo_channel_del(struct nouveau_channel **pevo)
33{
34 struct drm_nouveau_private *dev_priv;
35 struct nouveau_channel *evo = *pevo;
36
37 if (!evo)
38 return;
39 *pevo = NULL;
40
41 dev_priv = evo->dev->dev_private;
42 dev_priv->evo_alloc &= ~(1 << evo->id);
43
44 nouveau_gpuobj_channel_takedown(evo);
45 nouveau_bo_unmap(evo->pushbuf_bo);
46 nouveau_bo_ref(NULL, &evo->pushbuf_bo);
47
48 if (evo->user)
49 iounmap(evo->user);
50
51 kfree(evo);
52}
53
54int
55nv50_evo_dmaobj_new(struct nouveau_channel *evo, u32 class, u32 name,
56 u32 tile_flags, u32 magic_flags, u32 offset, u32 limit,
57 u32 flags5)
58{
59 struct drm_nouveau_private *dev_priv = evo->dev->dev_private;
60 struct drm_device *dev = evo->dev;
61 struct nouveau_gpuobj *obj = NULL;
62 int ret;
63
64 ret = nouveau_gpuobj_new(dev, dev_priv->evo, 6*4, 32, 0, &obj);
65 if (ret)
66 return ret;
67 obj->engine = NVOBJ_ENGINE_DISPLAY;
68
69 nv_wo32(obj, 0, (tile_flags << 22) | (magic_flags << 16) | class);
70 nv_wo32(obj, 4, limit);
71 nv_wo32(obj, 8, offset);
72 nv_wo32(obj, 12, 0x00000000);
73 nv_wo32(obj, 16, 0x00000000);
74 nv_wo32(obj, 20, flags5);
75 dev_priv->engine.instmem.flush(dev);
76
77 ret = nouveau_ramht_insert(evo, name, obj);
78 nouveau_gpuobj_ref(NULL, &obj);
79 if (ret) {
80 return ret;
81 }
82
83 return 0;
84}
85
86static int
87nv50_evo_channel_new(struct drm_device *dev, struct nouveau_channel **pevo)
88{
89 struct drm_nouveau_private *dev_priv = dev->dev_private;
90 struct nouveau_channel *evo;
91 int ret;
92
93 evo = kzalloc(sizeof(struct nouveau_channel), GFP_KERNEL);
94 if (!evo)
95 return -ENOMEM;
96 *pevo = evo;
97
98 for (evo->id = 0; evo->id < 5; evo->id++) {
99 if (dev_priv->evo_alloc & (1 << evo->id))
100 continue;
101
102 dev_priv->evo_alloc |= (1 << evo->id);
103 break;
104 }
105
106 if (evo->id == 5) {
107 kfree(evo);
108 return -ENODEV;
109 }
110
111 evo->dev = dev;
112 evo->user_get = 4;
113 evo->user_put = 0;
114
115 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0,
116 false, true, &evo->pushbuf_bo);
117 if (ret == 0)
118 ret = nouveau_bo_pin(evo->pushbuf_bo, TTM_PL_FLAG_VRAM);
119 if (ret) {
120 NV_ERROR(dev, "Error creating EVO DMA push buffer: %d\n", ret);
121 nv50_evo_channel_del(pevo);
122 return ret;
123 }
124
125 ret = nouveau_bo_map(evo->pushbuf_bo);
126 if (ret) {
127 NV_ERROR(dev, "Error mapping EVO DMA push buffer: %d\n", ret);
128 nv50_evo_channel_del(pevo);
129 return ret;
130 }
131
132 evo->user = ioremap(pci_resource_start(dev->pdev, 0) +
133 NV50_PDISPLAY_USER(evo->id), PAGE_SIZE);
134 if (!evo->user) {
135 NV_ERROR(dev, "Error mapping EVO control regs.\n");
136 nv50_evo_channel_del(pevo);
137 return -ENOMEM;
138 }
139
140 /* bind primary evo channel's ramht to the channel */
141 if (dev_priv->evo && evo != dev_priv->evo)
142 nouveau_ramht_ref(dev_priv->evo->ramht, &evo->ramht, NULL);
143
144 return 0;
145}
146
147static int
148nv50_evo_channel_init(struct nouveau_channel *evo)
149{
150 struct drm_device *dev = evo->dev;
151 int id = evo->id, ret, i;
152 u64 pushbuf = evo->pushbuf_bo->bo.mem.start << PAGE_SHIFT;
153 u32 tmp;
154
155 tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
156 if ((tmp & 0x009f0000) == 0x00020000)
157 nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00800000);
158
159 tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
160 if ((tmp & 0x003f0000) == 0x00030000)
161 nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00600000);
162
163 /* initialise fifo */
164 nv_wr32(dev, NV50_PDISPLAY_EVO_DMA_CB(id), pushbuf >> 8 |
165 NV50_PDISPLAY_EVO_DMA_CB_LOCATION_VRAM |
166 NV50_PDISPLAY_EVO_DMA_CB_VALID);
167 nv_wr32(dev, NV50_PDISPLAY_EVO_UNK2(id), 0x00010000);
168 nv_wr32(dev, NV50_PDISPLAY_EVO_HASH_TAG(id), id);
169 nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), NV50_PDISPLAY_EVO_CTRL_DMA,
170 NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
171
172 nv_wr32(dev, NV50_PDISPLAY_USER_PUT(id), 0x00000000);
173 nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x01000003 |
174 NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
175 if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x80000000, 0x00000000)) {
176 NV_ERROR(dev, "EvoCh %d init timeout: 0x%08x\n", id,
177 nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
178 return -EBUSY;
179 }
180
181 /* enable error reporting on the channel */
182 nv_mask(dev, 0x610028, 0x00000000, 0x00010001 << id);
183
184 evo->dma.max = (4096/4) - 2;
185 evo->dma.put = 0;
186 evo->dma.cur = evo->dma.put;
187 evo->dma.free = evo->dma.max - evo->dma.cur;
188
189 ret = RING_SPACE(evo, NOUVEAU_DMA_SKIPS);
190 if (ret)
191 return ret;
192
193 for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
194 OUT_RING(evo, 0);
195
196 return 0;
197}
198
199static void
200nv50_evo_channel_fini(struct nouveau_channel *evo)
201{
202 struct drm_device *dev = evo->dev;
203 int id = evo->id;
204
205 nv_mask(dev, 0x610028, 0x00010001 << id, 0x00000000);
206 nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00001010, 0x00001000);
207 nv_wr32(dev, NV50_PDISPLAY_INTR_0, (1 << id));
208 nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00000003, 0x00000000);
209 if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x001e0000, 0x00000000)) {
210 NV_ERROR(dev, "EvoCh %d takedown timeout: 0x%08x\n", id,
211 nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
212 }
213}
214
215static int
216nv50_evo_create(struct drm_device *dev)
217{
218 struct drm_nouveau_private *dev_priv = dev->dev_private;
219 struct nouveau_gpuobj *ramht = NULL;
220 struct nouveau_channel *evo;
221 int ret;
222
223 /* create primary evo channel, the one we use for modesetting
224 * purporses
225 */
226 ret = nv50_evo_channel_new(dev, &dev_priv->evo);
227 if (ret)
228 return ret;
229 evo = dev_priv->evo;
230
231 /* setup object management on it, any other evo channel will
232 * use this also as there's no per-channel support on the
233 * hardware
234 */
235 ret = nouveau_gpuobj_new(dev, NULL, 32768, 65536,
236 NVOBJ_FLAG_ZERO_ALLOC, &evo->ramin);
237 if (ret) {
238 NV_ERROR(dev, "Error allocating EVO channel memory: %d\n", ret);
239 nv50_evo_channel_del(&dev_priv->evo);
240 return ret;
241 }
242
243 ret = drm_mm_init(&evo->ramin_heap, 0, 32768);
244 if (ret) {
245 NV_ERROR(dev, "Error initialising EVO PRAMIN heap: %d\n", ret);
246 nv50_evo_channel_del(&dev_priv->evo);
247 return ret;
248 }
249
250 ret = nouveau_gpuobj_new(dev, evo, 4096, 16, 0, &ramht);
251 if (ret) {
252 NV_ERROR(dev, "Unable to allocate EVO RAMHT: %d\n", ret);
253 nv50_evo_channel_del(&dev_priv->evo);
254 return ret;
255 }
256
257 ret = nouveau_ramht_new(dev, ramht, &evo->ramht);
258 nouveau_gpuobj_ref(NULL, &ramht);
259 if (ret) {
260 nv50_evo_channel_del(&dev_priv->evo);
261 return ret;
262 }
263
264 /* create some default objects for the scanout memtypes we support */
265 if (dev_priv->card_type >= NV_C0) {
266 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoFB32, 0xfe, 0x19,
267 0, 0xffffffff, 0x00000000);
268 if (ret) {
269 nv50_evo_channel_del(&dev_priv->evo);
270 return ret;
271 }
272
273 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoVRAM, 0, 0x19,
274 0, dev_priv->vram_size, 0x00020000);
275 if (ret) {
276 nv50_evo_channel_del(&dev_priv->evo);
277 return ret;
278 }
279
280 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoVRAM_LP, 0, 0x19,
281 0, dev_priv->vram_size, 0x00000000);
282 if (ret) {
283 nv50_evo_channel_del(&dev_priv->evo);
284 return ret;
285 }
286 } else
287 if (dev_priv->chipset != 0x50) {
288 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoFB16, 0x70, 0x19,
289 0, 0xffffffff, 0x00010000);
290 if (ret) {
291 nv50_evo_channel_del(&dev_priv->evo);
292 return ret;
293 }
294
295
296 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoFB32, 0x7a, 0x19,
297 0, 0xffffffff, 0x00010000);
298 if (ret) {
299 nv50_evo_channel_del(&dev_priv->evo);
300 return ret;
301 }
302
303 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoVRAM, 0, 0x19,
304 0, dev_priv->vram_size, 0x00010000);
305 if (ret) {
306 nv50_evo_channel_del(&dev_priv->evo);
307 return ret;
308 }
309
310 ret = nv50_evo_dmaobj_new(evo, 0x3d, NvEvoVRAM_LP, 0, 0x19,
311 0, dev_priv->vram_size, 0x00010000);
312 if (ret) {
313 nv50_evo_channel_del(&dev_priv->evo);
314 return ret;
315 }
316 }
317
318 return 0;
319}
320
321int
322nv50_evo_init(struct drm_device *dev)
323{
324 struct drm_nouveau_private *dev_priv = dev->dev_private;
325 int ret;
326
327 if (!dev_priv->evo) {
328 ret = nv50_evo_create(dev);
329 if (ret)
330 return ret;
331 }
332
333 return nv50_evo_channel_init(dev_priv->evo);
334}
335
336void
337nv50_evo_fini(struct drm_device *dev)
338{
339 struct drm_nouveau_private *dev_priv = dev->dev_private;
340
341 if (dev_priv->evo) {
342 nv50_evo_channel_fini(dev_priv->evo);
343 nv50_evo_channel_del(&dev_priv->evo);
344 }
345}
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.h b/drivers/gpu/drm/nouveau/nv50_evo.h
index aae13343bcec..aa4f0d3cea8e 100644
--- a/drivers/gpu/drm/nouveau/nv50_evo.h
+++ b/drivers/gpu/drm/nouveau/nv50_evo.h
@@ -24,6 +24,15 @@
24 * 24 *
25 */ 25 */
26 26
27#ifndef __NV50_EVO_H__
28#define __NV50_EVO_H__
29
30int nv50_evo_init(struct drm_device *dev);
31void nv50_evo_fini(struct drm_device *dev);
32int nv50_evo_dmaobj_new(struct nouveau_channel *, u32 class, u32 name,
33 u32 tile_flags, u32 magic_flags,
34 u32 offset, u32 limit);
35
27#define NV50_EVO_UPDATE 0x00000080 36#define NV50_EVO_UPDATE 0x00000080
28#define NV50_EVO_UNK84 0x00000084 37#define NV50_EVO_UNK84 0x00000084
29#define NV50_EVO_UNK84_NOTIFY 0x40000000 38#define NV50_EVO_UNK84_NOTIFY 0x40000000
@@ -111,3 +120,4 @@
111#define NV50_EVO_CRTC_SCALE_RES1 0x000008d8 120#define NV50_EVO_CRTC_SCALE_RES1 0x000008d8
112#define NV50_EVO_CRTC_SCALE_RES2 0x000008dc 121#define NV50_EVO_CRTC_SCALE_RES2 0x000008dc
113 122
123#endif
diff --git a/drivers/gpu/drm/nouveau/nv50_fb.c b/drivers/gpu/drm/nouveau/nv50_fb.c
index cd1988b15d2c..50290dea0ac4 100644
--- a/drivers/gpu/drm/nouveau/nv50_fb.c
+++ b/drivers/gpu/drm/nouveau/nv50_fb.c
@@ -3,30 +3,75 @@
3#include "nouveau_drv.h" 3#include "nouveau_drv.h"
4#include "nouveau_drm.h" 4#include "nouveau_drm.h"
5 5
6struct nv50_fb_priv {
7 struct page *r100c08_page;
8 dma_addr_t r100c08;
9};
10
11static int
12nv50_fb_create(struct drm_device *dev)
13{
14 struct drm_nouveau_private *dev_priv = dev->dev_private;
15 struct nv50_fb_priv *priv;
16
17 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
18 if (!priv)
19 return -ENOMEM;
20
21 priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
22 if (!priv->r100c08_page) {
23 kfree(priv);
24 return -ENOMEM;
25 }
26
27 priv->r100c08 = pci_map_page(dev->pdev, priv->r100c08_page, 0,
28 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
29 if (pci_dma_mapping_error(dev->pdev, priv->r100c08)) {
30 __free_page(priv->r100c08_page);
31 kfree(priv);
32 return -EFAULT;
33 }
34
35 dev_priv->engine.fb.priv = priv;
36 return 0;
37}
38
6int 39int
7nv50_fb_init(struct drm_device *dev) 40nv50_fb_init(struct drm_device *dev)
8{ 41{
9 struct drm_nouveau_private *dev_priv = dev->dev_private; 42 struct drm_nouveau_private *dev_priv = dev->dev_private;
43 struct nv50_fb_priv *priv;
44 int ret;
45
46 if (!dev_priv->engine.fb.priv) {
47 ret = nv50_fb_create(dev);
48 if (ret)
49 return ret;
50 }
51 priv = dev_priv->engine.fb.priv;
10 52
11 /* Not a clue what this is exactly. Without pointing it at a 53 /* Not a clue what this is exactly. Without pointing it at a
12 * scratch page, VRAM->GART blits with M2MF (as in DDX DFS) 54 * scratch page, VRAM->GART blits with M2MF (as in DDX DFS)
13 * cause IOMMU "read from address 0" errors (rh#561267) 55 * cause IOMMU "read from address 0" errors (rh#561267)
14 */ 56 */
15 nv_wr32(dev, 0x100c08, dev_priv->gart_info.sg_dummy_bus >> 8); 57 nv_wr32(dev, 0x100c08, priv->r100c08 >> 8);
16 58
17 /* This is needed to get meaningful information from 100c90 59 /* This is needed to get meaningful information from 100c90
18 * on traps. No idea what these values mean exactly. */ 60 * on traps. No idea what these values mean exactly. */
19 switch (dev_priv->chipset) { 61 switch (dev_priv->chipset) {
20 case 0x50: 62 case 0x50:
21 nv_wr32(dev, 0x100c90, 0x0707ff); 63 nv_wr32(dev, 0x100c90, 0x000707ff);
22 break; 64 break;
23 case 0xa3: 65 case 0xa3:
24 case 0xa5: 66 case 0xa5:
25 case 0xa8: 67 case 0xa8:
26 nv_wr32(dev, 0x100c90, 0x0d0fff); 68 nv_wr32(dev, 0x100c90, 0x000d0fff);
69 break;
70 case 0xaf:
71 nv_wr32(dev, 0x100c90, 0x089d1fff);
27 break; 72 break;
28 default: 73 default:
29 nv_wr32(dev, 0x100c90, 0x1d07ff); 74 nv_wr32(dev, 0x100c90, 0x001d07ff);
30 break; 75 break;
31 } 76 }
32 77
@@ -36,12 +81,25 @@ nv50_fb_init(struct drm_device *dev)
36void 81void
37nv50_fb_takedown(struct drm_device *dev) 82nv50_fb_takedown(struct drm_device *dev)
38{ 83{
84 struct drm_nouveau_private *dev_priv = dev->dev_private;
85 struct nv50_fb_priv *priv;
86
87 priv = dev_priv->engine.fb.priv;
88 if (!priv)
89 return;
90 dev_priv->engine.fb.priv = NULL;
91
92 pci_unmap_page(dev->pdev, priv->r100c08, PAGE_SIZE,
93 PCI_DMA_BIDIRECTIONAL);
94 __free_page(priv->r100c08_page);
95 kfree(priv);
39} 96}
40 97
41void 98void
42nv50_fb_vm_trap(struct drm_device *dev, int display, const char *name) 99nv50_fb_vm_trap(struct drm_device *dev, int display, const char *name)
43{ 100{
44 struct drm_nouveau_private *dev_priv = dev->dev_private; 101 struct drm_nouveau_private *dev_priv = dev->dev_private;
102 unsigned long flags;
45 u32 trap[6], idx, chinst; 103 u32 trap[6], idx, chinst;
46 int i, ch; 104 int i, ch;
47 105
@@ -60,8 +118,10 @@ nv50_fb_vm_trap(struct drm_device *dev, int display, const char *name)
60 return; 118 return;
61 119
62 chinst = (trap[2] << 16) | trap[1]; 120 chinst = (trap[2] << 16) | trap[1];
121
122 spin_lock_irqsave(&dev_priv->channels.lock, flags);
63 for (ch = 0; ch < dev_priv->engine.fifo.channels; ch++) { 123 for (ch = 0; ch < dev_priv->engine.fifo.channels; ch++) {
64 struct nouveau_channel *chan = dev_priv->fifos[ch]; 124 struct nouveau_channel *chan = dev_priv->channels.ptr[ch];
65 125
66 if (!chan || !chan->ramin) 126 if (!chan || !chan->ramin)
67 continue; 127 continue;
@@ -69,6 +129,7 @@ nv50_fb_vm_trap(struct drm_device *dev, int display, const char *name)
69 if (chinst == chan->ramin->vinst >> 12) 129 if (chinst == chan->ramin->vinst >> 12)
70 break; 130 break;
71 } 131 }
132 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
72 133
73 NV_INFO(dev, "%s - VM: Trapped %s at %02x%04x%04x status %08x " 134 NV_INFO(dev, "%s - VM: Trapped %s at %02x%04x%04x status %08x "
74 "channel %d (0x%08x)\n", 135 "channel %d (0x%08x)\n",
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 6dcf048eddbc..791ded1c5c6d 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -1,29 +1,46 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
1#include "drmP.h" 25#include "drmP.h"
2#include "nouveau_drv.h" 26#include "nouveau_drv.h"
3#include "nouveau_dma.h" 27#include "nouveau_dma.h"
4#include "nouveau_ramht.h" 28#include "nouveau_ramht.h"
5#include "nouveau_fbcon.h" 29#include "nouveau_fbcon.h"
30#include "nouveau_mm.h"
6 31
7void 32int
8nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) 33nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
9{ 34{
10 struct nouveau_fbdev *nfbdev = info->par; 35 struct nouveau_fbdev *nfbdev = info->par;
11 struct drm_device *dev = nfbdev->dev; 36 struct drm_device *dev = nfbdev->dev;
12 struct drm_nouveau_private *dev_priv = dev->dev_private; 37 struct drm_nouveau_private *dev_priv = dev->dev_private;
13 struct nouveau_channel *chan = dev_priv->channel; 38 struct nouveau_channel *chan = dev_priv->channel;
39 int ret;
14 40
15 if (info->state != FBINFO_STATE_RUNNING) 41 ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
16 return; 42 if (ret)
17 43 return ret;
18 if (!(info->flags & FBINFO_HWACCEL_DISABLED) &&
19 RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11)) {
20 nouveau_fbcon_gpu_lockup(info);
21 }
22
23 if (info->flags & FBINFO_HWACCEL_DISABLED) {
24 cfb_fillrect(info, rect);
25 return;
26 }
27 44
28 if (rect->rop != ROP_COPY) { 45 if (rect->rop != ROP_COPY) {
29 BEGIN_RING(chan, NvSub2D, 0x02ac, 1); 46 BEGIN_RING(chan, NvSub2D, 0x02ac, 1);
@@ -45,27 +62,21 @@ nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
45 OUT_RING(chan, 3); 62 OUT_RING(chan, 3);
46 } 63 }
47 FIRE_RING(chan); 64 FIRE_RING(chan);
65 return 0;
48} 66}
49 67
50void 68int
51nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) 69nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
52{ 70{
53 struct nouveau_fbdev *nfbdev = info->par; 71 struct nouveau_fbdev *nfbdev = info->par;
54 struct drm_device *dev = nfbdev->dev; 72 struct drm_device *dev = nfbdev->dev;
55 struct drm_nouveau_private *dev_priv = dev->dev_private; 73 struct drm_nouveau_private *dev_priv = dev->dev_private;
56 struct nouveau_channel *chan = dev_priv->channel; 74 struct nouveau_channel *chan = dev_priv->channel;
75 int ret;
57 76
58 if (info->state != FBINFO_STATE_RUNNING) 77 ret = RING_SPACE(chan, 12);
59 return; 78 if (ret)
60 79 return ret;
61 if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 12)) {
62 nouveau_fbcon_gpu_lockup(info);
63 }
64
65 if (info->flags & FBINFO_HWACCEL_DISABLED) {
66 cfb_copyarea(info, region);
67 return;
68 }
69 80
70 BEGIN_RING(chan, NvSub2D, 0x0110, 1); 81 BEGIN_RING(chan, NvSub2D, 0x0110, 1);
71 OUT_RING(chan, 0); 82 OUT_RING(chan, 0);
@@ -80,9 +91,10 @@ nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
80 OUT_RING(chan, 0); 91 OUT_RING(chan, 0);
81 OUT_RING(chan, region->sy); 92 OUT_RING(chan, region->sy);
82 FIRE_RING(chan); 93 FIRE_RING(chan);
94 return 0;
83} 95}
84 96
85void 97int
86nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) 98nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
87{ 99{
88 struct nouveau_fbdev *nfbdev = info->par; 100 struct nouveau_fbdev *nfbdev = info->par;
@@ -92,23 +104,14 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
92 uint32_t width, dwords, *data = (uint32_t *)image->data; 104 uint32_t width, dwords, *data = (uint32_t *)image->data;
93 uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel)); 105 uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
94 uint32_t *palette = info->pseudo_palette; 106 uint32_t *palette = info->pseudo_palette;
107 int ret;
95 108
96 if (info->state != FBINFO_STATE_RUNNING) 109 if (image->depth != 1)
97 return; 110 return -ENODEV;
98
99 if (image->depth != 1) {
100 cfb_imageblit(info, image);
101 return;
102 }
103 111
104 if (!(info->flags & FBINFO_HWACCEL_DISABLED) && RING_SPACE(chan, 11)) { 112 ret = RING_SPACE(chan, 11);
105 nouveau_fbcon_gpu_lockup(info); 113 if (ret)
106 } 114 return ret;
107
108 if (info->flags & FBINFO_HWACCEL_DISABLED) {
109 cfb_imageblit(info, image);
110 return;
111 }
112 115
113 width = ALIGN(image->width, 32); 116 width = ALIGN(image->width, 32);
114 dwords = (width * image->height) >> 5; 117 dwords = (width * image->height) >> 5;
@@ -134,11 +137,9 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
134 while (dwords) { 137 while (dwords) {
135 int push = dwords > 2047 ? 2047 : dwords; 138 int push = dwords > 2047 ? 2047 : dwords;
136 139
137 if (RING_SPACE(chan, push + 1)) { 140 ret = RING_SPACE(chan, push + 1);
138 nouveau_fbcon_gpu_lockup(info); 141 if (ret)
139 cfb_imageblit(info, image); 142 return ret;
140 return;
141 }
142 143
143 dwords -= push; 144 dwords -= push;
144 145
@@ -148,6 +149,7 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
148 } 149 }
149 150
150 FIRE_RING(chan); 151 FIRE_RING(chan);
152 return 0;
151} 153}
152 154
153int 155int
@@ -157,12 +159,9 @@ nv50_fbcon_accel_init(struct fb_info *info)
157 struct drm_device *dev = nfbdev->dev; 159 struct drm_device *dev = nfbdev->dev;
158 struct drm_nouveau_private *dev_priv = dev->dev_private; 160 struct drm_nouveau_private *dev_priv = dev->dev_private;
159 struct nouveau_channel *chan = dev_priv->channel; 161 struct nouveau_channel *chan = dev_priv->channel;
160 struct nouveau_gpuobj *eng2d = NULL; 162 struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
161 uint64_t fb;
162 int ret, format; 163 int ret, format;
163 164
164 fb = info->fix.smem_start - dev_priv->fb_phys + dev_priv->vm_vram_base;
165
166 switch (info->var.bits_per_pixel) { 165 switch (info->var.bits_per_pixel) {
167 case 8: 166 case 8:
168 format = 0xf3; 167 format = 0xf3;
@@ -190,12 +189,7 @@ nv50_fbcon_accel_init(struct fb_info *info)
190 return -EINVAL; 189 return -EINVAL;
191 } 190 }
192 191
193 ret = nouveau_gpuobj_gr_new(dev_priv->channel, 0x502d, &eng2d); 192 ret = nouveau_gpuobj_gr_new(dev_priv->channel, Nv2D, 0x502d);
194 if (ret)
195 return ret;
196
197 ret = nouveau_ramht_insert(dev_priv->channel, Nv2D, eng2d);
198 nouveau_gpuobj_ref(NULL, &eng2d);
199 if (ret) 193 if (ret)
200 return ret; 194 return ret;
201 195
@@ -253,8 +247,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
253 OUT_RING(chan, info->fix.line_length); 247 OUT_RING(chan, info->fix.line_length);
254 OUT_RING(chan, info->var.xres_virtual); 248 OUT_RING(chan, info->var.xres_virtual);
255 OUT_RING(chan, info->var.yres_virtual); 249 OUT_RING(chan, info->var.yres_virtual);
256 OUT_RING(chan, upper_32_bits(fb)); 250 OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
257 OUT_RING(chan, lower_32_bits(fb)); 251 OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
258 BEGIN_RING(chan, NvSub2D, 0x0230, 2); 252 BEGIN_RING(chan, NvSub2D, 0x0230, 2);
259 OUT_RING(chan, format); 253 OUT_RING(chan, format);
260 OUT_RING(chan, 1); 254 OUT_RING(chan, 1);
@@ -262,8 +256,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
262 OUT_RING(chan, info->fix.line_length); 256 OUT_RING(chan, info->fix.line_length);
263 OUT_RING(chan, info->var.xres_virtual); 257 OUT_RING(chan, info->var.xres_virtual);
264 OUT_RING(chan, info->var.yres_virtual); 258 OUT_RING(chan, info->var.yres_virtual);
265 OUT_RING(chan, upper_32_bits(fb)); 259 OUT_RING(chan, upper_32_bits(nvbo->vma.offset));
266 OUT_RING(chan, lower_32_bits(fb)); 260 OUT_RING(chan, lower_32_bits(nvbo->vma.offset));
267 261
268 return 0; 262 return 0;
269} 263}
diff --git a/drivers/gpu/drm/nouveau/nv50_fifo.c b/drivers/gpu/drm/nouveau/nv50_fifo.c
index 1da65bd60c10..8dd04c5dac67 100644
--- a/drivers/gpu/drm/nouveau/nv50_fifo.c
+++ b/drivers/gpu/drm/nouveau/nv50_fifo.c
@@ -28,6 +28,7 @@
28#include "drm.h" 28#include "drm.h"
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_ramht.h" 30#include "nouveau_ramht.h"
31#include "nouveau_vm.h"
31 32
32static void 33static void
33nv50_fifo_playlist_update(struct drm_device *dev) 34nv50_fifo_playlist_update(struct drm_device *dev)
@@ -44,7 +45,8 @@ nv50_fifo_playlist_update(struct drm_device *dev)
44 45
45 /* We never schedule channel 0 or 127 */ 46 /* We never schedule channel 0 or 127 */
46 for (i = 1, nr = 0; i < 127; i++) { 47 for (i = 1, nr = 0; i < 127; i++) {
47 if (dev_priv->fifos[i] && dev_priv->fifos[i]->ramfc) { 48 if (dev_priv->channels.ptr[i] &&
49 dev_priv->channels.ptr[i]->ramfc) {
48 nv_wo32(cur, (nr * 4), i); 50 nv_wo32(cur, (nr * 4), i);
49 nr++; 51 nr++;
50 } 52 }
@@ -60,7 +62,7 @@ static void
60nv50_fifo_channel_enable(struct drm_device *dev, int channel) 62nv50_fifo_channel_enable(struct drm_device *dev, int channel)
61{ 63{
62 struct drm_nouveau_private *dev_priv = dev->dev_private; 64 struct drm_nouveau_private *dev_priv = dev->dev_private;
63 struct nouveau_channel *chan = dev_priv->fifos[channel]; 65 struct nouveau_channel *chan = dev_priv->channels.ptr[channel];
64 uint32_t inst; 66 uint32_t inst;
65 67
66 NV_DEBUG(dev, "ch%d\n", channel); 68 NV_DEBUG(dev, "ch%d\n", channel);
@@ -105,6 +107,7 @@ nv50_fifo_init_intr(struct drm_device *dev)
105{ 107{
106 NV_DEBUG(dev, "\n"); 108 NV_DEBUG(dev, "\n");
107 109
110 nouveau_irq_register(dev, 8, nv04_fifo_isr);
108 nv_wr32(dev, NV03_PFIFO_INTR_0, 0xFFFFFFFF); 111 nv_wr32(dev, NV03_PFIFO_INTR_0, 0xFFFFFFFF);
109 nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xFFFFFFFF); 112 nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xFFFFFFFF);
110} 113}
@@ -118,7 +121,7 @@ nv50_fifo_init_context_table(struct drm_device *dev)
118 NV_DEBUG(dev, "\n"); 121 NV_DEBUG(dev, "\n");
119 122
120 for (i = 0; i < NV50_PFIFO_CTX_TABLE__SIZE; i++) { 123 for (i = 0; i < NV50_PFIFO_CTX_TABLE__SIZE; i++) {
121 if (dev_priv->fifos[i]) 124 if (dev_priv->channels.ptr[i])
122 nv50_fifo_channel_enable(dev, i); 125 nv50_fifo_channel_enable(dev, i);
123 else 126 else
124 nv50_fifo_channel_disable(dev, i); 127 nv50_fifo_channel_disable(dev, i);
@@ -206,6 +209,9 @@ nv50_fifo_takedown(struct drm_device *dev)
206 if (!pfifo->playlist[0]) 209 if (!pfifo->playlist[0])
207 return; 210 return;
208 211
212 nv_wr32(dev, 0x2140, 0x00000000);
213 nouveau_irq_unregister(dev, 8);
214
209 nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]); 215 nouveau_gpuobj_ref(NULL, &pfifo->playlist[0]);
210 nouveau_gpuobj_ref(NULL, &pfifo->playlist[1]); 216 nouveau_gpuobj_ref(NULL, &pfifo->playlist[1]);
211} 217}
@@ -256,6 +262,11 @@ nv50_fifo_create_context(struct nouveau_channel *chan)
256 } 262 }
257 ramfc = chan->ramfc; 263 ramfc = chan->ramfc;
258 264
265 chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
266 NV50_USER(chan->id), PAGE_SIZE);
267 if (!chan->user)
268 return -ENOMEM;
269
259 spin_lock_irqsave(&dev_priv->context_switch_lock, flags); 270 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
260 271
261 nv_wo32(ramfc, 0x48, chan->pushbuf->cinst >> 4); 272 nv_wo32(ramfc, 0x48, chan->pushbuf->cinst >> 4);
@@ -291,10 +302,23 @@ void
291nv50_fifo_destroy_context(struct nouveau_channel *chan) 302nv50_fifo_destroy_context(struct nouveau_channel *chan)
292{ 303{
293 struct drm_device *dev = chan->dev; 304 struct drm_device *dev = chan->dev;
305 struct drm_nouveau_private *dev_priv = dev->dev_private;
306 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
294 struct nouveau_gpuobj *ramfc = NULL; 307 struct nouveau_gpuobj *ramfc = NULL;
308 unsigned long flags;
295 309
296 NV_DEBUG(dev, "ch%d\n", chan->id); 310 NV_DEBUG(dev, "ch%d\n", chan->id);
297 311
312 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
313 pfifo->reassign(dev, false);
314
315 /* Unload the context if it's the currently active one */
316 if (pfifo->channel_id(dev) == chan->id) {
317 pfifo->disable(dev);
318 pfifo->unload_context(dev);
319 pfifo->enable(dev);
320 }
321
298 /* This will ensure the channel is seen as disabled. */ 322 /* This will ensure the channel is seen as disabled. */
299 nouveau_gpuobj_ref(chan->ramfc, &ramfc); 323 nouveau_gpuobj_ref(chan->ramfc, &ramfc);
300 nouveau_gpuobj_ref(NULL, &chan->ramfc); 324 nouveau_gpuobj_ref(NULL, &chan->ramfc);
@@ -305,6 +329,14 @@ nv50_fifo_destroy_context(struct nouveau_channel *chan)
305 nv50_fifo_channel_disable(dev, 127); 329 nv50_fifo_channel_disable(dev, 127);
306 nv50_fifo_playlist_update(dev); 330 nv50_fifo_playlist_update(dev);
307 331
332 pfifo->reassign(dev, true);
333 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
334
335 /* Free the channel resources */
336 if (chan->user) {
337 iounmap(chan->user);
338 chan->user = NULL;
339 }
308 nouveau_gpuobj_ref(NULL, &ramfc); 340 nouveau_gpuobj_ref(NULL, &ramfc);
309 nouveau_gpuobj_ref(NULL, &chan->cache); 341 nouveau_gpuobj_ref(NULL, &chan->cache);
310} 342}
@@ -392,7 +424,7 @@ nv50_fifo_unload_context(struct drm_device *dev)
392 if (chid < 1 || chid >= dev_priv->engine.fifo.channels - 1) 424 if (chid < 1 || chid >= dev_priv->engine.fifo.channels - 1)
393 return 0; 425 return 0;
394 426
395 chan = dev_priv->fifos[chid]; 427 chan = dev_priv->channels.ptr[chid];
396 if (!chan) { 428 if (!chan) {
397 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid); 429 NV_ERROR(dev, "Inactive channel on PFIFO: %d\n", chid);
398 return -EINVAL; 430 return -EINVAL;
@@ -467,5 +499,5 @@ nv50_fifo_unload_context(struct drm_device *dev)
467void 499void
468nv50_fifo_tlb_flush(struct drm_device *dev) 500nv50_fifo_tlb_flush(struct drm_device *dev)
469{ 501{
470 nv50_vm_flush(dev, 5); 502 nv50_vm_flush_engine(dev, 5);
471} 503}
diff --git a/drivers/gpu/drm/nouveau/nv50_gpio.c b/drivers/gpu/drm/nouveau/nv50_gpio.c
index b2fab2bf3d61..6b149c0cc06d 100644
--- a/drivers/gpu/drm/nouveau/nv50_gpio.c
+++ b/drivers/gpu/drm/nouveau/nv50_gpio.c
@@ -26,6 +26,28 @@
26#include "nouveau_drv.h" 26#include "nouveau_drv.h"
27#include "nouveau_hw.h" 27#include "nouveau_hw.h"
28 28
29#include "nv50_display.h"
30
31static void nv50_gpio_isr(struct drm_device *dev);
32static void nv50_gpio_isr_bh(struct work_struct *work);
33
34struct nv50_gpio_priv {
35 struct list_head handlers;
36 spinlock_t lock;
37};
38
39struct nv50_gpio_handler {
40 struct drm_device *dev;
41 struct list_head head;
42 struct work_struct work;
43 bool inhibit;
44
45 struct dcb_gpio_entry *gpio;
46
47 void (*handler)(void *data, int state);
48 void *data;
49};
50
29static int 51static int
30nv50_gpio_location(struct dcb_gpio_entry *gpio, uint32_t *reg, uint32_t *shift) 52nv50_gpio_location(struct dcb_gpio_entry *gpio, uint32_t *reg, uint32_t *shift)
31{ 53{
@@ -75,29 +97,123 @@ nv50_gpio_set(struct drm_device *dev, enum dcb_gpio_tag tag, int state)
75 return 0; 97 return 0;
76} 98}
77 99
100int
101nv50_gpio_irq_register(struct drm_device *dev, enum dcb_gpio_tag tag,
102 void (*handler)(void *, int), void *data)
103{
104 struct drm_nouveau_private *dev_priv = dev->dev_private;
105 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
106 struct nv50_gpio_priv *priv = pgpio->priv;
107 struct nv50_gpio_handler *gpioh;
108 struct dcb_gpio_entry *gpio;
109 unsigned long flags;
110
111 gpio = nouveau_bios_gpio_entry(dev, tag);
112 if (!gpio)
113 return -ENOENT;
114
115 gpioh = kzalloc(sizeof(*gpioh), GFP_KERNEL);
116 if (!gpioh)
117 return -ENOMEM;
118
119 INIT_WORK(&gpioh->work, nv50_gpio_isr_bh);
120 gpioh->dev = dev;
121 gpioh->gpio = gpio;
122 gpioh->handler = handler;
123 gpioh->data = data;
124
125 spin_lock_irqsave(&priv->lock, flags);
126 list_add(&gpioh->head, &priv->handlers);
127 spin_unlock_irqrestore(&priv->lock, flags);
128 return 0;
129}
130
78void 131void
79nv50_gpio_irq_enable(struct drm_device *dev, enum dcb_gpio_tag tag, bool on) 132nv50_gpio_irq_unregister(struct drm_device *dev, enum dcb_gpio_tag tag,
133 void (*handler)(void *, int), void *data)
80{ 134{
135 struct drm_nouveau_private *dev_priv = dev->dev_private;
136 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
137 struct nv50_gpio_priv *priv = pgpio->priv;
138 struct nv50_gpio_handler *gpioh, *tmp;
81 struct dcb_gpio_entry *gpio; 139 struct dcb_gpio_entry *gpio;
82 u32 reg, mask; 140 unsigned long flags;
83 141
84 gpio = nouveau_bios_gpio_entry(dev, tag); 142 gpio = nouveau_bios_gpio_entry(dev, tag);
85 if (!gpio) { 143 if (!gpio)
86 NV_ERROR(dev, "gpio tag 0x%02x not found\n", tag);
87 return; 144 return;
145
146 spin_lock_irqsave(&priv->lock, flags);
147 list_for_each_entry_safe(gpioh, tmp, &priv->handlers, head) {
148 if (gpioh->gpio != gpio ||
149 gpioh->handler != handler ||
150 gpioh->data != data)
151 continue;
152 list_del(&gpioh->head);
153 kfree(gpioh);
88 } 154 }
155 spin_unlock_irqrestore(&priv->lock, flags);
156}
157
158bool
159nv50_gpio_irq_enable(struct drm_device *dev, enum dcb_gpio_tag tag, bool on)
160{
161 struct dcb_gpio_entry *gpio;
162 u32 reg, mask;
163
164 gpio = nouveau_bios_gpio_entry(dev, tag);
165 if (!gpio)
166 return false;
89 167
90 reg = gpio->line < 16 ? 0xe050 : 0xe070; 168 reg = gpio->line < 16 ? 0xe050 : 0xe070;
91 mask = 0x00010001 << (gpio->line & 0xf); 169 mask = 0x00010001 << (gpio->line & 0xf);
92 170
93 nv_wr32(dev, reg + 4, mask); 171 nv_wr32(dev, reg + 4, mask);
94 nv_mask(dev, reg + 0, mask, on ? mask : 0); 172 reg = nv_mask(dev, reg + 0, mask, on ? mask : 0);
173 return (reg & mask) == mask;
174}
175
176static int
177nv50_gpio_create(struct drm_device *dev)
178{
179 struct drm_nouveau_private *dev_priv = dev->dev_private;
180 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
181 struct nv50_gpio_priv *priv;
182
183 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
184 if (!priv)
185 return -ENOMEM;
186
187 INIT_LIST_HEAD(&priv->handlers);
188 spin_lock_init(&priv->lock);
189 pgpio->priv = priv;
190 return 0;
191}
192
193static void
194nv50_gpio_destroy(struct drm_device *dev)
195{
196 struct drm_nouveau_private *dev_priv = dev->dev_private;
197 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
198
199 kfree(pgpio->priv);
200 pgpio->priv = NULL;
95} 201}
96 202
97int 203int
98nv50_gpio_init(struct drm_device *dev) 204nv50_gpio_init(struct drm_device *dev)
99{ 205{
100 struct drm_nouveau_private *dev_priv = dev->dev_private; 206 struct drm_nouveau_private *dev_priv = dev->dev_private;
207 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
208 struct nv50_gpio_priv *priv;
209 int ret;
210
211 if (!pgpio->priv) {
212 ret = nv50_gpio_create(dev);
213 if (ret)
214 return ret;
215 }
216 priv = pgpio->priv;
101 217
102 /* disable, and ack any pending gpio interrupts */ 218 /* disable, and ack any pending gpio interrupts */
103 nv_wr32(dev, 0xe050, 0x00000000); 219 nv_wr32(dev, 0xe050, 0x00000000);
@@ -107,5 +223,77 @@ nv50_gpio_init(struct drm_device *dev)
107 nv_wr32(dev, 0xe074, 0xffffffff); 223 nv_wr32(dev, 0xe074, 0xffffffff);
108 } 224 }
109 225
226 nouveau_irq_register(dev, 21, nv50_gpio_isr);
110 return 0; 227 return 0;
111} 228}
229
230void
231nv50_gpio_fini(struct drm_device *dev)
232{
233 struct drm_nouveau_private *dev_priv = dev->dev_private;
234
235 nv_wr32(dev, 0xe050, 0x00000000);
236 if (dev_priv->chipset >= 0x90)
237 nv_wr32(dev, 0xe070, 0x00000000);
238 nouveau_irq_unregister(dev, 21);
239
240 nv50_gpio_destroy(dev);
241}
242
243static void
244nv50_gpio_isr_bh(struct work_struct *work)
245{
246 struct nv50_gpio_handler *gpioh =
247 container_of(work, struct nv50_gpio_handler, work);
248 struct drm_nouveau_private *dev_priv = gpioh->dev->dev_private;
249 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
250 struct nv50_gpio_priv *priv = pgpio->priv;
251 unsigned long flags;
252 int state;
253
254 state = pgpio->get(gpioh->dev, gpioh->gpio->tag);
255 if (state < 0)
256 return;
257
258 gpioh->handler(gpioh->data, state);
259
260 spin_lock_irqsave(&priv->lock, flags);
261 gpioh->inhibit = false;
262 spin_unlock_irqrestore(&priv->lock, flags);
263}
264
265static void
266nv50_gpio_isr(struct drm_device *dev)
267{
268 struct drm_nouveau_private *dev_priv = dev->dev_private;
269 struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
270 struct nv50_gpio_priv *priv = pgpio->priv;
271 struct nv50_gpio_handler *gpioh;
272 u32 intr0, intr1 = 0;
273 u32 hi, lo, ch;
274
275 intr0 = nv_rd32(dev, 0xe054) & nv_rd32(dev, 0xe050);
276 if (dev_priv->chipset >= 0x90)
277 intr1 = nv_rd32(dev, 0xe074) & nv_rd32(dev, 0xe070);
278
279 hi = (intr0 & 0x0000ffff) | (intr1 << 16);
280 lo = (intr0 >> 16) | (intr1 & 0xffff0000);
281 ch = hi | lo;
282
283 nv_wr32(dev, 0xe054, intr0);
284 if (dev_priv->chipset >= 0x90)
285 nv_wr32(dev, 0xe074, intr1);
286
287 spin_lock(&priv->lock);
288 list_for_each_entry(gpioh, &priv->handlers, head) {
289 if (!(ch & (1 << gpioh->gpio->line)))
290 continue;
291
292 if (gpioh->inhibit)
293 continue;
294 gpioh->inhibit = true;
295
296 queue_work(dev_priv->wq, &gpioh->work);
297 }
298 spin_unlock(&priv->lock);
299}
diff --git a/drivers/gpu/drm/nouveau/nv50_graph.c b/drivers/gpu/drm/nouveau/nv50_graph.c
index 8b669d0af610..2d7ea75a09d4 100644
--- a/drivers/gpu/drm/nouveau/nv50_graph.c
+++ b/drivers/gpu/drm/nouveau/nv50_graph.c
@@ -29,6 +29,12 @@
29#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_ramht.h" 30#include "nouveau_ramht.h"
31#include "nouveau_grctx.h" 31#include "nouveau_grctx.h"
32#include "nouveau_dma.h"
33#include "nouveau_vm.h"
34#include "nv50_evo.h"
35
36static int nv50_graph_register(struct drm_device *);
37static void nv50_graph_isr(struct drm_device *);
32 38
33static void 39static void
34nv50_graph_init_reset(struct drm_device *dev) 40nv50_graph_init_reset(struct drm_device *dev)
@@ -46,6 +52,7 @@ nv50_graph_init_intr(struct drm_device *dev)
46{ 52{
47 NV_DEBUG(dev, "\n"); 53 NV_DEBUG(dev, "\n");
48 54
55 nouveau_irq_register(dev, 12, nv50_graph_isr);
49 nv_wr32(dev, NV03_PGRAPH_INTR, 0xffffffff); 56 nv_wr32(dev, NV03_PGRAPH_INTR, 0xffffffff);
50 nv_wr32(dev, 0x400138, 0xffffffff); 57 nv_wr32(dev, 0x400138, 0xffffffff);
51 nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xffffffff); 58 nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xffffffff);
@@ -145,12 +152,15 @@ nv50_graph_init(struct drm_device *dev)
145 nv50_graph_init_reset(dev); 152 nv50_graph_init_reset(dev);
146 nv50_graph_init_regs__nv(dev); 153 nv50_graph_init_regs__nv(dev);
147 nv50_graph_init_regs(dev); 154 nv50_graph_init_regs(dev);
148 nv50_graph_init_intr(dev);
149 155
150 ret = nv50_graph_init_ctxctl(dev); 156 ret = nv50_graph_init_ctxctl(dev);
151 if (ret) 157 if (ret)
152 return ret; 158 return ret;
153 159
160 ret = nv50_graph_register(dev);
161 if (ret)
162 return ret;
163 nv50_graph_init_intr(dev);
154 return 0; 164 return 0;
155} 165}
156 166
@@ -158,6 +168,8 @@ void
158nv50_graph_takedown(struct drm_device *dev) 168nv50_graph_takedown(struct drm_device *dev)
159{ 169{
160 NV_DEBUG(dev, "\n"); 170 NV_DEBUG(dev, "\n");
171 nv_wr32(dev, 0x40013c, 0x00000000);
172 nouveau_irq_unregister(dev, 12);
161} 173}
162 174
163void 175void
@@ -190,7 +202,7 @@ nv50_graph_channel(struct drm_device *dev)
190 inst = (inst & NV50_PGRAPH_CTXCTL_CUR_INSTANCE) << 12; 202 inst = (inst & NV50_PGRAPH_CTXCTL_CUR_INSTANCE) << 12;
191 203
192 for (i = 0; i < dev_priv->engine.fifo.channels; i++) { 204 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
193 struct nouveau_channel *chan = dev_priv->fifos[i]; 205 struct nouveau_channel *chan = dev_priv->channels.ptr[i];
194 206
195 if (chan && chan->ramin && chan->ramin->vinst == inst) 207 if (chan && chan->ramin && chan->ramin->vinst == inst)
196 return chan; 208 return chan;
@@ -211,7 +223,7 @@ nv50_graph_create_context(struct nouveau_channel *chan)
211 223
212 NV_DEBUG(dev, "ch%d\n", chan->id); 224 NV_DEBUG(dev, "ch%d\n", chan->id);
213 225
214 ret = nouveau_gpuobj_new(dev, chan, pgraph->grctx_size, 0x1000, 226 ret = nouveau_gpuobj_new(dev, chan, pgraph->grctx_size, 0,
215 NVOBJ_FLAG_ZERO_ALLOC | 227 NVOBJ_FLAG_ZERO_ALLOC |
216 NVOBJ_FLAG_ZERO_FREE, &chan->ramin_grctx); 228 NVOBJ_FLAG_ZERO_FREE, &chan->ramin_grctx);
217 if (ret) 229 if (ret)
@@ -234,6 +246,7 @@ nv50_graph_create_context(struct nouveau_channel *chan)
234 nv_wo32(chan->ramin_grctx, 0x00000, chan->ramin->vinst >> 12); 246 nv_wo32(chan->ramin_grctx, 0x00000, chan->ramin->vinst >> 12);
235 247
236 dev_priv->engine.instmem.flush(dev); 248 dev_priv->engine.instmem.flush(dev);
249 atomic_inc(&chan->vm->pgraph_refs);
237 return 0; 250 return 0;
238} 251}
239 252
@@ -242,18 +255,31 @@ nv50_graph_destroy_context(struct nouveau_channel *chan)
242{ 255{
243 struct drm_device *dev = chan->dev; 256 struct drm_device *dev = chan->dev;
244 struct drm_nouveau_private *dev_priv = dev->dev_private; 257 struct drm_nouveau_private *dev_priv = dev->dev_private;
258 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
245 int i, hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20; 259 int i, hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
260 unsigned long flags;
246 261
247 NV_DEBUG(dev, "ch%d\n", chan->id); 262 NV_DEBUG(dev, "ch%d\n", chan->id);
248 263
249 if (!chan->ramin) 264 if (!chan->ramin)
250 return; 265 return;
251 266
267 spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
268 pgraph->fifo_access(dev, false);
269
270 if (pgraph->channel(dev) == chan)
271 pgraph->unload_context(dev);
272
252 for (i = hdr; i < hdr + 24; i += 4) 273 for (i = hdr; i < hdr + 24; i += 4)
253 nv_wo32(chan->ramin, i, 0); 274 nv_wo32(chan->ramin, i, 0);
254 dev_priv->engine.instmem.flush(dev); 275 dev_priv->engine.instmem.flush(dev);
255 276
277 pgraph->fifo_access(dev, true);
278 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
279
256 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx); 280 nouveau_gpuobj_ref(NULL, &chan->ramin_grctx);
281
282 atomic_dec(&chan->vm->pgraph_refs);
257} 283}
258 284
259static int 285static int
@@ -306,7 +332,7 @@ nv50_graph_unload_context(struct drm_device *dev)
306 return 0; 332 return 0;
307} 333}
308 334
309void 335static void
310nv50_graph_context_switch(struct drm_device *dev) 336nv50_graph_context_switch(struct drm_device *dev)
311{ 337{
312 uint32_t inst; 338 uint32_t inst;
@@ -322,8 +348,8 @@ nv50_graph_context_switch(struct drm_device *dev)
322} 348}
323 349
324static int 350static int
325nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan, int grclass, 351nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan,
326 int mthd, uint32_t data) 352 u32 class, u32 mthd, u32 data)
327{ 353{
328 struct nouveau_gpuobj *gpuobj; 354 struct nouveau_gpuobj *gpuobj;
329 355
@@ -340,8 +366,8 @@ nv50_graph_nvsw_dma_vblsem(struct nouveau_channel *chan, int grclass,
340} 366}
341 367
342static int 368static int
343nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan, int grclass, 369nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan,
344 int mthd, uint32_t data) 370 u32 class, u32 mthd, u32 data)
345{ 371{
346 if (nouveau_notifier_offset(chan->nvsw.vblsem, &data)) 372 if (nouveau_notifier_offset(chan->nvsw.vblsem, &data))
347 return -ERANGE; 373 return -ERANGE;
@@ -351,16 +377,16 @@ nv50_graph_nvsw_vblsem_offset(struct nouveau_channel *chan, int grclass,
351} 377}
352 378
353static int 379static int
354nv50_graph_nvsw_vblsem_release_val(struct nouveau_channel *chan, int grclass, 380nv50_graph_nvsw_vblsem_release_val(struct nouveau_channel *chan,
355 int mthd, uint32_t data) 381 u32 class, u32 mthd, u32 data)
356{ 382{
357 chan->nvsw.vblsem_rval = data; 383 chan->nvsw.vblsem_rval = data;
358 return 0; 384 return 0;
359} 385}
360 386
361static int 387static int
362nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan, int grclass, 388nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan,
363 int mthd, uint32_t data) 389 u32 class, u32 mthd, u32 data)
364{ 390{
365 struct drm_device *dev = chan->dev; 391 struct drm_device *dev = chan->dev;
366 struct drm_nouveau_private *dev_priv = dev->dev_private; 392 struct drm_nouveau_private *dev_priv = dev->dev_private;
@@ -368,45 +394,85 @@ nv50_graph_nvsw_vblsem_release(struct nouveau_channel *chan, int grclass,
368 if (!chan->nvsw.vblsem || chan->nvsw.vblsem_offset == ~0 || data > 1) 394 if (!chan->nvsw.vblsem || chan->nvsw.vblsem_offset == ~0 || data > 1)
369 return -EINVAL; 395 return -EINVAL;
370 396
371 if (!(nv_rd32(dev, NV50_PDISPLAY_INTR_EN) & 397 drm_vblank_get(dev, data);
372 NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(data))) {
373 nv_wr32(dev, NV50_PDISPLAY_INTR_1,
374 NV50_PDISPLAY_INTR_1_VBLANK_CRTC_(data));
375 nv_wr32(dev, NV50_PDISPLAY_INTR_EN, nv_rd32(dev,
376 NV50_PDISPLAY_INTR_EN) |
377 NV50_PDISPLAY_INTR_EN_VBLANK_CRTC_(data));
378 }
379 398
399 chan->nvsw.vblsem_head = data;
380 list_add(&chan->nvsw.vbl_wait, &dev_priv->vbl_waiting); 400 list_add(&chan->nvsw.vbl_wait, &dev_priv->vbl_waiting);
401
381 return 0; 402 return 0;
382} 403}
383 404
384static struct nouveau_pgraph_object_method nv50_graph_nvsw_methods[] = { 405static int
385 { 0x018c, nv50_graph_nvsw_dma_vblsem }, 406nv50_graph_nvsw_mthd_page_flip(struct nouveau_channel *chan,
386 { 0x0400, nv50_graph_nvsw_vblsem_offset }, 407 u32 class, u32 mthd, u32 data)
387 { 0x0404, nv50_graph_nvsw_vblsem_release_val }, 408{
388 { 0x0408, nv50_graph_nvsw_vblsem_release }, 409 struct nouveau_page_flip_state s;
389 {}
390};
391 410
392struct nouveau_pgraph_object_class nv50_graph_grclass[] = { 411 if (!nouveau_finish_page_flip(chan, &s)) {
393 { 0x506e, true, nv50_graph_nvsw_methods }, /* nvsw */ 412 /* XXX - Do something here */
394 { 0x0030, false, NULL }, /* null */ 413 }
395 { 0x5039, false, NULL }, /* m2mf */ 414
396 { 0x502d, false, NULL }, /* 2d */ 415 return 0;
397 { 0x50c0, false, NULL }, /* compute */ 416}
398 { 0x85c0, false, NULL }, /* compute (nva3, nva5, nva8) */ 417
399 { 0x5097, false, NULL }, /* tesla (nv50) */ 418static int
400 { 0x8297, false, NULL }, /* tesla (nv8x/nv9x) */ 419nv50_graph_register(struct drm_device *dev)
401 { 0x8397, false, NULL }, /* tesla (nva0, nvaa, nvac) */ 420{
402 { 0x8597, false, NULL }, /* tesla (nva3, nva5, nva8) */ 421 struct drm_nouveau_private *dev_priv = dev->dev_private;
403 {} 422
404}; 423 if (dev_priv->engine.graph.registered)
424 return 0;
425
426 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
427 NVOBJ_MTHD (dev, 0x506e, 0x018c, nv50_graph_nvsw_dma_vblsem);
428 NVOBJ_MTHD (dev, 0x506e, 0x0400, nv50_graph_nvsw_vblsem_offset);
429 NVOBJ_MTHD (dev, 0x506e, 0x0404, nv50_graph_nvsw_vblsem_release_val);
430 NVOBJ_MTHD (dev, 0x506e, 0x0408, nv50_graph_nvsw_vblsem_release);
431 NVOBJ_MTHD (dev, 0x506e, 0x0500, nv50_graph_nvsw_mthd_page_flip);
432
433 NVOBJ_CLASS(dev, 0x0030, GR); /* null */
434 NVOBJ_CLASS(dev, 0x5039, GR); /* m2mf */
435 NVOBJ_CLASS(dev, 0x502d, GR); /* 2d */
436
437 /* tesla */
438 if (dev_priv->chipset == 0x50)
439 NVOBJ_CLASS(dev, 0x5097, GR); /* tesla (nv50) */
440 else
441 if (dev_priv->chipset < 0xa0)
442 NVOBJ_CLASS(dev, 0x8297, GR); /* tesla (nv8x/nv9x) */
443 else {
444 switch (dev_priv->chipset) {
445 case 0xa0:
446 case 0xaa:
447 case 0xac:
448 NVOBJ_CLASS(dev, 0x8397, GR);
449 break;
450 case 0xa3:
451 case 0xa5:
452 case 0xa8:
453 NVOBJ_CLASS(dev, 0x8597, GR);
454 break;
455 case 0xaf:
456 NVOBJ_CLASS(dev, 0x8697, GR);
457 break;
458 }
459 }
460
461 /* compute */
462 NVOBJ_CLASS(dev, 0x50c0, GR);
463 if (dev_priv->chipset > 0xa0 &&
464 dev_priv->chipset != 0xaa &&
465 dev_priv->chipset != 0xac)
466 NVOBJ_CLASS(dev, 0x85c0, GR);
467
468 dev_priv->engine.graph.registered = true;
469 return 0;
470}
405 471
406void 472void
407nv50_graph_tlb_flush(struct drm_device *dev) 473nv50_graph_tlb_flush(struct drm_device *dev)
408{ 474{
409 nv50_vm_flush(dev, 0); 475 nv50_vm_flush_engine(dev, 0);
410} 476}
411 477
412void 478void
@@ -449,8 +515,535 @@ nv86_graph_tlb_flush(struct drm_device *dev)
449 nv_rd32(dev, 0x400384), nv_rd32(dev, 0x400388)); 515 nv_rd32(dev, 0x400384), nv_rd32(dev, 0x400388));
450 } 516 }
451 517
452 nv50_vm_flush(dev, 0); 518 nv50_vm_flush_engine(dev, 0);
453 519
454 nv_mask(dev, 0x400500, 0x00000001, 0x00000001); 520 nv_mask(dev, 0x400500, 0x00000001, 0x00000001);
455 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags); 521 spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
456} 522}
523
524static struct nouveau_enum nv50_mp_exec_error_names[] =
525{
526 { 3, "STACK_UNDERFLOW" },
527 { 4, "QUADON_ACTIVE" },
528 { 8, "TIMEOUT" },
529 { 0x10, "INVALID_OPCODE" },
530 { 0x40, "BREAKPOINT" },
531 {}
532};
533
534static struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
535 { 0x00000001, "NOTIFY" },
536 { 0x00000002, "IN" },
537 { 0x00000004, "OUT" },
538 {}
539};
540
541static struct nouveau_bitfield nv50_graph_trap_vfetch[] = {
542 { 0x00000001, "FAULT" },
543 {}
544};
545
546static struct nouveau_bitfield nv50_graph_trap_strmout[] = {
547 { 0x00000001, "FAULT" },
548 {}
549};
550
551static struct nouveau_bitfield nv50_graph_trap_ccache[] = {
552 { 0x00000001, "FAULT" },
553 {}
554};
555
556/* There must be a *lot* of these. Will take some time to gather them up. */
557struct nouveau_enum nv50_data_error_names[] = {
558 { 0x00000003, "INVALID_QUERY_OR_TEXTURE" },
559 { 0x00000004, "INVALID_VALUE" },
560 { 0x00000005, "INVALID_ENUM" },
561 { 0x00000008, "INVALID_OBJECT" },
562 { 0x00000009, "READ_ONLY_OBJECT" },
563 { 0x0000000a, "SUPERVISOR_OBJECT" },
564 { 0x0000000b, "INVALID_ADDRESS_ALIGNMENT" },
565 { 0x0000000c, "INVALID_BITFIELD" },
566 { 0x0000000d, "BEGIN_END_ACTIVE" },
567 { 0x0000000e, "SEMANTIC_COLOR_BACK_OVER_LIMIT" },
568 { 0x0000000f, "VIEWPORT_ID_NEEDS_GP" },
569 { 0x00000010, "RT_DOUBLE_BIND" },
570 { 0x00000011, "RT_TYPES_MISMATCH" },
571 { 0x00000012, "RT_LINEAR_WITH_ZETA" },
572 { 0x00000015, "FP_TOO_FEW_REGS" },
573 { 0x00000016, "ZETA_FORMAT_CSAA_MISMATCH" },
574 { 0x00000017, "RT_LINEAR_WITH_MSAA" },
575 { 0x00000018, "FP_INTERPOLANT_START_OVER_LIMIT" },
576 { 0x00000019, "SEMANTIC_LAYER_OVER_LIMIT" },
577 { 0x0000001a, "RT_INVALID_ALIGNMENT" },
578 { 0x0000001b, "SAMPLER_OVER_LIMIT" },
579 { 0x0000001c, "TEXTURE_OVER_LIMIT" },
580 { 0x0000001e, "GP_TOO_MANY_OUTPUTS" },
581 { 0x0000001f, "RT_BPP128_WITH_MS8" },
582 { 0x00000021, "Z_OUT_OF_BOUNDS" },
583 { 0x00000023, "XY_OUT_OF_BOUNDS" },
584 { 0x00000027, "CP_MORE_PARAMS_THAN_SHARED" },
585 { 0x00000028, "CP_NO_REG_SPACE_STRIPED" },
586 { 0x00000029, "CP_NO_REG_SPACE_PACKED" },
587 { 0x0000002a, "CP_NOT_ENOUGH_WARPS" },
588 { 0x0000002b, "CP_BLOCK_SIZE_MISMATCH" },
589 { 0x0000002c, "CP_NOT_ENOUGH_LOCAL_WARPS" },
590 { 0x0000002d, "CP_NOT_ENOUGH_STACK_WARPS" },
591 { 0x0000002e, "CP_NO_BLOCKDIM_LATCH" },
592 { 0x00000031, "ENG2D_FORMAT_MISMATCH" },
593 { 0x0000003f, "PRIMITIVE_ID_NEEDS_GP" },
594 { 0x00000044, "SEMANTIC_VIEWPORT_OVER_LIMIT" },
595 { 0x00000045, "SEMANTIC_COLOR_FRONT_OVER_LIMIT" },
596 { 0x00000046, "LAYER_ID_NEEDS_GP" },
597 { 0x00000047, "SEMANTIC_CLIP_OVER_LIMIT" },
598 { 0x00000048, "SEMANTIC_PTSZ_OVER_LIMIT" },
599 {}
600};
601
602static struct nouveau_bitfield nv50_graph_intr[] = {
603 { 0x00000001, "NOTIFY" },
604 { 0x00000002, "COMPUTE_QUERY" },
605 { 0x00000010, "ILLEGAL_MTHD" },
606 { 0x00000020, "ILLEGAL_CLASS" },
607 { 0x00000040, "DOUBLE_NOTIFY" },
608 { 0x00001000, "CONTEXT_SWITCH" },
609 { 0x00010000, "BUFFER_NOTIFY" },
610 { 0x00100000, "DATA_ERROR" },
611 { 0x00200000, "TRAP" },
612 { 0x01000000, "SINGLE_STEP" },
613 {}
614};
615
616static void
617nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
618{
619 struct drm_nouveau_private *dev_priv = dev->dev_private;
620 uint32_t units = nv_rd32(dev, 0x1540);
621 uint32_t addr, mp10, status, pc, oplow, ophigh;
622 int i;
623 int mps = 0;
624 for (i = 0; i < 4; i++) {
625 if (!(units & 1 << (i+24)))
626 continue;
627 if (dev_priv->chipset < 0xa0)
628 addr = 0x408200 + (tpid << 12) + (i << 7);
629 else
630 addr = 0x408100 + (tpid << 11) + (i << 7);
631 mp10 = nv_rd32(dev, addr + 0x10);
632 status = nv_rd32(dev, addr + 0x14);
633 if (!status)
634 continue;
635 if (display) {
636 nv_rd32(dev, addr + 0x20);
637 pc = nv_rd32(dev, addr + 0x24);
638 oplow = nv_rd32(dev, addr + 0x70);
639 ophigh= nv_rd32(dev, addr + 0x74);
640 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
641 "TP %d MP %d: ", tpid, i);
642 nouveau_enum_print(nv50_mp_exec_error_names, status);
643 printk(" at %06x warp %d, opcode %08x %08x\n",
644 pc&0xffffff, pc >> 24,
645 oplow, ophigh);
646 }
647 nv_wr32(dev, addr + 0x10, mp10);
648 nv_wr32(dev, addr + 0x14, 0);
649 mps++;
650 }
651 if (!mps && display)
652 NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
653 "No MPs claiming errors?\n", tpid);
654}
655
656static void
657nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
658 uint32_t ustatus_new, int display, const char *name)
659{
660 struct drm_nouveau_private *dev_priv = dev->dev_private;
661 int tps = 0;
662 uint32_t units = nv_rd32(dev, 0x1540);
663 int i, r;
664 uint32_t ustatus_addr, ustatus;
665 for (i = 0; i < 16; i++) {
666 if (!(units & (1 << i)))
667 continue;
668 if (dev_priv->chipset < 0xa0)
669 ustatus_addr = ustatus_old + (i << 12);
670 else
671 ustatus_addr = ustatus_new + (i << 11);
672 ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
673 if (!ustatus)
674 continue;
675 tps++;
676 switch (type) {
677 case 6: /* texture error... unknown for now */
678 nv50_fb_vm_trap(dev, display, name);
679 if (display) {
680 NV_ERROR(dev, "magic set %d:\n", i);
681 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
682 NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
683 nv_rd32(dev, r));
684 }
685 break;
686 case 7: /* MP error */
687 if (ustatus & 0x00010000) {
688 nv50_pgraph_mp_trap(dev, i, display);
689 ustatus &= ~0x00010000;
690 }
691 break;
692 case 8: /* TPDMA error */
693 {
694 uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
695 uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
696 uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
697 uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
698 uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
699 uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
700 uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
701 nv50_fb_vm_trap(dev, display, name);
702 /* 2d engine destination */
703 if (ustatus & 0x00000010) {
704 if (display) {
705 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
706 i, e14, e10);
707 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
708 i, e0c, e18, e1c, e20, e24);
709 }
710 ustatus &= ~0x00000010;
711 }
712 /* Render target */
713 if (ustatus & 0x00000040) {
714 if (display) {
715 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
716 i, e14, e10);
717 NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
718 i, e0c, e18, e1c, e20, e24);
719 }
720 ustatus &= ~0x00000040;
721 }
722 /* CUDA memory: l[], g[] or stack. */
723 if (ustatus & 0x00000080) {
724 if (display) {
725 if (e18 & 0x80000000) {
726 /* g[] read fault? */
727 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
728 i, e14, e10 | ((e18 >> 24) & 0x1f));
729 e18 &= ~0x1f000000;
730 } else if (e18 & 0xc) {
731 /* g[] write fault? */
732 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
733 i, e14, e10 | ((e18 >> 7) & 0x1f));
734 e18 &= ~0x00000f80;
735 } else {
736 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
737 i, e14, e10);
738 }
739 NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
740 i, e0c, e18, e1c, e20, e24);
741 }
742 ustatus &= ~0x00000080;
743 }
744 }
745 break;
746 }
747 if (ustatus) {
748 if (display)
749 NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
750 }
751 nv_wr32(dev, ustatus_addr, 0xc0000000);
752 }
753
754 if (!tps && display)
755 NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
756}
757
758static int
759nv50_pgraph_trap_handler(struct drm_device *dev, u32 display, u64 inst, u32 chid)
760{
761 u32 status = nv_rd32(dev, 0x400108);
762 u32 ustatus;
763
764 if (!status && display) {
765 NV_INFO(dev, "PGRAPH - TRAP: no units reporting traps?\n");
766 return 1;
767 }
768
769 /* DISPATCH: Relays commands to other units and handles NOTIFY,
770 * COND, QUERY. If you get a trap from it, the command is still stuck
771 * in DISPATCH and you need to do something about it. */
772 if (status & 0x001) {
773 ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
774 if (!ustatus && display) {
775 NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
776 }
777
778 nv_wr32(dev, 0x400500, 0x00000000);
779
780 /* Known to be triggered by screwed up NOTIFY and COND... */
781 if (ustatus & 0x00000001) {
782 u32 addr = nv_rd32(dev, 0x400808);
783 u32 subc = (addr & 0x00070000) >> 16;
784 u32 mthd = (addr & 0x00001ffc);
785 u32 datal = nv_rd32(dev, 0x40080c);
786 u32 datah = nv_rd32(dev, 0x400810);
787 u32 class = nv_rd32(dev, 0x400814);
788 u32 r848 = nv_rd32(dev, 0x400848);
789
790 NV_INFO(dev, "PGRAPH - TRAP DISPATCH_FAULT\n");
791 if (display && (addr & 0x80000000)) {
792 NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
793 "subc %d class 0x%04x mthd 0x%04x "
794 "data 0x%08x%08x "
795 "400808 0x%08x 400848 0x%08x\n",
796 chid, inst, subc, class, mthd, datah,
797 datal, addr, r848);
798 } else
799 if (display) {
800 NV_INFO(dev, "PGRAPH - no stuck command?\n");
801 }
802
803 nv_wr32(dev, 0x400808, 0);
804 nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
805 nv_wr32(dev, 0x400848, 0);
806 ustatus &= ~0x00000001;
807 }
808
809 if (ustatus & 0x00000002) {
810 u32 addr = nv_rd32(dev, 0x40084c);
811 u32 subc = (addr & 0x00070000) >> 16;
812 u32 mthd = (addr & 0x00001ffc);
813 u32 data = nv_rd32(dev, 0x40085c);
814 u32 class = nv_rd32(dev, 0x400814);
815
816 NV_INFO(dev, "PGRAPH - TRAP DISPATCH_QUERY\n");
817 if (display && (addr & 0x80000000)) {
818 NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
819 "subc %d class 0x%04x mthd 0x%04x "
820 "data 0x%08x 40084c 0x%08x\n",
821 chid, inst, subc, class, mthd,
822 data, addr);
823 } else
824 if (display) {
825 NV_INFO(dev, "PGRAPH - no stuck command?\n");
826 }
827
828 nv_wr32(dev, 0x40084c, 0);
829 ustatus &= ~0x00000002;
830 }
831
832 if (ustatus && display) {
833 NV_INFO(dev, "PGRAPH - TRAP_DISPATCH (unknown "
834 "0x%08x)\n", ustatus);
835 }
836
837 nv_wr32(dev, 0x400804, 0xc0000000);
838 nv_wr32(dev, 0x400108, 0x001);
839 status &= ~0x001;
840 if (!status)
841 return 0;
842 }
843
844 /* M2MF: Memory to memory copy engine. */
845 if (status & 0x002) {
846 u32 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
847 if (display) {
848 NV_INFO(dev, "PGRAPH - TRAP_M2MF");
849 nouveau_bitfield_print(nv50_graph_trap_m2mf, ustatus);
850 printk("\n");
851 NV_INFO(dev, "PGRAPH - TRAP_M2MF %08x %08x %08x %08x\n",
852 nv_rd32(dev, 0x406804), nv_rd32(dev, 0x406808),
853 nv_rd32(dev, 0x40680c), nv_rd32(dev, 0x406810));
854
855 }
856
857 /* No sane way found yet -- just reset the bugger. */
858 nv_wr32(dev, 0x400040, 2);
859 nv_wr32(dev, 0x400040, 0);
860 nv_wr32(dev, 0x406800, 0xc0000000);
861 nv_wr32(dev, 0x400108, 0x002);
862 status &= ~0x002;
863 }
864
865 /* VFETCH: Fetches data from vertex buffers. */
866 if (status & 0x004) {
867 u32 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
868 if (display) {
869 NV_INFO(dev, "PGRAPH - TRAP_VFETCH");
870 nouveau_bitfield_print(nv50_graph_trap_vfetch, ustatus);
871 printk("\n");
872 NV_INFO(dev, "PGRAPH - TRAP_VFETCH %08x %08x %08x %08x\n",
873 nv_rd32(dev, 0x400c00), nv_rd32(dev, 0x400c08),
874 nv_rd32(dev, 0x400c0c), nv_rd32(dev, 0x400c10));
875 }
876
877 nv_wr32(dev, 0x400c04, 0xc0000000);
878 nv_wr32(dev, 0x400108, 0x004);
879 status &= ~0x004;
880 }
881
882 /* STRMOUT: DirectX streamout / OpenGL transform feedback. */
883 if (status & 0x008) {
884 ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
885 if (display) {
886 NV_INFO(dev, "PGRAPH - TRAP_STRMOUT");
887 nouveau_bitfield_print(nv50_graph_trap_strmout, ustatus);
888 printk("\n");
889 NV_INFO(dev, "PGRAPH - TRAP_STRMOUT %08x %08x %08x %08x\n",
890 nv_rd32(dev, 0x401804), nv_rd32(dev, 0x401808),
891 nv_rd32(dev, 0x40180c), nv_rd32(dev, 0x401810));
892
893 }
894
895 /* No sane way found yet -- just reset the bugger. */
896 nv_wr32(dev, 0x400040, 0x80);
897 nv_wr32(dev, 0x400040, 0);
898 nv_wr32(dev, 0x401800, 0xc0000000);
899 nv_wr32(dev, 0x400108, 0x008);
900 status &= ~0x008;
901 }
902
903 /* CCACHE: Handles code and c[] caches and fills them. */
904 if (status & 0x010) {
905 ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
906 if (display) {
907 NV_INFO(dev, "PGRAPH - TRAP_CCACHE");
908 nouveau_bitfield_print(nv50_graph_trap_ccache, ustatus);
909 printk("\n");
910 NV_INFO(dev, "PGRAPH - TRAP_CCACHE %08x %08x %08x %08x"
911 " %08x %08x %08x\n",
912 nv_rd32(dev, 0x405800), nv_rd32(dev, 0x405804),
913 nv_rd32(dev, 0x405808), nv_rd32(dev, 0x40580c),
914 nv_rd32(dev, 0x405810), nv_rd32(dev, 0x405814),
915 nv_rd32(dev, 0x40581c));
916
917 }
918
919 nv_wr32(dev, 0x405018, 0xc0000000);
920 nv_wr32(dev, 0x400108, 0x010);
921 status &= ~0x010;
922 }
923
924 /* Unknown, not seen yet... 0x402000 is the only trap status reg
925 * remaining, so try to handle it anyway. Perhaps related to that
926 * unknown DMA slot on tesla? */
927 if (status & 0x20) {
928 ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
929 if (display)
930 NV_INFO(dev, "PGRAPH - TRAP_UNKC04 0x%08x\n", ustatus);
931 nv_wr32(dev, 0x402000, 0xc0000000);
932 /* no status modifiction on purpose */
933 }
934
935 /* TEXTURE: CUDA texturing units */
936 if (status & 0x040) {
937 nv50_pgraph_tp_trap(dev, 6, 0x408900, 0x408600, display,
938 "PGRAPH - TRAP_TEXTURE");
939 nv_wr32(dev, 0x400108, 0x040);
940 status &= ~0x040;
941 }
942
943 /* MP: CUDA execution engines. */
944 if (status & 0x080) {
945 nv50_pgraph_tp_trap(dev, 7, 0x408314, 0x40831c, display,
946 "PGRAPH - TRAP_MP");
947 nv_wr32(dev, 0x400108, 0x080);
948 status &= ~0x080;
949 }
950
951 /* TPDMA: Handles TP-initiated uncached memory accesses:
952 * l[], g[], stack, 2d surfaces, render targets. */
953 if (status & 0x100) {
954 nv50_pgraph_tp_trap(dev, 8, 0x408e08, 0x408708, display,
955 "PGRAPH - TRAP_TPDMA");
956 nv_wr32(dev, 0x400108, 0x100);
957 status &= ~0x100;
958 }
959
960 if (status) {
961 if (display)
962 NV_INFO(dev, "PGRAPH - TRAP: unknown 0x%08x\n", status);
963 nv_wr32(dev, 0x400108, status);
964 }
965
966 return 1;
967}
968
969static int
970nv50_graph_isr_chid(struct drm_device *dev, u64 inst)
971{
972 struct drm_nouveau_private *dev_priv = dev->dev_private;
973 struct nouveau_channel *chan;
974 unsigned long flags;
975 int i;
976
977 spin_lock_irqsave(&dev_priv->channels.lock, flags);
978 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
979 chan = dev_priv->channels.ptr[i];
980 if (!chan || !chan->ramin)
981 continue;
982
983 if (inst == chan->ramin->vinst)
984 break;
985 }
986 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
987 return i;
988}
989
990static void
991nv50_graph_isr(struct drm_device *dev)
992{
993 u32 stat;
994
995 while ((stat = nv_rd32(dev, 0x400100))) {
996 u64 inst = (u64)(nv_rd32(dev, 0x40032c) & 0x0fffffff) << 12;
997 u32 chid = nv50_graph_isr_chid(dev, inst);
998 u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
999 u32 subc = (addr & 0x00070000) >> 16;
1000 u32 mthd = (addr & 0x00001ffc);
1001 u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
1002 u32 class = nv_rd32(dev, 0x400814);
1003 u32 show = stat;
1004
1005 if (stat & 0x00000010) {
1006 if (!nouveau_gpuobj_mthd_call2(dev, chid, class,
1007 mthd, data))
1008 show &= ~0x00000010;
1009 }
1010
1011 if (stat & 0x00001000) {
1012 nv_wr32(dev, 0x400500, 0x00000000);
1013 nv_wr32(dev, 0x400100, 0x00001000);
1014 nv_mask(dev, 0x40013c, 0x00001000, 0x00000000);
1015 nv50_graph_context_switch(dev);
1016 stat &= ~0x00001000;
1017 show &= ~0x00001000;
1018 }
1019
1020 show = (show && nouveau_ratelimit()) ? show : 0;
1021
1022 if (show & 0x00100000) {
1023 u32 ecode = nv_rd32(dev, 0x400110);
1024 NV_INFO(dev, "PGRAPH - DATA_ERROR ");
1025 nouveau_enum_print(nv50_data_error_names, ecode);
1026 printk("\n");
1027 }
1028
1029 if (stat & 0x00200000) {
1030 if (!nv50_pgraph_trap_handler(dev, show, inst, chid))
1031 show &= ~0x00200000;
1032 }
1033
1034 nv_wr32(dev, 0x400100, stat);
1035 nv_wr32(dev, 0x400500, 0x00010001);
1036
1037 if (show) {
1038 NV_INFO(dev, "PGRAPH -");
1039 nouveau_bitfield_print(nv50_graph_intr, show);
1040 printk("\n");
1041 NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) subc %d "
1042 "class 0x%04x mthd 0x%04x data 0x%08x\n",
1043 chid, inst, subc, class, mthd, data);
1044 }
1045 }
1046
1047 if (nv_rd32(dev, 0x400824) & (1 << 31))
1048 nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
1049}
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
index b773229b7647..2e1b1cd19a4b 100644
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ b/drivers/gpu/drm/nouveau/nv50_instmem.c
@@ -27,14 +27,20 @@
27 27
28#include "drmP.h" 28#include "drmP.h"
29#include "drm.h" 29#include "drm.h"
30
30#include "nouveau_drv.h" 31#include "nouveau_drv.h"
32#include "nouveau_vm.h"
33
34#define BAR1_VM_BASE 0x0020000000ULL
35#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1)
36#define BAR3_VM_BASE 0x0000000000ULL
37#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3)
31 38
32struct nv50_instmem_priv { 39struct nv50_instmem_priv {
33 uint32_t save1700[5]; /* 0x1700->0x1710 */ 40 uint32_t save1700[5]; /* 0x1700->0x1710 */
34 41
35 struct nouveau_gpuobj *pramin_pt; 42 struct nouveau_gpuobj *bar1_dmaobj;
36 struct nouveau_gpuobj *pramin_bar; 43 struct nouveau_gpuobj *bar3_dmaobj;
37 struct nouveau_gpuobj *fb_bar;
38}; 44};
39 45
40static void 46static void
@@ -48,6 +54,7 @@ nv50_channel_del(struct nouveau_channel **pchan)
48 return; 54 return;
49 55
50 nouveau_gpuobj_ref(NULL, &chan->ramfc); 56 nouveau_gpuobj_ref(NULL, &chan->ramfc);
57 nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
51 nouveau_gpuobj_ref(NULL, &chan->vm_pd); 58 nouveau_gpuobj_ref(NULL, &chan->vm_pd);
52 if (chan->ramin_heap.free_stack.next) 59 if (chan->ramin_heap.free_stack.next)
53 drm_mm_takedown(&chan->ramin_heap); 60 drm_mm_takedown(&chan->ramin_heap);
@@ -56,14 +63,14 @@ nv50_channel_del(struct nouveau_channel **pchan)
56} 63}
57 64
58static int 65static int
59nv50_channel_new(struct drm_device *dev, u32 size, 66nv50_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
60 struct nouveau_channel **pchan) 67 struct nouveau_channel **pchan)
61{ 68{
62 struct drm_nouveau_private *dev_priv = dev->dev_private; 69 struct drm_nouveau_private *dev_priv = dev->dev_private;
63 u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200; 70 u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
64 u32 fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200; 71 u32 fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
65 struct nouveau_channel *chan; 72 struct nouveau_channel *chan;
66 int ret; 73 int ret, i;
67 74
68 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 75 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
69 if (!chan) 76 if (!chan)
@@ -92,6 +99,17 @@ nv50_channel_new(struct drm_device *dev, u32 size,
92 return ret; 99 return ret;
93 } 100 }
94 101
102 for (i = 0; i < 0x4000; i += 8) {
103 nv_wo32(chan->vm_pd, i + 0, 0x00000000);
104 nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
105 }
106
107 ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
108 if (ret) {
109 nv50_channel_del(&chan);
110 return ret;
111 }
112
95 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 : 113 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
96 chan->ramin->pinst + fc, 114 chan->ramin->pinst + fc,
97 chan->ramin->vinst + fc, 0x100, 115 chan->ramin->vinst + fc, 0x100,
@@ -111,6 +129,7 @@ nv50_instmem_init(struct drm_device *dev)
111 struct drm_nouveau_private *dev_priv = dev->dev_private; 129 struct drm_nouveau_private *dev_priv = dev->dev_private;
112 struct nv50_instmem_priv *priv; 130 struct nv50_instmem_priv *priv;
113 struct nouveau_channel *chan; 131 struct nouveau_channel *chan;
132 struct nouveau_vm *vm;
114 int ret, i; 133 int ret, i;
115 u32 tmp; 134 u32 tmp;
116 135
@@ -127,112 +146,87 @@ nv50_instmem_init(struct drm_device *dev)
127 ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size); 146 ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
128 if (ret) { 147 if (ret) {
129 NV_ERROR(dev, "Failed to init RAMIN heap\n"); 148 NV_ERROR(dev, "Failed to init RAMIN heap\n");
130 return -ENOMEM; 149 goto error;
131 } 150 }
132 151
133 /* we need a channel to plug into the hw to control the BARs */ 152 /* BAR3 */
134 ret = nv50_channel_new(dev, 128*1024, &dev_priv->fifos[0]); 153 ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE,
154 &dev_priv->bar3_vm);
135 if (ret) 155 if (ret)
136 return ret; 156 goto error;
137 chan = dev_priv->fifos[127] = dev_priv->fifos[0];
138 157
139 /* allocate page table for PRAMIN BAR */ 158 ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8,
140 ret = nouveau_gpuobj_new(dev, chan, (dev_priv->ramin_size >> 12) * 8, 159 0x1000, NVOBJ_FLAG_DONT_MAP |
141 0x1000, NVOBJ_FLAG_ZERO_ALLOC, 160 NVOBJ_FLAG_ZERO_ALLOC,
142 &priv->pramin_pt); 161 &dev_priv->bar3_vm->pgt[0].obj[0]);
143 if (ret) 162 if (ret)
144 return ret; 163 goto error;
164 dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
145 165
146 nv_wo32(chan->vm_pd, 0x0000, priv->pramin_pt->vinst | 0x63); 166 nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
147 nv_wo32(chan->vm_pd, 0x0004, 0);
148 167
149 /* DMA object for PRAMIN BAR */ 168 ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan);
150 ret = nouveau_gpuobj_new(dev, chan, 6*4, 16, 0, &priv->pramin_bar);
151 if (ret) 169 if (ret)
152 return ret; 170 goto error;
153 nv_wo32(priv->pramin_bar, 0x00, 0x7fc00000); 171 dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan;
154 nv_wo32(priv->pramin_bar, 0x04, dev_priv->ramin_size - 1); 172
155 nv_wo32(priv->pramin_bar, 0x08, 0x00000000); 173 ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE,
156 nv_wo32(priv->pramin_bar, 0x0c, 0x00000000); 174 NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
157 nv_wo32(priv->pramin_bar, 0x10, 0x00000000); 175 NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
158 nv_wo32(priv->pramin_bar, 0x14, 0x00000000); 176 &priv->bar3_dmaobj);
159
160 /* map channel into PRAMIN, gpuobj didn't do it for us */
161 ret = nv50_instmem_bind(dev, chan->ramin);
162 if (ret) 177 if (ret)
163 return ret; 178 goto error;
164 179
165 /* poke regs... */
166 nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12)); 180 nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12));
167 nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12)); 181 nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12));
168 nv_wr32(dev, 0x00170c, 0x80000000 | (priv->pramin_bar->cinst >> 4)); 182 nv_wr32(dev, 0x00170c, 0x80000000 | (priv->bar3_dmaobj->cinst >> 4));
169
170 tmp = nv_ri32(dev, 0);
171 nv_wi32(dev, 0, ~tmp);
172 if (nv_ri32(dev, 0) != ~tmp) {
173 NV_ERROR(dev, "PRAMIN readback failed\n");
174 return -EIO;
175 }
176 nv_wi32(dev, 0, tmp);
177 183
184 dev_priv->engine.instmem.flush(dev);
178 dev_priv->ramin_available = true; 185 dev_priv->ramin_available = true;
179 186
180 /* Determine VM layout */ 187 tmp = nv_ro32(chan->ramin, 0);
181 dev_priv->vm_gart_base = roundup(NV50_VM_BLOCK, NV50_VM_BLOCK); 188 nv_wo32(chan->ramin, 0, ~tmp);
182 dev_priv->vm_gart_size = NV50_VM_BLOCK; 189 if (nv_ro32(chan->ramin, 0) != ~tmp) {
183 190 NV_ERROR(dev, "PRAMIN readback failed\n");
184 dev_priv->vm_vram_base = dev_priv->vm_gart_base + dev_priv->vm_gart_size; 191 ret = -EIO;
185 dev_priv->vm_vram_size = dev_priv->vram_size; 192 goto error;
186 if (dev_priv->vm_vram_size > NV50_VM_MAX_VRAM)
187 dev_priv->vm_vram_size = NV50_VM_MAX_VRAM;
188 dev_priv->vm_vram_size = roundup(dev_priv->vm_vram_size, NV50_VM_BLOCK);
189 dev_priv->vm_vram_pt_nr = dev_priv->vm_vram_size / NV50_VM_BLOCK;
190
191 dev_priv->vm_end = dev_priv->vm_vram_base + dev_priv->vm_vram_size;
192
193 NV_DEBUG(dev, "NV50VM: GART 0x%016llx-0x%016llx\n",
194 dev_priv->vm_gart_base,
195 dev_priv->vm_gart_base + dev_priv->vm_gart_size - 1);
196 NV_DEBUG(dev, "NV50VM: VRAM 0x%016llx-0x%016llx\n",
197 dev_priv->vm_vram_base,
198 dev_priv->vm_vram_base + dev_priv->vm_vram_size - 1);
199
200 /* VRAM page table(s), mapped into VM at +1GiB */
201 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) {
202 ret = nouveau_gpuobj_new(dev, NULL, NV50_VM_BLOCK / 0x10000 * 8,
203 0, NVOBJ_FLAG_ZERO_ALLOC,
204 &chan->vm_vram_pt[i]);
205 if (ret) {
206 NV_ERROR(dev, "Error creating VRAM PGT: %d\n", ret);
207 dev_priv->vm_vram_pt_nr = i;
208 return ret;
209 }
210 dev_priv->vm_vram_pt[i] = chan->vm_vram_pt[i];
211
212 nv_wo32(chan->vm_pd, 0x10 + (i*8),
213 chan->vm_vram_pt[i]->vinst | 0x61);
214 nv_wo32(chan->vm_pd, 0x14 + (i*8), 0);
215 } 193 }
194 nv_wo32(chan->ramin, 0, tmp);
216 195
217 /* DMA object for FB BAR */ 196 /* BAR1 */
218 ret = nouveau_gpuobj_new(dev, chan, 6*4, 16, 0, &priv->fb_bar); 197 ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm);
219 if (ret) 198 if (ret)
220 return ret; 199 goto error;
221 nv_wo32(priv->fb_bar, 0x00, 0x7fc00000);
222 nv_wo32(priv->fb_bar, 0x04, 0x40000000 +
223 pci_resource_len(dev->pdev, 1) - 1);
224 nv_wo32(priv->fb_bar, 0x08, 0x40000000);
225 nv_wo32(priv->fb_bar, 0x0c, 0x00000000);
226 nv_wo32(priv->fb_bar, 0x10, 0x00000000);
227 nv_wo32(priv->fb_bar, 0x14, 0x00000000);
228 200
229 dev_priv->engine.instmem.flush(dev); 201 ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd);
202 if (ret)
203 goto error;
204 nouveau_vm_ref(NULL, &vm, NULL);
205
206 ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE,
207 NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
208 NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
209 &priv->bar1_dmaobj);
210 if (ret)
211 goto error;
230 212
231 nv_wr32(dev, 0x001708, 0x80000000 | (priv->fb_bar->cinst >> 4)); 213 nv_wr32(dev, 0x001708, 0x80000000 | (priv->bar1_dmaobj->cinst >> 4));
232 for (i = 0; i < 8; i++) 214 for (i = 0; i < 8; i++)
233 nv_wr32(dev, 0x1900 + (i*4), 0); 215 nv_wr32(dev, 0x1900 + (i*4), 0);
234 216
217 /* Create shared channel VM, space is reserved at the beginning
218 * to catch "NULL pointer" references
219 */
220 ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
221 &dev_priv->chan_vm);
222 if (ret)
223 return ret;
224
235 return 0; 225 return 0;
226
227error:
228 nv50_instmem_takedown(dev);
229 return ret;
236} 230}
237 231
238void 232void
@@ -240,7 +234,7 @@ nv50_instmem_takedown(struct drm_device *dev)
240{ 234{
241 struct drm_nouveau_private *dev_priv = dev->dev_private; 235 struct drm_nouveau_private *dev_priv = dev->dev_private;
242 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv; 236 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
243 struct nouveau_channel *chan = dev_priv->fifos[0]; 237 struct nouveau_channel *chan = dev_priv->channels.ptr[0];
244 int i; 238 int i;
245 239
246 NV_DEBUG(dev, "\n"); 240 NV_DEBUG(dev, "\n");
@@ -250,23 +244,23 @@ nv50_instmem_takedown(struct drm_device *dev)
250 244
251 dev_priv->ramin_available = false; 245 dev_priv->ramin_available = false;
252 246
253 /* Restore state from before init */ 247 nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
248
254 for (i = 0x1700; i <= 0x1710; i += 4) 249 for (i = 0x1700; i <= 0x1710; i += 4)
255 nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]); 250 nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);
256 251
257 nouveau_gpuobj_ref(NULL, &priv->fb_bar); 252 nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj);
258 nouveau_gpuobj_ref(NULL, &priv->pramin_bar); 253 nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj);
259 nouveau_gpuobj_ref(NULL, &priv->pramin_pt);
260 254
261 /* Destroy dummy channel */ 255 nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd);
262 if (chan) { 256 dev_priv->channels.ptr[127] = 0;
263 for (i = 0; i < dev_priv->vm_vram_pt_nr; i++) 257 nv50_channel_del(&dev_priv->channels.ptr[0]);
264 nouveau_gpuobj_ref(NULL, &chan->vm_vram_pt[i]);
265 dev_priv->vm_vram_pt_nr = 0;
266 258
267 nv50_channel_del(&dev_priv->fifos[0]); 259 nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
268 dev_priv->fifos[127] = NULL; 260 nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
269 } 261
262 if (dev_priv->ramin_heap.free_stack.next)
263 drm_mm_takedown(&dev_priv->ramin_heap);
270 264
271 dev_priv->engine.instmem.priv = NULL; 265 dev_priv->engine.instmem.priv = NULL;
272 kfree(priv); 266 kfree(priv);
@@ -276,16 +270,8 @@ int
276nv50_instmem_suspend(struct drm_device *dev) 270nv50_instmem_suspend(struct drm_device *dev)
277{ 271{
278 struct drm_nouveau_private *dev_priv = dev->dev_private; 272 struct drm_nouveau_private *dev_priv = dev->dev_private;
279 struct nouveau_channel *chan = dev_priv->fifos[0];
280 struct nouveau_gpuobj *ramin = chan->ramin;
281 int i;
282 273
283 ramin->im_backing_suspend = vmalloc(ramin->size); 274 dev_priv->ramin_available = false;
284 if (!ramin->im_backing_suspend)
285 return -ENOMEM;
286
287 for (i = 0; i < ramin->size; i += 4)
288 ramin->im_backing_suspend[i/4] = nv_ri32(dev, i);
289 return 0; 275 return 0;
290} 276}
291 277
@@ -294,146 +280,121 @@ nv50_instmem_resume(struct drm_device *dev)
294{ 280{
295 struct drm_nouveau_private *dev_priv = dev->dev_private; 281 struct drm_nouveau_private *dev_priv = dev->dev_private;
296 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv; 282 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
297 struct nouveau_channel *chan = dev_priv->fifos[0]; 283 struct nouveau_channel *chan = dev_priv->channels.ptr[0];
298 struct nouveau_gpuobj *ramin = chan->ramin;
299 int i; 284 int i;
300 285
301 dev_priv->ramin_available = false;
302 dev_priv->ramin_base = ~0;
303 for (i = 0; i < ramin->size; i += 4)
304 nv_wo32(ramin, i, ramin->im_backing_suspend[i/4]);
305 dev_priv->ramin_available = true;
306 vfree(ramin->im_backing_suspend);
307 ramin->im_backing_suspend = NULL;
308
309 /* Poke the relevant regs, and pray it works :) */ 286 /* Poke the relevant regs, and pray it works :) */
310 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12)); 287 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
311 nv_wr32(dev, NV50_PUNK_UNK1710, 0); 288 nv_wr32(dev, NV50_PUNK_UNK1710, 0);
312 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) | 289 nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) |
313 NV50_PUNK_BAR_CFG_BASE_VALID); 290 NV50_PUNK_BAR_CFG_BASE_VALID);
314 nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->fb_bar->cinst >> 4) | 291 nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) |
315 NV50_PUNK_BAR1_CTXDMA_VALID); 292 NV50_PUNK_BAR1_CTXDMA_VALID);
316 nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->pramin_bar->cinst >> 4) | 293 nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) |
317 NV50_PUNK_BAR3_CTXDMA_VALID); 294 NV50_PUNK_BAR3_CTXDMA_VALID);
318 295
319 for (i = 0; i < 8; i++) 296 for (i = 0; i < 8; i++)
320 nv_wr32(dev, 0x1900 + (i*4), 0); 297 nv_wr32(dev, 0x1900 + (i*4), 0);
298
299 dev_priv->ramin_available = true;
321} 300}
322 301
302struct nv50_gpuobj_node {
303 struct nouveau_vram *vram;
304 struct nouveau_vma chan_vma;
305 u32 align;
306};
307
308
323int 309int
324nv50_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, 310nv50_instmem_get(struct nouveau_gpuobj *gpuobj, u32 size, u32 align)
325 uint32_t *sz)
326{ 311{
312 struct drm_device *dev = gpuobj->dev;
313 struct drm_nouveau_private *dev_priv = dev->dev_private;
314 struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
315 struct nv50_gpuobj_node *node = NULL;
327 int ret; 316 int ret;
328 317
329 if (gpuobj->im_backing) 318 node = kzalloc(sizeof(*node), GFP_KERNEL);
330 return -EINVAL; 319 if (!node)
320 return -ENOMEM;
321 node->align = align;
331 322
332 *sz = ALIGN(*sz, 4096); 323 size = (size + 4095) & ~4095;
333 if (*sz == 0) 324 align = max(align, (u32)4096);
334 return -EINVAL;
335 325
336 ret = nouveau_bo_new(dev, NULL, *sz, 0, TTM_PL_FLAG_VRAM, 0, 0x0000, 326 ret = vram->get(dev, size, align, 0, 0, &node->vram);
337 true, false, &gpuobj->im_backing);
338 if (ret) { 327 if (ret) {
339 NV_ERROR(dev, "error getting PRAMIN backing pages: %d\n", ret); 328 kfree(node);
340 return ret; 329 return ret;
341 } 330 }
342 331
343 ret = nouveau_bo_pin(gpuobj->im_backing, TTM_PL_FLAG_VRAM); 332 gpuobj->vinst = node->vram->offset;
344 if (ret) { 333
345 NV_ERROR(dev, "error pinning PRAMIN backing VRAM: %d\n", ret); 334 if (gpuobj->flags & NVOBJ_FLAG_VM) {
346 nouveau_bo_ref(NULL, &gpuobj->im_backing); 335 ret = nouveau_vm_get(dev_priv->chan_vm, size, 12,
347 return ret; 336 NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
337 &node->chan_vma);
338 if (ret) {
339 vram->put(dev, &node->vram);
340 kfree(node);
341 return ret;
342 }
343
344 nouveau_vm_map(&node->chan_vma, node->vram);
345 gpuobj->vinst = node->chan_vma.offset;
348 } 346 }
349 347
350 gpuobj->vinst = gpuobj->im_backing->bo.mem.start << PAGE_SHIFT; 348 gpuobj->size = size;
349 gpuobj->node = node;
351 return 0; 350 return 0;
352} 351}
353 352
354void 353void
355nv50_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 354nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
356{ 355{
356 struct drm_device *dev = gpuobj->dev;
357 struct drm_nouveau_private *dev_priv = dev->dev_private; 357 struct drm_nouveau_private *dev_priv = dev->dev_private;
358 struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
359 struct nv50_gpuobj_node *node;
360
361 node = gpuobj->node;
362 gpuobj->node = NULL;
358 363
359 if (gpuobj && gpuobj->im_backing) { 364 if (node->chan_vma.node) {
360 if (gpuobj->im_bound) 365 nouveau_vm_unmap(&node->chan_vma);
361 dev_priv->engine.instmem.unbind(dev, gpuobj); 366 nouveau_vm_put(&node->chan_vma);
362 nouveau_bo_unpin(gpuobj->im_backing);
363 nouveau_bo_ref(NULL, &gpuobj->im_backing);
364 gpuobj->im_backing = NULL;
365 } 367 }
368 vram->put(dev, &node->vram);
369 kfree(node);
366} 370}
367 371
368int 372int
369nv50_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 373nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
370{ 374{
371 struct drm_nouveau_private *dev_priv = dev->dev_private; 375 struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
372 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv; 376 struct nv50_gpuobj_node *node = gpuobj->node;
373 struct nouveau_gpuobj *pramin_pt = priv->pramin_pt; 377 int ret;
374 uint32_t pte, pte_end;
375 uint64_t vram;
376
377 if (!gpuobj->im_backing || !gpuobj->im_pramin || gpuobj->im_bound)
378 return -EINVAL;
379
380 NV_DEBUG(dev, "st=0x%lx sz=0x%lx\n",
381 gpuobj->im_pramin->start, gpuobj->im_pramin->size);
382
383 pte = (gpuobj->im_pramin->start >> 12) << 1;
384 pte_end = ((gpuobj->im_pramin->size >> 12) << 1) + pte;
385 vram = gpuobj->vinst;
386
387 NV_DEBUG(dev, "pramin=0x%lx, pte=%d, pte_end=%d\n",
388 gpuobj->im_pramin->start, pte, pte_end);
389 NV_DEBUG(dev, "first vram page: 0x%010llx\n", gpuobj->vinst);
390
391 vram |= 1;
392 if (dev_priv->vram_sys_base) {
393 vram += dev_priv->vram_sys_base;
394 vram |= 0x30;
395 }
396
397 while (pte < pte_end) {
398 nv_wo32(pramin_pt, (pte * 4) + 0, lower_32_bits(vram));
399 nv_wo32(pramin_pt, (pte * 4) + 4, upper_32_bits(vram));
400 vram += 0x1000;
401 pte += 2;
402 }
403 dev_priv->engine.instmem.flush(dev);
404 378
405 nv50_vm_flush(dev, 6); 379 ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12,
380 NV_MEM_ACCESS_RW, &node->vram->bar_vma);
381 if (ret)
382 return ret;
406 383
407 gpuobj->im_bound = 1; 384 nouveau_vm_map(&node->vram->bar_vma, node->vram);
385 gpuobj->pinst = node->vram->bar_vma.offset;
408 return 0; 386 return 0;
409} 387}
410 388
411int 389void
412nv50_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 390nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
413{ 391{
414 struct drm_nouveau_private *dev_priv = dev->dev_private; 392 struct nv50_gpuobj_node *node = gpuobj->node;
415 struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
416 uint32_t pte, pte_end;
417
418 if (gpuobj->im_bound == 0)
419 return -EINVAL;
420
421 /* can happen during late takedown */
422 if (unlikely(!dev_priv->ramin_available))
423 return 0;
424 393
425 pte = (gpuobj->im_pramin->start >> 12) << 1; 394 if (node->vram->bar_vma.node) {
426 pte_end = ((gpuobj->im_pramin->size >> 12) << 1) + pte; 395 nouveau_vm_unmap(&node->vram->bar_vma);
427 396 nouveau_vm_put(&node->vram->bar_vma);
428 while (pte < pte_end) {
429 nv_wo32(priv->pramin_pt, (pte * 4) + 0, 0x00000000);
430 nv_wo32(priv->pramin_pt, (pte * 4) + 4, 0x00000000);
431 pte += 2;
432 } 397 }
433 dev_priv->engine.instmem.flush(dev);
434
435 gpuobj->im_bound = 0;
436 return 0;
437} 398}
438 399
439void 400void
@@ -452,11 +413,3 @@ nv84_instmem_flush(struct drm_device *dev)
452 NV_ERROR(dev, "PRAMIN flush timeout\n"); 413 NV_ERROR(dev, "PRAMIN flush timeout\n");
453} 414}
454 415
455void
456nv50_vm_flush(struct drm_device *dev, int engine)
457{
458 nv_wr32(dev, 0x100c80, (engine << 16) | 1);
459 if (!nv_wait(dev, 0x100c80, 0x00000001, 0x00000000))
460 NV_ERROR(dev, "vm flush timeout: engine %d\n", engine);
461}
462
diff --git a/drivers/gpu/drm/nouveau/nv50_vm.c b/drivers/gpu/drm/nouveau/nv50_vm.c
new file mode 100644
index 000000000000..38e523e10995
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_vm.c
@@ -0,0 +1,180 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26
27#include "nouveau_drv.h"
28#include "nouveau_vm.h"
29
30void
31nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
32 struct nouveau_gpuobj *pgt[2])
33{
34 struct drm_nouveau_private *dev_priv = pgd->dev->dev_private;
35 u64 phys = 0xdeadcafe00000000ULL;
36 u32 coverage = 0;
37
38 if (pgt[0]) {
39 phys = 0x00000003 | pgt[0]->vinst; /* present, 4KiB pages */
40 coverage = (pgt[0]->size >> 3) << 12;
41 } else
42 if (pgt[1]) {
43 phys = 0x00000001 | pgt[1]->vinst; /* present */
44 coverage = (pgt[1]->size >> 3) << 16;
45 }
46
47 if (phys & 1) {
48 if (dev_priv->vram_sys_base) {
49 phys += dev_priv->vram_sys_base;
50 phys |= 0x30;
51 }
52
53 if (coverage <= 32 * 1024 * 1024)
54 phys |= 0x60;
55 else if (coverage <= 64 * 1024 * 1024)
56 phys |= 0x40;
57 else if (coverage < 128 * 1024 * 1024)
58 phys |= 0x20;
59 }
60
61 nv_wo32(pgd, (pde * 8) + 0, lower_32_bits(phys));
62 nv_wo32(pgd, (pde * 8) + 4, upper_32_bits(phys));
63}
64
65static inline u64
66nv50_vm_addr(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
67 u64 phys, u32 memtype, u32 target)
68{
69 struct drm_nouveau_private *dev_priv = pgt->dev->dev_private;
70
71 phys |= 1; /* present */
72 phys |= (u64)memtype << 40;
73
74 /* IGPs don't have real VRAM, re-target to stolen system memory */
75 if (target == 0 && dev_priv->vram_sys_base) {
76 phys += dev_priv->vram_sys_base;
77 target = 3;
78 }
79
80 phys |= target << 4;
81
82 if (vma->access & NV_MEM_ACCESS_SYS)
83 phys |= (1 << 6);
84
85 if (!(vma->access & NV_MEM_ACCESS_WO))
86 phys |= (1 << 3);
87
88 return phys;
89}
90
91void
92nv50_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
93 struct nouveau_vram *mem, u32 pte, u32 cnt, u64 phys)
94{
95 u32 block;
96 int i;
97
98 phys = nv50_vm_addr(vma, pgt, phys, mem->memtype, 0);
99 pte <<= 3;
100 cnt <<= 3;
101
102 while (cnt) {
103 u32 offset_h = upper_32_bits(phys);
104 u32 offset_l = lower_32_bits(phys);
105
106 for (i = 7; i >= 0; i--) {
107 block = 1 << (i + 3);
108 if (cnt >= block && !(pte & (block - 1)))
109 break;
110 }
111 offset_l |= (i << 7);
112
113 phys += block << (vma->node->type - 3);
114 cnt -= block;
115
116 while (block) {
117 nv_wo32(pgt, pte + 0, offset_l);
118 nv_wo32(pgt, pte + 4, offset_h);
119 pte += 8;
120 block -= 8;
121 }
122 }
123}
124
125void
126nv50_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
127 u32 pte, dma_addr_t *list, u32 cnt)
128{
129 pte <<= 3;
130 while (cnt--) {
131 u64 phys = nv50_vm_addr(vma, pgt, (u64)*list++, 0, 2);
132 nv_wo32(pgt, pte + 0, lower_32_bits(phys));
133 nv_wo32(pgt, pte + 4, upper_32_bits(phys));
134 pte += 8;
135 }
136}
137
138void
139nv50_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
140{
141 pte <<= 3;
142 while (cnt--) {
143 nv_wo32(pgt, pte + 0, 0x00000000);
144 nv_wo32(pgt, pte + 4, 0x00000000);
145 pte += 8;
146 }
147}
148
149void
150nv50_vm_flush(struct nouveau_vm *vm)
151{
152 struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
153 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
154 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
155 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
156 struct nouveau_crypt_engine *pcrypt = &dev_priv->engine.crypt;
157
158 pinstmem->flush(vm->dev);
159
160 /* BAR */
161 if (vm != dev_priv->chan_vm) {
162 nv50_vm_flush_engine(vm->dev, 6);
163 return;
164 }
165
166 pfifo->tlb_flush(vm->dev);
167
168 if (atomic_read(&vm->pgraph_refs))
169 pgraph->tlb_flush(vm->dev);
170 if (atomic_read(&vm->pcrypt_refs))
171 pcrypt->tlb_flush(vm->dev);
172}
173
174void
175nv50_vm_flush_engine(struct drm_device *dev, int engine)
176{
177 nv_wr32(dev, 0x100c80, (engine << 16) | 1);
178 if (!nv_wait(dev, 0x100c80, 0x00000001, 0x00000000))
179 NV_ERROR(dev, "vm flush timeout: engine %d\n", engine);
180}
diff --git a/drivers/gpu/drm/nouveau/nv50_vram.c b/drivers/gpu/drm/nouveau/nv50_vram.c
new file mode 100644
index 000000000000..58e98ad36347
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_vram.c
@@ -0,0 +1,190 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_mm.h"
28
29static int types[0x80] = {
30 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
31 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0,
32 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0,
33 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
34 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0,
35 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
36 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2,
37 1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
38};
39
40bool
41nv50_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
42{
43 int type = (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) >> 8;
44
45 if (likely(type < ARRAY_SIZE(types) && types[type]))
46 return true;
47 return false;
48}
49
50void
51nv50_vram_del(struct drm_device *dev, struct nouveau_vram **pvram)
52{
53 struct drm_nouveau_private *dev_priv = dev->dev_private;
54 struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
55 struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
56 struct nouveau_mm *mm = man->priv;
57 struct nouveau_mm_node *this;
58 struct nouveau_vram *vram;
59
60 vram = *pvram;
61 *pvram = NULL;
62 if (unlikely(vram == NULL))
63 return;
64
65 mutex_lock(&mm->mutex);
66 while (!list_empty(&vram->regions)) {
67 this = list_first_entry(&vram->regions, struct nouveau_mm_node, rl_entry);
68
69 list_del(&this->rl_entry);
70 nouveau_mm_put(mm, this);
71 }
72 mutex_unlock(&mm->mutex);
73
74 kfree(vram);
75}
76
77int
78nv50_vram_new(struct drm_device *dev, u64 size, u32 align, u32 size_nc,
79 u32 type, struct nouveau_vram **pvram)
80{
81 struct drm_nouveau_private *dev_priv = dev->dev_private;
82 struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
83 struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
84 struct nouveau_mm *mm = man->priv;
85 struct nouveau_mm_node *r;
86 struct nouveau_vram *vram;
87 int ret;
88
89 if (!types[type])
90 return -EINVAL;
91 size >>= 12;
92 align >>= 12;
93 size_nc >>= 12;
94
95 vram = kzalloc(sizeof(*vram), GFP_KERNEL);
96 if (!vram)
97 return -ENOMEM;
98
99 INIT_LIST_HEAD(&vram->regions);
100 vram->dev = dev_priv->dev;
101 vram->memtype = type;
102 vram->size = size;
103
104 mutex_lock(&mm->mutex);
105 do {
106 ret = nouveau_mm_get(mm, types[type], size, size_nc, align, &r);
107 if (ret) {
108 mutex_unlock(&mm->mutex);
109 nv50_vram_del(dev, &vram);
110 return ret;
111 }
112
113 list_add_tail(&r->rl_entry, &vram->regions);
114 size -= r->length;
115 } while (size);
116 mutex_unlock(&mm->mutex);
117
118 r = list_first_entry(&vram->regions, struct nouveau_mm_node, rl_entry);
119 vram->offset = (u64)r->offset << 12;
120 *pvram = vram;
121 return 0;
122}
123
124static u32
125nv50_vram_rblock(struct drm_device *dev)
126{
127 struct drm_nouveau_private *dev_priv = dev->dev_private;
128 int i, parts, colbits, rowbitsa, rowbitsb, banks;
129 u64 rowsize, predicted;
130 u32 r0, r4, rt, ru, rblock_size;
131
132 r0 = nv_rd32(dev, 0x100200);
133 r4 = nv_rd32(dev, 0x100204);
134 rt = nv_rd32(dev, 0x100250);
135 ru = nv_rd32(dev, 0x001540);
136 NV_DEBUG(dev, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n", r0, r4, rt, ru);
137
138 for (i = 0, parts = 0; i < 8; i++) {
139 if (ru & (0x00010000 << i))
140 parts++;
141 }
142
143 colbits = (r4 & 0x0000f000) >> 12;
144 rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
145 rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
146 banks = ((r4 & 0x01000000) ? 8 : 4);
147
148 rowsize = parts * banks * (1 << colbits) * 8;
149 predicted = rowsize << rowbitsa;
150 if (r0 & 0x00000004)
151 predicted += rowsize << rowbitsb;
152
153 if (predicted != dev_priv->vram_size) {
154 NV_WARN(dev, "memory controller reports %dMiB VRAM\n",
155 (u32)(dev_priv->vram_size >> 20));
156 NV_WARN(dev, "we calculated %dMiB VRAM\n",
157 (u32)(predicted >> 20));
158 }
159
160 rblock_size = rowsize;
161 if (rt & 1)
162 rblock_size *= 3;
163
164 NV_DEBUG(dev, "rblock %d bytes\n", rblock_size);
165 return rblock_size;
166}
167
168int
169nv50_vram_init(struct drm_device *dev)
170{
171 struct drm_nouveau_private *dev_priv = dev->dev_private;
172
173 dev_priv->vram_size = nv_rd32(dev, 0x10020c);
174 dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
175 dev_priv->vram_size &= 0xffffffff00ULL;
176
177 switch (dev_priv->chipset) {
178 case 0xaa:
179 case 0xac:
180 case 0xaf:
181 dev_priv->vram_sys_base = (u64)nv_rd32(dev, 0x100e10) << 12;
182 dev_priv->vram_rblock_size = 4096;
183 break;
184 default:
185 dev_priv->vram_rblock_size = nv50_vram_rblock(dev);
186 break;
187 }
188
189 return 0;
190}
diff --git a/drivers/gpu/drm/nouveau/nv84_crypt.c b/drivers/gpu/drm/nouveau/nv84_crypt.c
new file mode 100644
index 000000000000..ec18ae1c3886
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv84_crypt.c
@@ -0,0 +1,140 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_util.h"
28#include "nouveau_vm.h"
29
30static void nv84_crypt_isr(struct drm_device *);
31
32int
33nv84_crypt_create_context(struct nouveau_channel *chan)
34{
35 struct drm_device *dev = chan->dev;
36 struct drm_nouveau_private *dev_priv = dev->dev_private;
37 struct nouveau_gpuobj *ramin = chan->ramin;
38 int ret;
39
40 NV_DEBUG(dev, "ch%d\n", chan->id);
41
42 ret = nouveau_gpuobj_new(dev, chan, 256, 0,
43 NVOBJ_FLAG_ZERO_ALLOC | NVOBJ_FLAG_ZERO_FREE,
44 &chan->crypt_ctx);
45 if (ret)
46 return ret;
47
48 nv_wo32(ramin, 0xa0, 0x00190000);
49 nv_wo32(ramin, 0xa4, chan->crypt_ctx->vinst + 0xff);
50 nv_wo32(ramin, 0xa8, chan->crypt_ctx->vinst);
51 nv_wo32(ramin, 0xac, 0);
52 nv_wo32(ramin, 0xb0, 0);
53 nv_wo32(ramin, 0xb4, 0);
54
55 dev_priv->engine.instmem.flush(dev);
56 atomic_inc(&chan->vm->pcrypt_refs);
57 return 0;
58}
59
60void
61nv84_crypt_destroy_context(struct nouveau_channel *chan)
62{
63 struct drm_device *dev = chan->dev;
64 u32 inst;
65
66 if (!chan->crypt_ctx)
67 return;
68
69 inst = (chan->ramin->vinst >> 12);
70 inst |= 0x80000000;
71
72 /* mark context as invalid if still on the hardware, not
73 * doing this causes issues the next time PCRYPT is used,
74 * unsurprisingly :)
75 */
76 nv_wr32(dev, 0x10200c, 0x00000000);
77 if (nv_rd32(dev, 0x102188) == inst)
78 nv_mask(dev, 0x102188, 0x80000000, 0x00000000);
79 if (nv_rd32(dev, 0x10218c) == inst)
80 nv_mask(dev, 0x10218c, 0x80000000, 0x00000000);
81 nv_wr32(dev, 0x10200c, 0x00000010);
82
83 nouveau_gpuobj_ref(NULL, &chan->crypt_ctx);
84 atomic_dec(&chan->vm->pcrypt_refs);
85}
86
87void
88nv84_crypt_tlb_flush(struct drm_device *dev)
89{
90 nv50_vm_flush_engine(dev, 0x0a);
91}
92
93int
94nv84_crypt_init(struct drm_device *dev)
95{
96 struct drm_nouveau_private *dev_priv = dev->dev_private;
97 struct nouveau_crypt_engine *pcrypt = &dev_priv->engine.crypt;
98
99 if (!pcrypt->registered) {
100 NVOBJ_CLASS(dev, 0x74c1, CRYPT);
101 pcrypt->registered = true;
102 }
103
104 nv_mask(dev, 0x000200, 0x00004000, 0x00000000);
105 nv_mask(dev, 0x000200, 0x00004000, 0x00004000);
106
107 nouveau_irq_register(dev, 14, nv84_crypt_isr);
108 nv_wr32(dev, 0x102130, 0xffffffff);
109 nv_wr32(dev, 0x102140, 0xffffffbf);
110
111 nv_wr32(dev, 0x10200c, 0x00000010);
112 return 0;
113}
114
115void
116nv84_crypt_fini(struct drm_device *dev)
117{
118 nv_wr32(dev, 0x102140, 0x00000000);
119 nouveau_irq_unregister(dev, 14);
120}
121
122static void
123nv84_crypt_isr(struct drm_device *dev)
124{
125 u32 stat = nv_rd32(dev, 0x102130);
126 u32 mthd = nv_rd32(dev, 0x102190);
127 u32 data = nv_rd32(dev, 0x102194);
128 u32 inst = nv_rd32(dev, 0x102188) & 0x7fffffff;
129 int show = nouveau_ratelimit();
130
131 if (show) {
132 NV_INFO(dev, "PCRYPT_INTR: 0x%08x 0x%08x 0x%08x 0x%08x\n",
133 stat, mthd, data, inst);
134 }
135
136 nv_wr32(dev, 0x102130, stat);
137 nv_wr32(dev, 0x10200c, 0x10);
138
139 nv50_fb_vm_trap(dev, show, "PCRYPT");
140}
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
new file mode 100644
index 000000000000..fa5d4c234383
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -0,0 +1,269 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_dma.h"
28#include "nouveau_ramht.h"
29#include "nouveau_fbcon.h"
30#include "nouveau_mm.h"
31
32int
33nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
34{
35 struct nouveau_fbdev *nfbdev = info->par;
36 struct drm_device *dev = nfbdev->dev;
37 struct drm_nouveau_private *dev_priv = dev->dev_private;
38 struct nouveau_channel *chan = dev_priv->channel;
39 int ret;
40
41 ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
42 if (ret)
43 return ret;
44
45 if (rect->rop != ROP_COPY) {
46 BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
47 OUT_RING (chan, 1);
48 }
49 BEGIN_NVC0(chan, 2, NvSub2D, 0x0588, 1);
50 if (info->fix.visual == FB_VISUAL_TRUECOLOR ||
51 info->fix.visual == FB_VISUAL_DIRECTCOLOR)
52 OUT_RING (chan, ((uint32_t *)info->pseudo_palette)[rect->color]);
53 else
54 OUT_RING (chan, rect->color);
55 BEGIN_NVC0(chan, 2, NvSub2D, 0x0600, 4);
56 OUT_RING (chan, rect->dx);
57 OUT_RING (chan, rect->dy);
58 OUT_RING (chan, rect->dx + rect->width);
59 OUT_RING (chan, rect->dy + rect->height);
60 if (rect->rop != ROP_COPY) {
61 BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
62 OUT_RING (chan, 3);
63 }
64 FIRE_RING(chan);
65 return 0;
66}
67
68int
69nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
70{
71 struct nouveau_fbdev *nfbdev = info->par;
72 struct drm_device *dev = nfbdev->dev;
73 struct drm_nouveau_private *dev_priv = dev->dev_private;
74 struct nouveau_channel *chan = dev_priv->channel;
75 int ret;
76
77 ret = RING_SPACE(chan, 12);
78 if (ret)
79 return ret;
80
81 BEGIN_NVC0(chan, 2, NvSub2D, 0x0110, 1);
82 OUT_RING (chan, 0);
83 BEGIN_NVC0(chan, 2, NvSub2D, 0x08b0, 4);
84 OUT_RING (chan, region->dx);
85 OUT_RING (chan, region->dy);
86 OUT_RING (chan, region->width);
87 OUT_RING (chan, region->height);
88 BEGIN_NVC0(chan, 2, NvSub2D, 0x08d0, 4);
89 OUT_RING (chan, 0);
90 OUT_RING (chan, region->sx);
91 OUT_RING (chan, 0);
92 OUT_RING (chan, region->sy);
93 FIRE_RING(chan);
94 return 0;
95}
96
97int
98nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
99{
100 struct nouveau_fbdev *nfbdev = info->par;
101 struct drm_device *dev = nfbdev->dev;
102 struct drm_nouveau_private *dev_priv = dev->dev_private;
103 struct nouveau_channel *chan = dev_priv->channel;
104 uint32_t width, dwords, *data = (uint32_t *)image->data;
105 uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
106 uint32_t *palette = info->pseudo_palette;
107 int ret;
108
109 if (image->depth != 1)
110 return -ENODEV;
111
112 ret = RING_SPACE(chan, 11);
113 if (ret)
114 return ret;
115
116 width = ALIGN(image->width, 32);
117 dwords = (width * image->height) >> 5;
118
119 BEGIN_NVC0(chan, 2, NvSub2D, 0x0814, 2);
120 if (info->fix.visual == FB_VISUAL_TRUECOLOR ||
121 info->fix.visual == FB_VISUAL_DIRECTCOLOR) {
122 OUT_RING (chan, palette[image->bg_color] | mask);
123 OUT_RING (chan, palette[image->fg_color] | mask);
124 } else {
125 OUT_RING (chan, image->bg_color);
126 OUT_RING (chan, image->fg_color);
127 }
128 BEGIN_NVC0(chan, 2, NvSub2D, 0x0838, 2);
129 OUT_RING (chan, image->width);
130 OUT_RING (chan, image->height);
131 BEGIN_NVC0(chan, 2, NvSub2D, 0x0850, 4);
132 OUT_RING (chan, 0);
133 OUT_RING (chan, image->dx);
134 OUT_RING (chan, 0);
135 OUT_RING (chan, image->dy);
136
137 while (dwords) {
138 int push = dwords > 2047 ? 2047 : dwords;
139
140 ret = RING_SPACE(chan, push + 1);
141 if (ret)
142 return ret;
143
144 dwords -= push;
145
146 BEGIN_NVC0(chan, 6, NvSub2D, 0x0860, push);
147 OUT_RINGp(chan, data, push);
148 data += push;
149 }
150
151 FIRE_RING(chan);
152 return 0;
153}
154
155int
156nvc0_fbcon_accel_init(struct fb_info *info)
157{
158 struct nouveau_fbdev *nfbdev = info->par;
159 struct drm_device *dev = nfbdev->dev;
160 struct drm_nouveau_private *dev_priv = dev->dev_private;
161 struct nouveau_channel *chan = dev_priv->channel;
162 struct nouveau_bo *nvbo = nfbdev->nouveau_fb.nvbo;
163 int ret, format;
164
165 ret = nouveau_gpuobj_gr_new(chan, 0x902d, 0x902d);
166 if (ret)
167 return ret;
168
169 switch (info->var.bits_per_pixel) {
170 case 8:
171 format = 0xf3;
172 break;
173 case 15:
174 format = 0xf8;
175 break;
176 case 16:
177 format = 0xe8;
178 break;
179 case 32:
180 switch (info->var.transp.length) {
181 case 0: /* depth 24 */
182 case 8: /* depth 32, just use 24.. */
183 format = 0xe6;
184 break;
185 case 2: /* depth 30 */
186 format = 0xd1;
187 break;
188 default:
189 return -EINVAL;
190 }
191 break;
192 default:
193 return -EINVAL;
194 }
195
196 ret = RING_SPACE(chan, 60);
197 if (ret) {
198 WARN_ON(1);
199 nouveau_fbcon_gpu_lockup(info);
200 return ret;
201 }
202
203 BEGIN_NVC0(chan, 2, NvSub2D, 0x0000, 1);
204 OUT_RING (chan, 0x0000902d);
205 BEGIN_NVC0(chan, 2, NvSub2D, 0x0104, 2);
206 OUT_RING (chan, upper_32_bits(chan->notifier_bo->bo.offset));
207 OUT_RING (chan, lower_32_bits(chan->notifier_bo->bo.offset));
208 BEGIN_NVC0(chan, 2, NvSub2D, 0x0290, 1);
209 OUT_RING (chan, 0);
210 BEGIN_NVC0(chan, 2, NvSub2D, 0x0888, 1);
211 OUT_RING (chan, 1);
212 BEGIN_NVC0(chan, 2, NvSub2D, 0x02ac, 1);
213 OUT_RING (chan, 3);
214 BEGIN_NVC0(chan, 2, NvSub2D, 0x02a0, 1);
215 OUT_RING (chan, 0x55);
216 BEGIN_NVC0(chan, 2, NvSub2D, 0x08c0, 4);
217 OUT_RING (chan, 0);
218 OUT_RING (chan, 1);
219 OUT_RING (chan, 0);
220 OUT_RING (chan, 1);
221 BEGIN_NVC0(chan, 2, NvSub2D, 0x0580, 2);
222 OUT_RING (chan, 4);
223 OUT_RING (chan, format);
224 BEGIN_NVC0(chan, 2, NvSub2D, 0x02e8, 2);
225 OUT_RING (chan, 2);
226 OUT_RING (chan, 1);
227
228 BEGIN_NVC0(chan, 2, NvSub2D, 0x0804, 1);
229 OUT_RING (chan, format);
230 BEGIN_NVC0(chan, 2, NvSub2D, 0x0800, 1);
231 OUT_RING (chan, 1);
232 BEGIN_NVC0(chan, 2, NvSub2D, 0x0808, 3);
233 OUT_RING (chan, 0);
234 OUT_RING (chan, 0);
235 OUT_RING (chan, 1);
236 BEGIN_NVC0(chan, 2, NvSub2D, 0x081c, 1);
237 OUT_RING (chan, 1);
238 BEGIN_NVC0(chan, 2, NvSub2D, 0x0840, 4);
239 OUT_RING (chan, 0);
240 OUT_RING (chan, 1);
241 OUT_RING (chan, 0);
242 OUT_RING (chan, 1);
243 BEGIN_NVC0(chan, 2, NvSub2D, 0x0200, 10);
244 OUT_RING (chan, format);
245 OUT_RING (chan, 1);
246 OUT_RING (chan, 0);
247 OUT_RING (chan, 1);
248 OUT_RING (chan, 0);
249 OUT_RING (chan, info->fix.line_length);
250 OUT_RING (chan, info->var.xres_virtual);
251 OUT_RING (chan, info->var.yres_virtual);
252 OUT_RING (chan, upper_32_bits(nvbo->vma.offset));
253 OUT_RING (chan, lower_32_bits(nvbo->vma.offset));
254 BEGIN_NVC0(chan, 2, NvSub2D, 0x0230, 10);
255 OUT_RING (chan, format);
256 OUT_RING (chan, 1);
257 OUT_RING (chan, 0);
258 OUT_RING (chan, 1);
259 OUT_RING (chan, 0);
260 OUT_RING (chan, info->fix.line_length);
261 OUT_RING (chan, info->var.xres_virtual);
262 OUT_RING (chan, info->var.yres_virtual);
263 OUT_RING (chan, upper_32_bits(nvbo->vma.offset));
264 OUT_RING (chan, lower_32_bits(nvbo->vma.offset));
265 FIRE_RING (chan);
266
267 return 0;
268}
269
diff --git a/drivers/gpu/drm/nouveau/nvc0_fifo.c b/drivers/gpu/drm/nouveau/nvc0_fifo.c
index 890c2b95fbc1..e6f92c541dba 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fifo.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fifo.c
@@ -25,6 +25,49 @@
25#include "drmP.h" 25#include "drmP.h"
26 26
27#include "nouveau_drv.h" 27#include "nouveau_drv.h"
28#include "nouveau_mm.h"
29
30static void nvc0_fifo_isr(struct drm_device *);
31
32struct nvc0_fifo_priv {
33 struct nouveau_gpuobj *playlist[2];
34 int cur_playlist;
35 struct nouveau_vma user_vma;
36 int spoon_nr;
37};
38
39struct nvc0_fifo_chan {
40 struct nouveau_bo *user;
41 struct nouveau_gpuobj *ramfc;
42};
43
44static void
45nvc0_fifo_playlist_update(struct drm_device *dev)
46{
47 struct drm_nouveau_private *dev_priv = dev->dev_private;
48 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
49 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
50 struct nvc0_fifo_priv *priv = pfifo->priv;
51 struct nouveau_gpuobj *cur;
52 int i, p;
53
54 cur = priv->playlist[priv->cur_playlist];
55 priv->cur_playlist = !priv->cur_playlist;
56
57 for (i = 0, p = 0; i < 128; i++) {
58 if (!(nv_rd32(dev, 0x3004 + (i * 8)) & 1))
59 continue;
60 nv_wo32(cur, p + 0, i);
61 nv_wo32(cur, p + 4, 0x00000004);
62 p += 8;
63 }
64 pinstmem->flush(dev);
65
66 nv_wr32(dev, 0x002270, cur->vinst >> 12);
67 nv_wr32(dev, 0x002274, 0x01f00000 | (p >> 3));
68 if (!nv_wait(dev, 0x00227c, 0x00100000, 0x00000000))
69 NV_ERROR(dev, "PFIFO - playlist update failed\n");
70}
28 71
29void 72void
30nvc0_fifo_disable(struct drm_device *dev) 73nvc0_fifo_disable(struct drm_device *dev)
@@ -57,12 +100,135 @@ nvc0_fifo_channel_id(struct drm_device *dev)
57int 100int
58nvc0_fifo_create_context(struct nouveau_channel *chan) 101nvc0_fifo_create_context(struct nouveau_channel *chan)
59{ 102{
103 struct drm_device *dev = chan->dev;
104 struct drm_nouveau_private *dev_priv = dev->dev_private;
105 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
106 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
107 struct nvc0_fifo_priv *priv = pfifo->priv;
108 struct nvc0_fifo_chan *fifoch;
109 u64 ib_virt, user_vinst;
110 int ret;
111
112 chan->fifo_priv = kzalloc(sizeof(*fifoch), GFP_KERNEL);
113 if (!chan->fifo_priv)
114 return -ENOMEM;
115 fifoch = chan->fifo_priv;
116
117 /* allocate vram for control regs, map into polling area */
118 ret = nouveau_bo_new(dev, NULL, 0x1000, 0, TTM_PL_FLAG_VRAM,
119 0, 0, true, true, &fifoch->user);
120 if (ret)
121 goto error;
122
123 ret = nouveau_bo_pin(fifoch->user, TTM_PL_FLAG_VRAM);
124 if (ret) {
125 nouveau_bo_ref(NULL, &fifoch->user);
126 goto error;
127 }
128
129 user_vinst = fifoch->user->bo.mem.start << PAGE_SHIFT;
130
131 ret = nouveau_bo_map(fifoch->user);
132 if (ret) {
133 nouveau_bo_unpin(fifoch->user);
134 nouveau_bo_ref(NULL, &fifoch->user);
135 goto error;
136 }
137
138 nouveau_vm_map_at(&priv->user_vma, chan->id * 0x1000,
139 fifoch->user->bo.mem.mm_node);
140
141 chan->user = ioremap_wc(pci_resource_start(dev->pdev, 1) +
142 priv->user_vma.offset + (chan->id * 0x1000),
143 PAGE_SIZE);
144 if (!chan->user) {
145 ret = -ENOMEM;
146 goto error;
147 }
148
149 ib_virt = chan->pushbuf_base + chan->dma.ib_base * 4;
150
151 /* zero channel regs */
152 nouveau_bo_wr32(fifoch->user, 0x0040/4, 0);
153 nouveau_bo_wr32(fifoch->user, 0x0044/4, 0);
154 nouveau_bo_wr32(fifoch->user, 0x0048/4, 0);
155 nouveau_bo_wr32(fifoch->user, 0x004c/4, 0);
156 nouveau_bo_wr32(fifoch->user, 0x0050/4, 0);
157 nouveau_bo_wr32(fifoch->user, 0x0058/4, 0);
158 nouveau_bo_wr32(fifoch->user, 0x005c/4, 0);
159 nouveau_bo_wr32(fifoch->user, 0x0060/4, 0);
160 nouveau_bo_wr32(fifoch->user, 0x0088/4, 0);
161 nouveau_bo_wr32(fifoch->user, 0x008c/4, 0);
162
163 /* ramfc */
164 ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst,
165 chan->ramin->vinst, 0x100,
166 NVOBJ_FLAG_ZERO_ALLOC, &fifoch->ramfc);
167 if (ret)
168 goto error;
169
170 nv_wo32(fifoch->ramfc, 0x08, lower_32_bits(user_vinst));
171 nv_wo32(fifoch->ramfc, 0x0c, upper_32_bits(user_vinst));
172 nv_wo32(fifoch->ramfc, 0x10, 0x0000face);
173 nv_wo32(fifoch->ramfc, 0x30, 0xfffff902);
174 nv_wo32(fifoch->ramfc, 0x48, lower_32_bits(ib_virt));
175 nv_wo32(fifoch->ramfc, 0x4c, drm_order(chan->dma.ib_max + 1) << 16 |
176 upper_32_bits(ib_virt));
177 nv_wo32(fifoch->ramfc, 0x54, 0x00000002);
178 nv_wo32(fifoch->ramfc, 0x84, 0x20400000);
179 nv_wo32(fifoch->ramfc, 0x94, 0x30000001);
180 nv_wo32(fifoch->ramfc, 0x9c, 0x00000100);
181 nv_wo32(fifoch->ramfc, 0xa4, 0x1f1f1f1f);
182 nv_wo32(fifoch->ramfc, 0xa8, 0x1f1f1f1f);
183 nv_wo32(fifoch->ramfc, 0xac, 0x0000001f);
184 nv_wo32(fifoch->ramfc, 0xb8, 0xf8000000);
185 nv_wo32(fifoch->ramfc, 0xf8, 0x10003080); /* 0x002310 */
186 nv_wo32(fifoch->ramfc, 0xfc, 0x10000010); /* 0x002350 */
187 pinstmem->flush(dev);
188
189 nv_wr32(dev, 0x003000 + (chan->id * 8), 0xc0000000 |
190 (chan->ramin->vinst >> 12));
191 nv_wr32(dev, 0x003004 + (chan->id * 8), 0x001f0001);
192 nvc0_fifo_playlist_update(dev);
60 return 0; 193 return 0;
194
195error:
196 pfifo->destroy_context(chan);
197 return ret;
61} 198}
62 199
63void 200void
64nvc0_fifo_destroy_context(struct nouveau_channel *chan) 201nvc0_fifo_destroy_context(struct nouveau_channel *chan)
65{ 202{
203 struct drm_device *dev = chan->dev;
204 struct nvc0_fifo_chan *fifoch;
205
206 nv_mask(dev, 0x003004 + (chan->id * 8), 0x00000001, 0x00000000);
207 nv_wr32(dev, 0x002634, chan->id);
208 if (!nv_wait(dev, 0x0002634, 0xffffffff, chan->id))
209 NV_WARN(dev, "0x2634 != chid: 0x%08x\n", nv_rd32(dev, 0x2634));
210
211 nvc0_fifo_playlist_update(dev);
212
213 nv_wr32(dev, 0x003000 + (chan->id * 8), 0x00000000);
214
215 if (chan->user) {
216 iounmap(chan->user);
217 chan->user = NULL;
218 }
219
220 fifoch = chan->fifo_priv;
221 chan->fifo_priv = NULL;
222 if (!fifoch)
223 return;
224
225 nouveau_gpuobj_ref(NULL, &fifoch->ramfc);
226 if (fifoch->user) {
227 nouveau_bo_unmap(fifoch->user);
228 nouveau_bo_unpin(fifoch->user);
229 nouveau_bo_ref(NULL, &fifoch->user);
230 }
231 kfree(fifoch);
66} 232}
67 233
68int 234int
@@ -77,14 +243,213 @@ nvc0_fifo_unload_context(struct drm_device *dev)
77 return 0; 243 return 0;
78} 244}
79 245
246static void
247nvc0_fifo_destroy(struct drm_device *dev)
248{
249 struct drm_nouveau_private *dev_priv = dev->dev_private;
250 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
251 struct nvc0_fifo_priv *priv;
252
253 priv = pfifo->priv;
254 if (!priv)
255 return;
256
257 nouveau_vm_put(&priv->user_vma);
258 nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
259 nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
260 kfree(priv);
261}
262
80void 263void
81nvc0_fifo_takedown(struct drm_device *dev) 264nvc0_fifo_takedown(struct drm_device *dev)
82{ 265{
266 nv_wr32(dev, 0x002140, 0x00000000);
267 nvc0_fifo_destroy(dev);
268}
269
270static int
271nvc0_fifo_create(struct drm_device *dev)
272{
273 struct drm_nouveau_private *dev_priv = dev->dev_private;
274 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
275 struct nvc0_fifo_priv *priv;
276 int ret;
277
278 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
279 if (!priv)
280 return -ENOMEM;
281 pfifo->priv = priv;
282
283 ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000, 0,
284 &priv->playlist[0]);
285 if (ret)
286 goto error;
287
288 ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000, 0,
289 &priv->playlist[1]);
290 if (ret)
291 goto error;
292
293 ret = nouveau_vm_get(dev_priv->bar1_vm, pfifo->channels * 0x1000,
294 12, NV_MEM_ACCESS_RW, &priv->user_vma);
295 if (ret)
296 goto error;
297
298 nouveau_irq_register(dev, 8, nvc0_fifo_isr);
299 NVOBJ_CLASS(dev, 0x506e, SW); /* nvsw */
300 return 0;
301
302error:
303 nvc0_fifo_destroy(dev);
304 return ret;
83} 305}
84 306
85int 307int
86nvc0_fifo_init(struct drm_device *dev) 308nvc0_fifo_init(struct drm_device *dev)
87{ 309{
310 struct drm_nouveau_private *dev_priv = dev->dev_private;
311 struct nouveau_fifo_engine *pfifo = &dev_priv->engine.fifo;
312 struct nvc0_fifo_priv *priv;
313 int ret, i;
314
315 if (!pfifo->priv) {
316 ret = nvc0_fifo_create(dev);
317 if (ret)
318 return ret;
319 }
320 priv = pfifo->priv;
321
322 /* reset PFIFO, enable all available PSUBFIFO areas */
323 nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
324 nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
325 nv_wr32(dev, 0x000204, 0xffffffff);
326 nv_wr32(dev, 0x002204, 0xffffffff);
327
328 priv->spoon_nr = hweight32(nv_rd32(dev, 0x002204));
329 NV_DEBUG(dev, "PFIFO: %d subfifo(s)\n", priv->spoon_nr);
330
331 /* assign engines to subfifos */
332 if (priv->spoon_nr >= 3) {
333 nv_wr32(dev, 0x002208, ~(1 << 0)); /* PGRAPH */
334 nv_wr32(dev, 0x00220c, ~(1 << 1)); /* PVP */
335 nv_wr32(dev, 0x002210, ~(1 << 1)); /* PPP */
336 nv_wr32(dev, 0x002214, ~(1 << 1)); /* PBSP */
337 nv_wr32(dev, 0x002218, ~(1 << 2)); /* PCE0 */
338 nv_wr32(dev, 0x00221c, ~(1 << 1)); /* PCE1 */
339 }
340
341 /* PSUBFIFO[n] */
342 for (i = 0; i < 3; i++) {
343 nv_mask(dev, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
344 nv_wr32(dev, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
345 nv_wr32(dev, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTR_EN */
346 }
347
348 nv_mask(dev, 0x002200, 0x00000001, 0x00000001);
349 nv_wr32(dev, 0x002254, 0x10000000 | priv->user_vma.offset >> 12);
350
351 nv_wr32(dev, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
352 nv_wr32(dev, 0x002100, 0xffffffff);
353 nv_wr32(dev, 0x002140, 0xbfffffff);
88 return 0; 354 return 0;
89} 355}
90 356
357struct nouveau_enum nvc0_fifo_fault_unit[] = {
358 { 0, "PGRAPH" },
359 { 3, "PEEPHOLE" },
360 { 4, "BAR1" },
361 { 5, "BAR3" },
362 { 7, "PFIFO" },
363 {}
364};
365
366struct nouveau_enum nvc0_fifo_fault_reason[] = {
367 { 0, "PT_NOT_PRESENT" },
368 { 1, "PT_TOO_SHORT" },
369 { 2, "PAGE_NOT_PRESENT" },
370 { 3, "VM_LIMIT_EXCEEDED" },
371 {}
372};
373
374struct nouveau_bitfield nvc0_fifo_subfifo_intr[] = {
375/* { 0x00008000, "" } seen with null ib push */
376 { 0x00200000, "ILLEGAL_MTHD" },
377 { 0x00800000, "EMPTY_SUBC" },
378 {}
379};
380
381static void
382nvc0_fifo_isr_vm_fault(struct drm_device *dev, int unit)
383{
384 u32 inst = nv_rd32(dev, 0x2800 + (unit * 0x10));
385 u32 valo = nv_rd32(dev, 0x2804 + (unit * 0x10));
386 u32 vahi = nv_rd32(dev, 0x2808 + (unit * 0x10));
387 u32 stat = nv_rd32(dev, 0x280c + (unit * 0x10));
388
389 NV_INFO(dev, "PFIFO: %s fault at 0x%010llx [",
390 (stat & 0x00000080) ? "write" : "read", (u64)vahi << 32 | valo);
391 nouveau_enum_print(nvc0_fifo_fault_reason, stat & 0x0000000f);
392 printk("] from ");
393 nouveau_enum_print(nvc0_fifo_fault_unit, unit);
394 printk(" on channel 0x%010llx\n", (u64)inst << 12);
395}
396
397static void
398nvc0_fifo_isr_subfifo_intr(struct drm_device *dev, int unit)
399{
400 u32 stat = nv_rd32(dev, 0x040108 + (unit * 0x2000));
401 u32 addr = nv_rd32(dev, 0x0400c0 + (unit * 0x2000));
402 u32 data = nv_rd32(dev, 0x0400c4 + (unit * 0x2000));
403 u32 chid = nv_rd32(dev, 0x040120 + (unit * 0x2000)) & 0x7f;
404 u32 subc = (addr & 0x00070000);
405 u32 mthd = (addr & 0x00003ffc);
406
407 NV_INFO(dev, "PSUBFIFO %d:", unit);
408 nouveau_bitfield_print(nvc0_fifo_subfifo_intr, stat);
409 NV_INFO(dev, "PSUBFIFO %d: ch %d subc %d mthd 0x%04x data 0x%08x\n",
410 unit, chid, subc, mthd, data);
411
412 nv_wr32(dev, 0x0400c0 + (unit * 0x2000), 0x80600008);
413 nv_wr32(dev, 0x040108 + (unit * 0x2000), stat);
414}
415
416static void
417nvc0_fifo_isr(struct drm_device *dev)
418{
419 u32 stat = nv_rd32(dev, 0x002100);
420
421 if (stat & 0x10000000) {
422 u32 units = nv_rd32(dev, 0x00259c);
423 u32 u = units;
424
425 while (u) {
426 int i = ffs(u) - 1;
427 nvc0_fifo_isr_vm_fault(dev, i);
428 u &= ~(1 << i);
429 }
430
431 nv_wr32(dev, 0x00259c, units);
432 stat &= ~0x10000000;
433 }
434
435 if (stat & 0x20000000) {
436 u32 units = nv_rd32(dev, 0x0025a0);
437 u32 u = units;
438
439 while (u) {
440 int i = ffs(u) - 1;
441 nvc0_fifo_isr_subfifo_intr(dev, i);
442 u &= ~(1 << i);
443 }
444
445 nv_wr32(dev, 0x0025a0, units);
446 stat &= ~0x20000000;
447 }
448
449 if (stat) {
450 NV_INFO(dev, "PFIFO: unhandled status 0x%08x\n", stat);
451 nv_wr32(dev, 0x002100, stat);
452 }
453
454 nv_wr32(dev, 0x2140, 0);
455}
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.c b/drivers/gpu/drm/nouveau/nvc0_graph.c
index 717a5177a8d8..5feacd5d5fa4 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.c
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.c
@@ -22,9 +22,16 @@
22 * Authors: Ben Skeggs 22 * Authors: Ben Skeggs
23 */ 23 */
24 24
25#include <linux/firmware.h>
26
25#include "drmP.h" 27#include "drmP.h"
26 28
27#include "nouveau_drv.h" 29#include "nouveau_drv.h"
30#include "nouveau_mm.h"
31#include "nvc0_graph.h"
32
33static void nvc0_graph_isr(struct drm_device *);
34static int nvc0_graph_unload_context_to(struct drm_device *dev, u64 chan);
28 35
29void 36void
30nvc0_graph_fifo_access(struct drm_device *dev, bool enabled) 37nvc0_graph_fifo_access(struct drm_device *dev, bool enabled)
@@ -37,39 +44,735 @@ nvc0_graph_channel(struct drm_device *dev)
37 return NULL; 44 return NULL;
38} 45}
39 46
47static int
48nvc0_graph_construct_context(struct nouveau_channel *chan)
49{
50 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
51 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
52 struct nvc0_graph_chan *grch = chan->pgraph_ctx;
53 struct drm_device *dev = chan->dev;
54 int ret, i;
55 u32 *ctx;
56
57 ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
58 if (!ctx)
59 return -ENOMEM;
60
61 nvc0_graph_load_context(chan);
62
63 nv_wo32(grch->grctx, 0x1c, 1);
64 nv_wo32(grch->grctx, 0x20, 0);
65 nv_wo32(grch->grctx, 0x28, 0);
66 nv_wo32(grch->grctx, 0x2c, 0);
67 dev_priv->engine.instmem.flush(dev);
68
69 ret = nvc0_grctx_generate(chan);
70 if (ret) {
71 kfree(ctx);
72 return ret;
73 }
74
75 ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
76 if (ret) {
77 kfree(ctx);
78 return ret;
79 }
80
81 for (i = 0; i < priv->grctx_size; i += 4)
82 ctx[i / 4] = nv_ro32(grch->grctx, i);
83
84 priv->grctx_vals = ctx;
85 return 0;
86}
87
88static int
89nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
90{
91 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
92 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
93 struct nvc0_graph_chan *grch = chan->pgraph_ctx;
94 struct drm_device *dev = chan->dev;
95 int i = 0, gpc, tp, ret;
96 u32 magic;
97
98 ret = nouveau_gpuobj_new(dev, NULL, 0x2000, 256, NVOBJ_FLAG_VM,
99 &grch->unk408004);
100 if (ret)
101 return ret;
102
103 ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 256, NVOBJ_FLAG_VM,
104 &grch->unk40800c);
105 if (ret)
106 return ret;
107
108 ret = nouveau_gpuobj_new(dev, NULL, 384 * 1024, 4096, NVOBJ_FLAG_VM,
109 &grch->unk418810);
110 if (ret)
111 return ret;
112
113 ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0, NVOBJ_FLAG_VM,
114 &grch->mmio);
115 if (ret)
116 return ret;
117
118
119 nv_wo32(grch->mmio, i++ * 4, 0x00408004);
120 nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
121 nv_wo32(grch->mmio, i++ * 4, 0x00408008);
122 nv_wo32(grch->mmio, i++ * 4, 0x80000018);
123
124 nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
125 nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
126 nv_wo32(grch->mmio, i++ * 4, 0x00408010);
127 nv_wo32(grch->mmio, i++ * 4, 0x80000000);
128
129 nv_wo32(grch->mmio, i++ * 4, 0x00418810);
130 nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->vinst >> 12);
131 nv_wo32(grch->mmio, i++ * 4, 0x00419848);
132 nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->vinst >> 12);
133
134 nv_wo32(grch->mmio, i++ * 4, 0x00419004);
135 nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->vinst >> 8);
136 nv_wo32(grch->mmio, i++ * 4, 0x00419008);
137 nv_wo32(grch->mmio, i++ * 4, 0x00000000);
138
139 nv_wo32(grch->mmio, i++ * 4, 0x00418808);
140 nv_wo32(grch->mmio, i++ * 4, grch->unk408004->vinst >> 8);
141 nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
142 nv_wo32(grch->mmio, i++ * 4, 0x80000018);
143
144 magic = 0x02180000;
145 nv_wo32(grch->mmio, i++ * 4, 0x00405830);
146 nv_wo32(grch->mmio, i++ * 4, magic);
147 for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
148 for (tp = 0; tp < priv->tp_nr[gpc]; tp++, magic += 0x02fc) {
149 u32 reg = 0x504520 + (gpc * 0x8000) + (tp * 0x0800);
150 nv_wo32(grch->mmio, i++ * 4, reg);
151 nv_wo32(grch->mmio, i++ * 4, magic);
152 }
153 }
154
155 grch->mmio_nr = i / 2;
156 return 0;
157}
158
40int 159int
41nvc0_graph_create_context(struct nouveau_channel *chan) 160nvc0_graph_create_context(struct nouveau_channel *chan)
42{ 161{
162 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
163 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
164 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
165 struct nvc0_graph_priv *priv = pgraph->priv;
166 struct nvc0_graph_chan *grch;
167 struct drm_device *dev = chan->dev;
168 struct nouveau_gpuobj *grctx;
169 int ret, i;
170
171 chan->pgraph_ctx = kzalloc(sizeof(*grch), GFP_KERNEL);
172 if (!chan->pgraph_ctx)
173 return -ENOMEM;
174 grch = chan->pgraph_ctx;
175
176 ret = nouveau_gpuobj_new(dev, NULL, priv->grctx_size, 256,
177 NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
178 &grch->grctx);
179 if (ret)
180 goto error;
181 chan->ramin_grctx = grch->grctx;
182 grctx = grch->grctx;
183
184 ret = nvc0_graph_create_context_mmio_list(chan);
185 if (ret)
186 goto error;
187
188 nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->vinst) | 4);
189 nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->vinst));
190 pinstmem->flush(dev);
191
192 if (!priv->grctx_vals) {
193 ret = nvc0_graph_construct_context(chan);
194 if (ret)
195 goto error;
196 }
197
198 for (i = 0; i < priv->grctx_size; i += 4)
199 nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
200
201 nv_wo32(grctx, 0xf4, 0);
202 nv_wo32(grctx, 0xf8, 0);
203 nv_wo32(grctx, 0x10, grch->mmio_nr);
204 nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->vinst));
205 nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->vinst));
206 nv_wo32(grctx, 0x1c, 1);
207 nv_wo32(grctx, 0x20, 0);
208 nv_wo32(grctx, 0x28, 0);
209 nv_wo32(grctx, 0x2c, 0);
210 pinstmem->flush(dev);
43 return 0; 211 return 0;
212
213error:
214 pgraph->destroy_context(chan);
215 return ret;
44} 216}
45 217
46void 218void
47nvc0_graph_destroy_context(struct nouveau_channel *chan) 219nvc0_graph_destroy_context(struct nouveau_channel *chan)
48{ 220{
221 struct nvc0_graph_chan *grch;
222
223 grch = chan->pgraph_ctx;
224 chan->pgraph_ctx = NULL;
225 if (!grch)
226 return;
227
228 nouveau_gpuobj_ref(NULL, &grch->mmio);
229 nouveau_gpuobj_ref(NULL, &grch->unk418810);
230 nouveau_gpuobj_ref(NULL, &grch->unk40800c);
231 nouveau_gpuobj_ref(NULL, &grch->unk408004);
232 nouveau_gpuobj_ref(NULL, &grch->grctx);
233 chan->ramin_grctx = NULL;
49} 234}
50 235
51int 236int
52nvc0_graph_load_context(struct nouveau_channel *chan) 237nvc0_graph_load_context(struct nouveau_channel *chan)
53{ 238{
239 struct drm_device *dev = chan->dev;
240
241 nv_wr32(dev, 0x409840, 0x00000030);
242 nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
243 nv_wr32(dev, 0x409504, 0x00000003);
244 if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
245 NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
246
247 return 0;
248}
249
250static int
251nvc0_graph_unload_context_to(struct drm_device *dev, u64 chan)
252{
253 nv_wr32(dev, 0x409840, 0x00000003);
254 nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
255 nv_wr32(dev, 0x409504, 0x00000009);
256 if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
257 NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
258 return -EBUSY;
259 }
260
54 return 0; 261 return 0;
55} 262}
56 263
57int 264int
58nvc0_graph_unload_context(struct drm_device *dev) 265nvc0_graph_unload_context(struct drm_device *dev)
59{ 266{
60 return 0; 267 u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
268 return nvc0_graph_unload_context_to(dev, inst);
269}
270
271static void
272nvc0_graph_destroy(struct drm_device *dev)
273{
274 struct drm_nouveau_private *dev_priv = dev->dev_private;
275 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
276 struct nvc0_graph_priv *priv;
277
278 priv = pgraph->priv;
279 if (!priv)
280 return;
281
282 nouveau_irq_unregister(dev, 12);
283
284 nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
285 nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
286
287 if (priv->grctx_vals)
288 kfree(priv->grctx_vals);
289 kfree(priv);
61} 290}
62 291
63void 292void
64nvc0_graph_takedown(struct drm_device *dev) 293nvc0_graph_takedown(struct drm_device *dev)
65{ 294{
295 nvc0_graph_destroy(dev);
296}
297
298static int
299nvc0_graph_create(struct drm_device *dev)
300{
301 struct drm_nouveau_private *dev_priv = dev->dev_private;
302 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
303 struct nvc0_graph_priv *priv;
304 int ret, gpc, i;
305
306 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
307 if (!priv)
308 return -ENOMEM;
309 pgraph->priv = priv;
310
311 ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
312 if (ret)
313 goto error;
314
315 ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
316 if (ret)
317 goto error;
318
319 for (i = 0; i < 0x1000; i += 4) {
320 nv_wo32(priv->unk4188b4, i, 0x00000010);
321 nv_wo32(priv->unk4188b8, i, 0x00000010);
322 }
323
324 priv->gpc_nr = nv_rd32(dev, 0x409604) & 0x0000001f;
325 priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
326 for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
327 priv->tp_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
328 priv->tp_total += priv->tp_nr[gpc];
329 }
330
331 /*XXX: these need figuring out... */
332 switch (dev_priv->chipset) {
333 case 0xc0:
334 if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
335 priv->magic_not_rop_nr = 0x07;
336 /* filled values up to tp_total, the rest 0 */
337 priv->magicgpc980[0] = 0x22111000;
338 priv->magicgpc980[1] = 0x00000233;
339 priv->magicgpc980[2] = 0x00000000;
340 priv->magicgpc980[3] = 0x00000000;
341 priv->magicgpc918 = 0x000ba2e9;
342 } else
343 if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
344 priv->magic_not_rop_nr = 0x05;
345 priv->magicgpc980[0] = 0x11110000;
346 priv->magicgpc980[1] = 0x00233222;
347 priv->magicgpc980[2] = 0x00000000;
348 priv->magicgpc980[3] = 0x00000000;
349 priv->magicgpc918 = 0x00092493;
350 } else
351 if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
352 priv->magic_not_rop_nr = 0x06;
353 priv->magicgpc980[0] = 0x11110000;
354 priv->magicgpc980[1] = 0x03332222;
355 priv->magicgpc980[2] = 0x00000000;
356 priv->magicgpc980[3] = 0x00000000;
357 priv->magicgpc918 = 0x00088889;
358 }
359 break;
360 case 0xc3: /* 450, 4/0/0/0, 2 */
361 priv->magic_not_rop_nr = 0x03;
362 priv->magicgpc980[0] = 0x00003210;
363 priv->magicgpc980[1] = 0x00000000;
364 priv->magicgpc980[2] = 0x00000000;
365 priv->magicgpc980[3] = 0x00000000;
366 priv->magicgpc918 = 0x00200000;
367 break;
368 case 0xc4: /* 460, 3/4/0/0, 4 */
369 priv->magic_not_rop_nr = 0x01;
370 priv->magicgpc980[0] = 0x02321100;
371 priv->magicgpc980[1] = 0x00000000;
372 priv->magicgpc980[2] = 0x00000000;
373 priv->magicgpc980[3] = 0x00000000;
374 priv->magicgpc918 = 0x00124925;
375 break;
376 }
377
378 if (!priv->magic_not_rop_nr) {
379 NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
380 priv->tp_nr[0], priv->tp_nr[1], priv->tp_nr[2],
381 priv->tp_nr[3], priv->rop_nr);
382 /* use 0xc3's values... */
383 priv->magic_not_rop_nr = 0x03;
384 priv->magicgpc980[0] = 0x00003210;
385 priv->magicgpc980[1] = 0x00000000;
386 priv->magicgpc980[2] = 0x00000000;
387 priv->magicgpc980[3] = 0x00000000;
388 priv->magicgpc918 = 0x00200000;
389 }
390
391 nouveau_irq_register(dev, 12, nvc0_graph_isr);
392 NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
393 NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
394 NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
395 NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
396 return 0;
397
398error:
399 nvc0_graph_destroy(dev);
400 return ret;
401}
402
403static void
404nvc0_graph_init_obj418880(struct drm_device *dev)
405{
406 struct drm_nouveau_private *dev_priv = dev->dev_private;
407 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
408 struct nvc0_graph_priv *priv = pgraph->priv;
409 int i;
410
411 nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
412 nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
413 for (i = 0; i < 4; i++)
414 nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
415 nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
416 nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
417}
418
419static void
420nvc0_graph_init_regs(struct drm_device *dev)
421{
422 nv_wr32(dev, 0x400080, 0x003083c2);
423 nv_wr32(dev, 0x400088, 0x00006fe7);
424 nv_wr32(dev, 0x40008c, 0x00000000);
425 nv_wr32(dev, 0x400090, 0x00000030);
426 nv_wr32(dev, 0x40013c, 0x013901f7);
427 nv_wr32(dev, 0x400140, 0x00000100);
428 nv_wr32(dev, 0x400144, 0x00000000);
429 nv_wr32(dev, 0x400148, 0x00000110);
430 nv_wr32(dev, 0x400138, 0x00000000);
431 nv_wr32(dev, 0x400130, 0x00000000);
432 nv_wr32(dev, 0x400134, 0x00000000);
433 nv_wr32(dev, 0x400124, 0x00000002);
434}
435
436static void
437nvc0_graph_init_gpc_0(struct drm_device *dev)
438{
439 struct drm_nouveau_private *dev_priv = dev->dev_private;
440 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
441 int gpc;
442
443 // TP ROP UNKVAL(magic_not_rop_nr)
444 // 450: 4/0/0/0 2 3
445 // 460: 3/4/0/0 4 1
446 // 465: 3/4/4/0 4 7
447 // 470: 3/3/4/4 5 5
448 // 480: 3/4/4/4 6 6
449
450 // magicgpc918
451 // 450: 00200000 00000000001000000000000000000000
452 // 460: 00124925 00000000000100100100100100100101
453 // 465: 000ba2e9 00000000000010111010001011101001
454 // 470: 00092493 00000000000010010010010010010011
455 // 480: 00088889 00000000000010001000100010001001
456
457 /* filled values up to tp_total, remainder 0 */
458 // 450: 00003210 00000000 00000000 00000000
459 // 460: 02321100 00000000 00000000 00000000
460 // 465: 22111000 00000233 00000000 00000000
461 // 470: 11110000 00233222 00000000 00000000
462 // 480: 11110000 03332222 00000000 00000000
463
464 nv_wr32(dev, GPC_BCAST(0x0980), priv->magicgpc980[0]);
465 nv_wr32(dev, GPC_BCAST(0x0984), priv->magicgpc980[1]);
466 nv_wr32(dev, GPC_BCAST(0x0988), priv->magicgpc980[2]);
467 nv_wr32(dev, GPC_BCAST(0x098c), priv->magicgpc980[3]);
468
469 for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
470 nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
471 priv->tp_nr[gpc]);
472 nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
473 nv_wr32(dev, GPC_UNIT(gpc, 0x0918), priv->magicgpc918);
474 }
475
476 nv_wr32(dev, GPC_BCAST(0x1bd4), priv->magicgpc918);
477 nv_wr32(dev, GPC_BCAST(0x08ac), priv->rop_nr);
478}
479
480static void
481nvc0_graph_init_units(struct drm_device *dev)
482{
483 nv_wr32(dev, 0x409c24, 0x000f0000);
484 nv_wr32(dev, 0x404000, 0xc0000000); /* DISPATCH */
485 nv_wr32(dev, 0x404600, 0xc0000000); /* M2MF */
486 nv_wr32(dev, 0x408030, 0xc0000000);
487 nv_wr32(dev, 0x40601c, 0xc0000000);
488 nv_wr32(dev, 0x404490, 0xc0000000); /* MACRO */
489 nv_wr32(dev, 0x406018, 0xc0000000);
490 nv_wr32(dev, 0x405840, 0xc0000000);
491 nv_wr32(dev, 0x405844, 0x00ffffff);
492 nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
493 nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
494}
495
496static void
497nvc0_graph_init_gpc_1(struct drm_device *dev)
498{
499 struct drm_nouveau_private *dev_priv = dev->dev_private;
500 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
501 int gpc, tp;
502
503 for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
504 nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
505 nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
506 nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
507 nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
508 for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
509 nv_wr32(dev, TP_UNIT(gpc, tp, 0x508), 0xffffffff);
510 nv_wr32(dev, TP_UNIT(gpc, tp, 0x50c), 0xffffffff);
511 nv_wr32(dev, TP_UNIT(gpc, tp, 0x224), 0xc0000000);
512 nv_wr32(dev, TP_UNIT(gpc, tp, 0x48c), 0xc0000000);
513 nv_wr32(dev, TP_UNIT(gpc, tp, 0x084), 0xc0000000);
514 nv_wr32(dev, TP_UNIT(gpc, tp, 0xe44), 0x001ffffe);
515 nv_wr32(dev, TP_UNIT(gpc, tp, 0xe4c), 0x0000000f);
516 }
517 nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
518 nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
519 }
520}
521
522static void
523nvc0_graph_init_rop(struct drm_device *dev)
524{
525 struct drm_nouveau_private *dev_priv = dev->dev_private;
526 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
527 int rop;
528
529 for (rop = 0; rop < priv->rop_nr; rop++) {
530 nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
531 nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
532 nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
533 nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
534 }
535}
536
537static int
538nvc0_fuc_load_fw(struct drm_device *dev, u32 fuc_base,
539 const char *code_fw, const char *data_fw)
540{
541 const struct firmware *fw;
542 char name[32];
543 int ret, i;
544
545 snprintf(name, sizeof(name), "nouveau/%s", data_fw);
546 ret = request_firmware(&fw, name, &dev->pdev->dev);
547 if (ret) {
548 NV_ERROR(dev, "failed to load %s\n", data_fw);
549 return ret;
550 }
551
552 nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
553 for (i = 0; i < fw->size / 4; i++)
554 nv_wr32(dev, fuc_base + 0x01c4, ((u32 *)fw->data)[i]);
555 release_firmware(fw);
556
557 snprintf(name, sizeof(name), "nouveau/%s", code_fw);
558 ret = request_firmware(&fw, name, &dev->pdev->dev);
559 if (ret) {
560 NV_ERROR(dev, "failed to load %s\n", code_fw);
561 return ret;
562 }
563
564 nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
565 for (i = 0; i < fw->size / 4; i++) {
566 if ((i & 0x3f) == 0)
567 nv_wr32(dev, fuc_base + 0x0188, i >> 6);
568 nv_wr32(dev, fuc_base + 0x0184, ((u32 *)fw->data)[i]);
569 }
570 release_firmware(fw);
571
572 return 0;
573}
574
575static int
576nvc0_graph_init_ctxctl(struct drm_device *dev)
577{
578 struct drm_nouveau_private *dev_priv = dev->dev_private;
579 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
580 u32 r000260;
581 int ret;
582
583 /* load fuc microcode */
584 r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
585 ret = nvc0_fuc_load_fw(dev, 0x409000, "fuc409c", "fuc409d");
586 if (ret == 0)
587 ret = nvc0_fuc_load_fw(dev, 0x41a000, "fuc41ac", "fuc41ad");
588 nv_wr32(dev, 0x000260, r000260);
589
590 if (ret)
591 return ret;
592
593 /* start both of them running */
594 nv_wr32(dev, 0x409840, 0xffffffff);
595 nv_wr32(dev, 0x41a10c, 0x00000000);
596 nv_wr32(dev, 0x40910c, 0x00000000);
597 nv_wr32(dev, 0x41a100, 0x00000002);
598 nv_wr32(dev, 0x409100, 0x00000002);
599 if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
600 NV_INFO(dev, "0x409800 wait failed\n");
601
602 nv_wr32(dev, 0x409840, 0xffffffff);
603 nv_wr32(dev, 0x409500, 0x7fffffff);
604 nv_wr32(dev, 0x409504, 0x00000021);
605
606 nv_wr32(dev, 0x409840, 0xffffffff);
607 nv_wr32(dev, 0x409500, 0x00000000);
608 nv_wr32(dev, 0x409504, 0x00000010);
609 if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
610 NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
611 return -EBUSY;
612 }
613 priv->grctx_size = nv_rd32(dev, 0x409800);
614
615 nv_wr32(dev, 0x409840, 0xffffffff);
616 nv_wr32(dev, 0x409500, 0x00000000);
617 nv_wr32(dev, 0x409504, 0x00000016);
618 if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
619 NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
620 return -EBUSY;
621 }
622
623 nv_wr32(dev, 0x409840, 0xffffffff);
624 nv_wr32(dev, 0x409500, 0x00000000);
625 nv_wr32(dev, 0x409504, 0x00000025);
626 if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
627 NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
628 return -EBUSY;
629 }
630
631 return 0;
66} 632}
67 633
68int 634int
69nvc0_graph_init(struct drm_device *dev) 635nvc0_graph_init(struct drm_device *dev)
70{ 636{
71 struct drm_nouveau_private *dev_priv = dev->dev_private; 637 struct drm_nouveau_private *dev_priv = dev->dev_private;
638 struct nouveau_pgraph_engine *pgraph = &dev_priv->engine.graph;
639 struct nvc0_graph_priv *priv;
640 int ret;
641
72 dev_priv->engine.graph.accel_blocked = true; 642 dev_priv->engine.graph.accel_blocked = true;
643
644 switch (dev_priv->chipset) {
645 case 0xc0:
646 case 0xc3:
647 case 0xc4:
648 break;
649 default:
650 NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
651 if (nouveau_noaccel != 0)
652 return 0;
653 break;
654 }
655
656 nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
657 nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
658
659 if (!pgraph->priv) {
660 ret = nvc0_graph_create(dev);
661 if (ret)
662 return ret;
663 }
664 priv = pgraph->priv;
665
666 nvc0_graph_init_obj418880(dev);
667 nvc0_graph_init_regs(dev);
668 //nvc0_graph_init_unitplemented_magics(dev);
669 nvc0_graph_init_gpc_0(dev);
670 //nvc0_graph_init_unitplemented_c242(dev);
671
672 nv_wr32(dev, 0x400500, 0x00010001);
673 nv_wr32(dev, 0x400100, 0xffffffff);
674 nv_wr32(dev, 0x40013c, 0xffffffff);
675
676 nvc0_graph_init_units(dev);
677 nvc0_graph_init_gpc_1(dev);
678 nvc0_graph_init_rop(dev);
679
680 nv_wr32(dev, 0x400108, 0xffffffff);
681 nv_wr32(dev, 0x400138, 0xffffffff);
682 nv_wr32(dev, 0x400118, 0xffffffff);
683 nv_wr32(dev, 0x400130, 0xffffffff);
684 nv_wr32(dev, 0x40011c, 0xffffffff);
685 nv_wr32(dev, 0x400134, 0xffffffff);
686 nv_wr32(dev, 0x400054, 0x34ce3464);
687
688 ret = nvc0_graph_init_ctxctl(dev);
689 if (ret == 0)
690 dev_priv->engine.graph.accel_blocked = false;
73 return 0; 691 return 0;
74} 692}
75 693
694static int
695nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
696{
697 struct drm_nouveau_private *dev_priv = dev->dev_private;
698 struct nouveau_channel *chan;
699 unsigned long flags;
700 int i;
701
702 spin_lock_irqsave(&dev_priv->channels.lock, flags);
703 for (i = 0; i < dev_priv->engine.fifo.channels; i++) {
704 chan = dev_priv->channels.ptr[i];
705 if (!chan || !chan->ramin)
706 continue;
707
708 if (inst == chan->ramin->vinst)
709 break;
710 }
711 spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
712 return i;
713}
714
715static void
716nvc0_graph_isr(struct drm_device *dev)
717{
718 u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
719 u32 chid = nvc0_graph_isr_chid(dev, inst);
720 u32 stat = nv_rd32(dev, 0x400100);
721 u32 addr = nv_rd32(dev, 0x400704);
722 u32 mthd = (addr & 0x00003ffc);
723 u32 subc = (addr & 0x00070000) >> 16;
724 u32 data = nv_rd32(dev, 0x400708);
725 u32 code = nv_rd32(dev, 0x400110);
726 u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
727
728 if (stat & 0x00000010) {
729 NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] subc %d "
730 "class 0x%04x mthd 0x%04x data 0x%08x\n",
731 chid, inst, subc, class, mthd, data);
732 nv_wr32(dev, 0x400100, 0x00000010);
733 stat &= ~0x00000010;
734 }
735
736 if (stat & 0x00000020) {
737 NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
738 "class 0x%04x mthd 0x%04x data 0x%08x\n",
739 chid, inst, subc, class, mthd, data);
740 nv_wr32(dev, 0x400100, 0x00000020);
741 stat &= ~0x00000020;
742 }
743
744 if (stat & 0x00100000) {
745 NV_INFO(dev, "PGRAPH: DATA_ERROR [");
746 nouveau_enum_print(nv50_data_error_names, code);
747 printk("] ch %d [0x%010llx] subc %d class 0x%04x "
748 "mthd 0x%04x data 0x%08x\n",
749 chid, inst, subc, class, mthd, data);
750 nv_wr32(dev, 0x400100, 0x00100000);
751 stat &= ~0x00100000;
752 }
753
754 if (stat & 0x00200000) {
755 u32 trap = nv_rd32(dev, 0x400108);
756 NV_INFO(dev, "PGRAPH: TRAP ch %d status 0x%08x\n", chid, trap);
757 nv_wr32(dev, 0x400108, trap);
758 nv_wr32(dev, 0x400100, 0x00200000);
759 stat &= ~0x00200000;
760 }
761
762 if (stat & 0x00080000) {
763 u32 ustat = nv_rd32(dev, 0x409c18);
764
765 NV_INFO(dev, "PGRAPH: CTXCTRL ustat 0x%08x\n", ustat);
766
767 nv_wr32(dev, 0x409c20, ustat);
768 nv_wr32(dev, 0x400100, 0x00080000);
769 stat &= ~0x00080000;
770 }
771
772 if (stat) {
773 NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
774 nv_wr32(dev, 0x400100, stat);
775 }
776
777 nv_wr32(dev, 0x400500, 0x00010001);
778}
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.h b/drivers/gpu/drm/nouveau/nvc0_graph.h
new file mode 100644
index 000000000000..40e26f9c56c4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_graph.h
@@ -0,0 +1,64 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#ifndef __NVC0_GRAPH_H__
26#define __NVC0_GRAPH_H__
27
28#define GPC_MAX 4
29#define TP_MAX 32
30
31#define ROP_BCAST(r) (0x408800 + (r))
32#define ROP_UNIT(u,r) (0x410000 + (u) * 0x400 + (r))
33#define GPC_BCAST(r) (0x418000 + (r))
34#define GPC_UNIT(t,r) (0x500000 + (t) * 0x8000 + (r))
35#define TP_UNIT(t,m,r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
36
37struct nvc0_graph_priv {
38 u8 gpc_nr;
39 u8 rop_nr;
40 u8 tp_nr[GPC_MAX];
41 u8 tp_total;
42
43 u32 grctx_size;
44 u32 *grctx_vals;
45 struct nouveau_gpuobj *unk4188b4;
46 struct nouveau_gpuobj *unk4188b8;
47
48 u8 magic_not_rop_nr;
49 u32 magicgpc980[4];
50 u32 magicgpc918;
51};
52
53struct nvc0_graph_chan {
54 struct nouveau_gpuobj *grctx;
55 struct nouveau_gpuobj *unk408004; // 0x418810 too
56 struct nouveau_gpuobj *unk40800c; // 0x419004 too
57 struct nouveau_gpuobj *unk418810; // 0x419848 too
58 struct nouveau_gpuobj *mmio;
59 int mmio_nr;
60};
61
62int nvc0_grctx_generate(struct nouveau_channel *);
63
64#endif
diff --git a/drivers/gpu/drm/nouveau/nvc0_grctx.c b/drivers/gpu/drm/nouveau/nvc0_grctx.c
new file mode 100644
index 000000000000..b9e68b2d30aa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_grctx.c
@@ -0,0 +1,2874 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_mm.h"
28#include "nvc0_graph.h"
29
30static void
31nv_icmd(struct drm_device *dev, u32 icmd, u32 data)
32{
33 nv_wr32(dev, 0x400204, data);
34 nv_wr32(dev, 0x400200, icmd);
35 while (nv_rd32(dev, 0x400700) & 2) {}
36}
37
38static void
39nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
40{
41 nv_wr32(dev, 0x40448c, data);
42 nv_wr32(dev, 0x404488, 0x80000000 | (mthd << 14) | class);
43}
44
45static void
46nvc0_grctx_generate_9097(struct drm_device *dev)
47{
48 nv_mthd(dev, 0x9097, 0x0800, 0x00000000);
49 nv_mthd(dev, 0x9097, 0x0840, 0x00000000);
50 nv_mthd(dev, 0x9097, 0x0880, 0x00000000);
51 nv_mthd(dev, 0x9097, 0x08c0, 0x00000000);
52 nv_mthd(dev, 0x9097, 0x0900, 0x00000000);
53 nv_mthd(dev, 0x9097, 0x0940, 0x00000000);
54 nv_mthd(dev, 0x9097, 0x0980, 0x00000000);
55 nv_mthd(dev, 0x9097, 0x09c0, 0x00000000);
56 nv_mthd(dev, 0x9097, 0x0804, 0x00000000);
57 nv_mthd(dev, 0x9097, 0x0844, 0x00000000);
58 nv_mthd(dev, 0x9097, 0x0884, 0x00000000);
59 nv_mthd(dev, 0x9097, 0x08c4, 0x00000000);
60 nv_mthd(dev, 0x9097, 0x0904, 0x00000000);
61 nv_mthd(dev, 0x9097, 0x0944, 0x00000000);
62 nv_mthd(dev, 0x9097, 0x0984, 0x00000000);
63 nv_mthd(dev, 0x9097, 0x09c4, 0x00000000);
64 nv_mthd(dev, 0x9097, 0x0808, 0x00000400);
65 nv_mthd(dev, 0x9097, 0x0848, 0x00000400);
66 nv_mthd(dev, 0x9097, 0x0888, 0x00000400);
67 nv_mthd(dev, 0x9097, 0x08c8, 0x00000400);
68 nv_mthd(dev, 0x9097, 0x0908, 0x00000400);
69 nv_mthd(dev, 0x9097, 0x0948, 0x00000400);
70 nv_mthd(dev, 0x9097, 0x0988, 0x00000400);
71 nv_mthd(dev, 0x9097, 0x09c8, 0x00000400);
72 nv_mthd(dev, 0x9097, 0x080c, 0x00000300);
73 nv_mthd(dev, 0x9097, 0x084c, 0x00000300);
74 nv_mthd(dev, 0x9097, 0x088c, 0x00000300);
75 nv_mthd(dev, 0x9097, 0x08cc, 0x00000300);
76 nv_mthd(dev, 0x9097, 0x090c, 0x00000300);
77 nv_mthd(dev, 0x9097, 0x094c, 0x00000300);
78 nv_mthd(dev, 0x9097, 0x098c, 0x00000300);
79 nv_mthd(dev, 0x9097, 0x09cc, 0x00000300);
80 nv_mthd(dev, 0x9097, 0x0810, 0x000000cf);
81 nv_mthd(dev, 0x9097, 0x0850, 0x00000000);
82 nv_mthd(dev, 0x9097, 0x0890, 0x00000000);
83 nv_mthd(dev, 0x9097, 0x08d0, 0x00000000);
84 nv_mthd(dev, 0x9097, 0x0910, 0x00000000);
85 nv_mthd(dev, 0x9097, 0x0950, 0x00000000);
86 nv_mthd(dev, 0x9097, 0x0990, 0x00000000);
87 nv_mthd(dev, 0x9097, 0x09d0, 0x00000000);
88 nv_mthd(dev, 0x9097, 0x0814, 0x00000040);
89 nv_mthd(dev, 0x9097, 0x0854, 0x00000040);
90 nv_mthd(dev, 0x9097, 0x0894, 0x00000040);
91 nv_mthd(dev, 0x9097, 0x08d4, 0x00000040);
92 nv_mthd(dev, 0x9097, 0x0914, 0x00000040);
93 nv_mthd(dev, 0x9097, 0x0954, 0x00000040);
94 nv_mthd(dev, 0x9097, 0x0994, 0x00000040);
95 nv_mthd(dev, 0x9097, 0x09d4, 0x00000040);
96 nv_mthd(dev, 0x9097, 0x0818, 0x00000001);
97 nv_mthd(dev, 0x9097, 0x0858, 0x00000001);
98 nv_mthd(dev, 0x9097, 0x0898, 0x00000001);
99 nv_mthd(dev, 0x9097, 0x08d8, 0x00000001);
100 nv_mthd(dev, 0x9097, 0x0918, 0x00000001);
101 nv_mthd(dev, 0x9097, 0x0958, 0x00000001);
102 nv_mthd(dev, 0x9097, 0x0998, 0x00000001);
103 nv_mthd(dev, 0x9097, 0x09d8, 0x00000001);
104 nv_mthd(dev, 0x9097, 0x081c, 0x00000000);
105 nv_mthd(dev, 0x9097, 0x085c, 0x00000000);
106 nv_mthd(dev, 0x9097, 0x089c, 0x00000000);
107 nv_mthd(dev, 0x9097, 0x08dc, 0x00000000);
108 nv_mthd(dev, 0x9097, 0x091c, 0x00000000);
109 nv_mthd(dev, 0x9097, 0x095c, 0x00000000);
110 nv_mthd(dev, 0x9097, 0x099c, 0x00000000);
111 nv_mthd(dev, 0x9097, 0x09dc, 0x00000000);
112 nv_mthd(dev, 0x9097, 0x0820, 0x00000000);
113 nv_mthd(dev, 0x9097, 0x0860, 0x00000000);
114 nv_mthd(dev, 0x9097, 0x08a0, 0x00000000);
115 nv_mthd(dev, 0x9097, 0x08e0, 0x00000000);
116 nv_mthd(dev, 0x9097, 0x0920, 0x00000000);
117 nv_mthd(dev, 0x9097, 0x0960, 0x00000000);
118 nv_mthd(dev, 0x9097, 0x09a0, 0x00000000);
119 nv_mthd(dev, 0x9097, 0x09e0, 0x00000000);
120 nv_mthd(dev, 0x9097, 0x2700, 0x00000000);
121 nv_mthd(dev, 0x9097, 0x2720, 0x00000000);
122 nv_mthd(dev, 0x9097, 0x2740, 0x00000000);
123 nv_mthd(dev, 0x9097, 0x2760, 0x00000000);
124 nv_mthd(dev, 0x9097, 0x2780, 0x00000000);
125 nv_mthd(dev, 0x9097, 0x27a0, 0x00000000);
126 nv_mthd(dev, 0x9097, 0x27c0, 0x00000000);
127 nv_mthd(dev, 0x9097, 0x27e0, 0x00000000);
128 nv_mthd(dev, 0x9097, 0x2704, 0x00000000);
129 nv_mthd(dev, 0x9097, 0x2724, 0x00000000);
130 nv_mthd(dev, 0x9097, 0x2744, 0x00000000);
131 nv_mthd(dev, 0x9097, 0x2764, 0x00000000);
132 nv_mthd(dev, 0x9097, 0x2784, 0x00000000);
133 nv_mthd(dev, 0x9097, 0x27a4, 0x00000000);
134 nv_mthd(dev, 0x9097, 0x27c4, 0x00000000);
135 nv_mthd(dev, 0x9097, 0x27e4, 0x00000000);
136 nv_mthd(dev, 0x9097, 0x2708, 0x00000000);
137 nv_mthd(dev, 0x9097, 0x2728, 0x00000000);
138 nv_mthd(dev, 0x9097, 0x2748, 0x00000000);
139 nv_mthd(dev, 0x9097, 0x2768, 0x00000000);
140 nv_mthd(dev, 0x9097, 0x2788, 0x00000000);
141 nv_mthd(dev, 0x9097, 0x27a8, 0x00000000);
142 nv_mthd(dev, 0x9097, 0x27c8, 0x00000000);
143 nv_mthd(dev, 0x9097, 0x27e8, 0x00000000);
144 nv_mthd(dev, 0x9097, 0x270c, 0x00000000);
145 nv_mthd(dev, 0x9097, 0x272c, 0x00000000);
146 nv_mthd(dev, 0x9097, 0x274c, 0x00000000);
147 nv_mthd(dev, 0x9097, 0x276c, 0x00000000);
148 nv_mthd(dev, 0x9097, 0x278c, 0x00000000);
149 nv_mthd(dev, 0x9097, 0x27ac, 0x00000000);
150 nv_mthd(dev, 0x9097, 0x27cc, 0x00000000);
151 nv_mthd(dev, 0x9097, 0x27ec, 0x00000000);
152 nv_mthd(dev, 0x9097, 0x2710, 0x00014000);
153 nv_mthd(dev, 0x9097, 0x2730, 0x00014000);
154 nv_mthd(dev, 0x9097, 0x2750, 0x00014000);
155 nv_mthd(dev, 0x9097, 0x2770, 0x00014000);
156 nv_mthd(dev, 0x9097, 0x2790, 0x00014000);
157 nv_mthd(dev, 0x9097, 0x27b0, 0x00014000);
158 nv_mthd(dev, 0x9097, 0x27d0, 0x00014000);
159 nv_mthd(dev, 0x9097, 0x27f0, 0x00014000);
160 nv_mthd(dev, 0x9097, 0x2714, 0x00000040);
161 nv_mthd(dev, 0x9097, 0x2734, 0x00000040);
162 nv_mthd(dev, 0x9097, 0x2754, 0x00000040);
163 nv_mthd(dev, 0x9097, 0x2774, 0x00000040);
164 nv_mthd(dev, 0x9097, 0x2794, 0x00000040);
165 nv_mthd(dev, 0x9097, 0x27b4, 0x00000040);
166 nv_mthd(dev, 0x9097, 0x27d4, 0x00000040);
167 nv_mthd(dev, 0x9097, 0x27f4, 0x00000040);
168 nv_mthd(dev, 0x9097, 0x1c00, 0x00000000);
169 nv_mthd(dev, 0x9097, 0x1c10, 0x00000000);
170 nv_mthd(dev, 0x9097, 0x1c20, 0x00000000);
171 nv_mthd(dev, 0x9097, 0x1c30, 0x00000000);
172 nv_mthd(dev, 0x9097, 0x1c40, 0x00000000);
173 nv_mthd(dev, 0x9097, 0x1c50, 0x00000000);
174 nv_mthd(dev, 0x9097, 0x1c60, 0x00000000);
175 nv_mthd(dev, 0x9097, 0x1c70, 0x00000000);
176 nv_mthd(dev, 0x9097, 0x1c80, 0x00000000);
177 nv_mthd(dev, 0x9097, 0x1c90, 0x00000000);
178 nv_mthd(dev, 0x9097, 0x1ca0, 0x00000000);
179 nv_mthd(dev, 0x9097, 0x1cb0, 0x00000000);
180 nv_mthd(dev, 0x9097, 0x1cc0, 0x00000000);
181 nv_mthd(dev, 0x9097, 0x1cd0, 0x00000000);
182 nv_mthd(dev, 0x9097, 0x1ce0, 0x00000000);
183 nv_mthd(dev, 0x9097, 0x1cf0, 0x00000000);
184 nv_mthd(dev, 0x9097, 0x1c04, 0x00000000);
185 nv_mthd(dev, 0x9097, 0x1c14, 0x00000000);
186 nv_mthd(dev, 0x9097, 0x1c24, 0x00000000);
187 nv_mthd(dev, 0x9097, 0x1c34, 0x00000000);
188 nv_mthd(dev, 0x9097, 0x1c44, 0x00000000);
189 nv_mthd(dev, 0x9097, 0x1c54, 0x00000000);
190 nv_mthd(dev, 0x9097, 0x1c64, 0x00000000);
191 nv_mthd(dev, 0x9097, 0x1c74, 0x00000000);
192 nv_mthd(dev, 0x9097, 0x1c84, 0x00000000);
193 nv_mthd(dev, 0x9097, 0x1c94, 0x00000000);
194 nv_mthd(dev, 0x9097, 0x1ca4, 0x00000000);
195 nv_mthd(dev, 0x9097, 0x1cb4, 0x00000000);
196 nv_mthd(dev, 0x9097, 0x1cc4, 0x00000000);
197 nv_mthd(dev, 0x9097, 0x1cd4, 0x00000000);
198 nv_mthd(dev, 0x9097, 0x1ce4, 0x00000000);
199 nv_mthd(dev, 0x9097, 0x1cf4, 0x00000000);
200 nv_mthd(dev, 0x9097, 0x1c08, 0x00000000);
201 nv_mthd(dev, 0x9097, 0x1c18, 0x00000000);
202 nv_mthd(dev, 0x9097, 0x1c28, 0x00000000);
203 nv_mthd(dev, 0x9097, 0x1c38, 0x00000000);
204 nv_mthd(dev, 0x9097, 0x1c48, 0x00000000);
205 nv_mthd(dev, 0x9097, 0x1c58, 0x00000000);
206 nv_mthd(dev, 0x9097, 0x1c68, 0x00000000);
207 nv_mthd(dev, 0x9097, 0x1c78, 0x00000000);
208 nv_mthd(dev, 0x9097, 0x1c88, 0x00000000);
209 nv_mthd(dev, 0x9097, 0x1c98, 0x00000000);
210 nv_mthd(dev, 0x9097, 0x1ca8, 0x00000000);
211 nv_mthd(dev, 0x9097, 0x1cb8, 0x00000000);
212 nv_mthd(dev, 0x9097, 0x1cc8, 0x00000000);
213 nv_mthd(dev, 0x9097, 0x1cd8, 0x00000000);
214 nv_mthd(dev, 0x9097, 0x1ce8, 0x00000000);
215 nv_mthd(dev, 0x9097, 0x1cf8, 0x00000000);
216 nv_mthd(dev, 0x9097, 0x1c0c, 0x00000000);
217 nv_mthd(dev, 0x9097, 0x1c1c, 0x00000000);
218 nv_mthd(dev, 0x9097, 0x1c2c, 0x00000000);
219 nv_mthd(dev, 0x9097, 0x1c3c, 0x00000000);
220 nv_mthd(dev, 0x9097, 0x1c4c, 0x00000000);
221 nv_mthd(dev, 0x9097, 0x1c5c, 0x00000000);
222 nv_mthd(dev, 0x9097, 0x1c6c, 0x00000000);
223 nv_mthd(dev, 0x9097, 0x1c7c, 0x00000000);
224 nv_mthd(dev, 0x9097, 0x1c8c, 0x00000000);
225 nv_mthd(dev, 0x9097, 0x1c9c, 0x00000000);
226 nv_mthd(dev, 0x9097, 0x1cac, 0x00000000);
227 nv_mthd(dev, 0x9097, 0x1cbc, 0x00000000);
228 nv_mthd(dev, 0x9097, 0x1ccc, 0x00000000);
229 nv_mthd(dev, 0x9097, 0x1cdc, 0x00000000);
230 nv_mthd(dev, 0x9097, 0x1cec, 0x00000000);
231 nv_mthd(dev, 0x9097, 0x1cfc, 0x00000000);
232 nv_mthd(dev, 0x9097, 0x1d00, 0x00000000);
233 nv_mthd(dev, 0x9097, 0x1d10, 0x00000000);
234 nv_mthd(dev, 0x9097, 0x1d20, 0x00000000);
235 nv_mthd(dev, 0x9097, 0x1d30, 0x00000000);
236 nv_mthd(dev, 0x9097, 0x1d40, 0x00000000);
237 nv_mthd(dev, 0x9097, 0x1d50, 0x00000000);
238 nv_mthd(dev, 0x9097, 0x1d60, 0x00000000);
239 nv_mthd(dev, 0x9097, 0x1d70, 0x00000000);
240 nv_mthd(dev, 0x9097, 0x1d80, 0x00000000);
241 nv_mthd(dev, 0x9097, 0x1d90, 0x00000000);
242 nv_mthd(dev, 0x9097, 0x1da0, 0x00000000);
243 nv_mthd(dev, 0x9097, 0x1db0, 0x00000000);
244 nv_mthd(dev, 0x9097, 0x1dc0, 0x00000000);
245 nv_mthd(dev, 0x9097, 0x1dd0, 0x00000000);
246 nv_mthd(dev, 0x9097, 0x1de0, 0x00000000);
247 nv_mthd(dev, 0x9097, 0x1df0, 0x00000000);
248 nv_mthd(dev, 0x9097, 0x1d04, 0x00000000);
249 nv_mthd(dev, 0x9097, 0x1d14, 0x00000000);
250 nv_mthd(dev, 0x9097, 0x1d24, 0x00000000);
251 nv_mthd(dev, 0x9097, 0x1d34, 0x00000000);
252 nv_mthd(dev, 0x9097, 0x1d44, 0x00000000);
253 nv_mthd(dev, 0x9097, 0x1d54, 0x00000000);
254 nv_mthd(dev, 0x9097, 0x1d64, 0x00000000);
255 nv_mthd(dev, 0x9097, 0x1d74, 0x00000000);
256 nv_mthd(dev, 0x9097, 0x1d84, 0x00000000);
257 nv_mthd(dev, 0x9097, 0x1d94, 0x00000000);
258 nv_mthd(dev, 0x9097, 0x1da4, 0x00000000);
259 nv_mthd(dev, 0x9097, 0x1db4, 0x00000000);
260 nv_mthd(dev, 0x9097, 0x1dc4, 0x00000000);
261 nv_mthd(dev, 0x9097, 0x1dd4, 0x00000000);
262 nv_mthd(dev, 0x9097, 0x1de4, 0x00000000);
263 nv_mthd(dev, 0x9097, 0x1df4, 0x00000000);
264 nv_mthd(dev, 0x9097, 0x1d08, 0x00000000);
265 nv_mthd(dev, 0x9097, 0x1d18, 0x00000000);
266 nv_mthd(dev, 0x9097, 0x1d28, 0x00000000);
267 nv_mthd(dev, 0x9097, 0x1d38, 0x00000000);
268 nv_mthd(dev, 0x9097, 0x1d48, 0x00000000);
269 nv_mthd(dev, 0x9097, 0x1d58, 0x00000000);
270 nv_mthd(dev, 0x9097, 0x1d68, 0x00000000);
271 nv_mthd(dev, 0x9097, 0x1d78, 0x00000000);
272 nv_mthd(dev, 0x9097, 0x1d88, 0x00000000);
273 nv_mthd(dev, 0x9097, 0x1d98, 0x00000000);
274 nv_mthd(dev, 0x9097, 0x1da8, 0x00000000);
275 nv_mthd(dev, 0x9097, 0x1db8, 0x00000000);
276 nv_mthd(dev, 0x9097, 0x1dc8, 0x00000000);
277 nv_mthd(dev, 0x9097, 0x1dd8, 0x00000000);
278 nv_mthd(dev, 0x9097, 0x1de8, 0x00000000);
279 nv_mthd(dev, 0x9097, 0x1df8, 0x00000000);
280 nv_mthd(dev, 0x9097, 0x1d0c, 0x00000000);
281 nv_mthd(dev, 0x9097, 0x1d1c, 0x00000000);
282 nv_mthd(dev, 0x9097, 0x1d2c, 0x00000000);
283 nv_mthd(dev, 0x9097, 0x1d3c, 0x00000000);
284 nv_mthd(dev, 0x9097, 0x1d4c, 0x00000000);
285 nv_mthd(dev, 0x9097, 0x1d5c, 0x00000000);
286 nv_mthd(dev, 0x9097, 0x1d6c, 0x00000000);
287 nv_mthd(dev, 0x9097, 0x1d7c, 0x00000000);
288 nv_mthd(dev, 0x9097, 0x1d8c, 0x00000000);
289 nv_mthd(dev, 0x9097, 0x1d9c, 0x00000000);
290 nv_mthd(dev, 0x9097, 0x1dac, 0x00000000);
291 nv_mthd(dev, 0x9097, 0x1dbc, 0x00000000);
292 nv_mthd(dev, 0x9097, 0x1dcc, 0x00000000);
293 nv_mthd(dev, 0x9097, 0x1ddc, 0x00000000);
294 nv_mthd(dev, 0x9097, 0x1dec, 0x00000000);
295 nv_mthd(dev, 0x9097, 0x1dfc, 0x00000000);
296 nv_mthd(dev, 0x9097, 0x1f00, 0x00000000);
297 nv_mthd(dev, 0x9097, 0x1f08, 0x00000000);
298 nv_mthd(dev, 0x9097, 0x1f10, 0x00000000);
299 nv_mthd(dev, 0x9097, 0x1f18, 0x00000000);
300 nv_mthd(dev, 0x9097, 0x1f20, 0x00000000);
301 nv_mthd(dev, 0x9097, 0x1f28, 0x00000000);
302 nv_mthd(dev, 0x9097, 0x1f30, 0x00000000);
303 nv_mthd(dev, 0x9097, 0x1f38, 0x00000000);
304 nv_mthd(dev, 0x9097, 0x1f40, 0x00000000);
305 nv_mthd(dev, 0x9097, 0x1f48, 0x00000000);
306 nv_mthd(dev, 0x9097, 0x1f50, 0x00000000);
307 nv_mthd(dev, 0x9097, 0x1f58, 0x00000000);
308 nv_mthd(dev, 0x9097, 0x1f60, 0x00000000);
309 nv_mthd(dev, 0x9097, 0x1f68, 0x00000000);
310 nv_mthd(dev, 0x9097, 0x1f70, 0x00000000);
311 nv_mthd(dev, 0x9097, 0x1f78, 0x00000000);
312 nv_mthd(dev, 0x9097, 0x1f04, 0x00000000);
313 nv_mthd(dev, 0x9097, 0x1f0c, 0x00000000);
314 nv_mthd(dev, 0x9097, 0x1f14, 0x00000000);
315 nv_mthd(dev, 0x9097, 0x1f1c, 0x00000000);
316 nv_mthd(dev, 0x9097, 0x1f24, 0x00000000);
317 nv_mthd(dev, 0x9097, 0x1f2c, 0x00000000);
318 nv_mthd(dev, 0x9097, 0x1f34, 0x00000000);
319 nv_mthd(dev, 0x9097, 0x1f3c, 0x00000000);
320 nv_mthd(dev, 0x9097, 0x1f44, 0x00000000);
321 nv_mthd(dev, 0x9097, 0x1f4c, 0x00000000);
322 nv_mthd(dev, 0x9097, 0x1f54, 0x00000000);
323 nv_mthd(dev, 0x9097, 0x1f5c, 0x00000000);
324 nv_mthd(dev, 0x9097, 0x1f64, 0x00000000);
325 nv_mthd(dev, 0x9097, 0x1f6c, 0x00000000);
326 nv_mthd(dev, 0x9097, 0x1f74, 0x00000000);
327 nv_mthd(dev, 0x9097, 0x1f7c, 0x00000000);
328 nv_mthd(dev, 0x9097, 0x1f80, 0x00000000);
329 nv_mthd(dev, 0x9097, 0x1f88, 0x00000000);
330 nv_mthd(dev, 0x9097, 0x1f90, 0x00000000);
331 nv_mthd(dev, 0x9097, 0x1f98, 0x00000000);
332 nv_mthd(dev, 0x9097, 0x1fa0, 0x00000000);
333 nv_mthd(dev, 0x9097, 0x1fa8, 0x00000000);
334 nv_mthd(dev, 0x9097, 0x1fb0, 0x00000000);
335 nv_mthd(dev, 0x9097, 0x1fb8, 0x00000000);
336 nv_mthd(dev, 0x9097, 0x1fc0, 0x00000000);
337 nv_mthd(dev, 0x9097, 0x1fc8, 0x00000000);
338 nv_mthd(dev, 0x9097, 0x1fd0, 0x00000000);
339 nv_mthd(dev, 0x9097, 0x1fd8, 0x00000000);
340 nv_mthd(dev, 0x9097, 0x1fe0, 0x00000000);
341 nv_mthd(dev, 0x9097, 0x1fe8, 0x00000000);
342 nv_mthd(dev, 0x9097, 0x1ff0, 0x00000000);
343 nv_mthd(dev, 0x9097, 0x1ff8, 0x00000000);
344 nv_mthd(dev, 0x9097, 0x1f84, 0x00000000);
345 nv_mthd(dev, 0x9097, 0x1f8c, 0x00000000);
346 nv_mthd(dev, 0x9097, 0x1f94, 0x00000000);
347 nv_mthd(dev, 0x9097, 0x1f9c, 0x00000000);
348 nv_mthd(dev, 0x9097, 0x1fa4, 0x00000000);
349 nv_mthd(dev, 0x9097, 0x1fac, 0x00000000);
350 nv_mthd(dev, 0x9097, 0x1fb4, 0x00000000);
351 nv_mthd(dev, 0x9097, 0x1fbc, 0x00000000);
352 nv_mthd(dev, 0x9097, 0x1fc4, 0x00000000);
353 nv_mthd(dev, 0x9097, 0x1fcc, 0x00000000);
354 nv_mthd(dev, 0x9097, 0x1fd4, 0x00000000);
355 nv_mthd(dev, 0x9097, 0x1fdc, 0x00000000);
356 nv_mthd(dev, 0x9097, 0x1fe4, 0x00000000);
357 nv_mthd(dev, 0x9097, 0x1fec, 0x00000000);
358 nv_mthd(dev, 0x9097, 0x1ff4, 0x00000000);
359 nv_mthd(dev, 0x9097, 0x1ffc, 0x00000000);
360 nv_mthd(dev, 0x9097, 0x2200, 0x00000022);
361 nv_mthd(dev, 0x9097, 0x2210, 0x00000022);
362 nv_mthd(dev, 0x9097, 0x2220, 0x00000022);
363 nv_mthd(dev, 0x9097, 0x2230, 0x00000022);
364 nv_mthd(dev, 0x9097, 0x2240, 0x00000022);
365 nv_mthd(dev, 0x9097, 0x2000, 0x00000000);
366 nv_mthd(dev, 0x9097, 0x2040, 0x00000011);
367 nv_mthd(dev, 0x9097, 0x2080, 0x00000020);
368 nv_mthd(dev, 0x9097, 0x20c0, 0x00000030);
369 nv_mthd(dev, 0x9097, 0x2100, 0x00000040);
370 nv_mthd(dev, 0x9097, 0x2140, 0x00000051);
371 nv_mthd(dev, 0x9097, 0x200c, 0x00000001);
372 nv_mthd(dev, 0x9097, 0x204c, 0x00000001);
373 nv_mthd(dev, 0x9097, 0x208c, 0x00000001);
374 nv_mthd(dev, 0x9097, 0x20cc, 0x00000001);
375 nv_mthd(dev, 0x9097, 0x210c, 0x00000001);
376 nv_mthd(dev, 0x9097, 0x214c, 0x00000001);
377 nv_mthd(dev, 0x9097, 0x2010, 0x00000000);
378 nv_mthd(dev, 0x9097, 0x2050, 0x00000000);
379 nv_mthd(dev, 0x9097, 0x2090, 0x00000001);
380 nv_mthd(dev, 0x9097, 0x20d0, 0x00000002);
381 nv_mthd(dev, 0x9097, 0x2110, 0x00000003);
382 nv_mthd(dev, 0x9097, 0x2150, 0x00000004);
383 nv_mthd(dev, 0x9097, 0x0380, 0x00000000);
384 nv_mthd(dev, 0x9097, 0x03a0, 0x00000000);
385 nv_mthd(dev, 0x9097, 0x03c0, 0x00000000);
386 nv_mthd(dev, 0x9097, 0x03e0, 0x00000000);
387 nv_mthd(dev, 0x9097, 0x0384, 0x00000000);
388 nv_mthd(dev, 0x9097, 0x03a4, 0x00000000);
389 nv_mthd(dev, 0x9097, 0x03c4, 0x00000000);
390 nv_mthd(dev, 0x9097, 0x03e4, 0x00000000);
391 nv_mthd(dev, 0x9097, 0x0388, 0x00000000);
392 nv_mthd(dev, 0x9097, 0x03a8, 0x00000000);
393 nv_mthd(dev, 0x9097, 0x03c8, 0x00000000);
394 nv_mthd(dev, 0x9097, 0x03e8, 0x00000000);
395 nv_mthd(dev, 0x9097, 0x038c, 0x00000000);
396 nv_mthd(dev, 0x9097, 0x03ac, 0x00000000);
397 nv_mthd(dev, 0x9097, 0x03cc, 0x00000000);
398 nv_mthd(dev, 0x9097, 0x03ec, 0x00000000);
399 nv_mthd(dev, 0x9097, 0x0700, 0x00000000);
400 nv_mthd(dev, 0x9097, 0x0710, 0x00000000);
401 nv_mthd(dev, 0x9097, 0x0720, 0x00000000);
402 nv_mthd(dev, 0x9097, 0x0730, 0x00000000);
403 nv_mthd(dev, 0x9097, 0x0704, 0x00000000);
404 nv_mthd(dev, 0x9097, 0x0714, 0x00000000);
405 nv_mthd(dev, 0x9097, 0x0724, 0x00000000);
406 nv_mthd(dev, 0x9097, 0x0734, 0x00000000);
407 nv_mthd(dev, 0x9097, 0x0708, 0x00000000);
408 nv_mthd(dev, 0x9097, 0x0718, 0x00000000);
409 nv_mthd(dev, 0x9097, 0x0728, 0x00000000);
410 nv_mthd(dev, 0x9097, 0x0738, 0x00000000);
411 nv_mthd(dev, 0x9097, 0x2800, 0x00000000);
412 nv_mthd(dev, 0x9097, 0x2804, 0x00000000);
413 nv_mthd(dev, 0x9097, 0x2808, 0x00000000);
414 nv_mthd(dev, 0x9097, 0x280c, 0x00000000);
415 nv_mthd(dev, 0x9097, 0x2810, 0x00000000);
416 nv_mthd(dev, 0x9097, 0x2814, 0x00000000);
417 nv_mthd(dev, 0x9097, 0x2818, 0x00000000);
418 nv_mthd(dev, 0x9097, 0x281c, 0x00000000);
419 nv_mthd(dev, 0x9097, 0x2820, 0x00000000);
420 nv_mthd(dev, 0x9097, 0x2824, 0x00000000);
421 nv_mthd(dev, 0x9097, 0x2828, 0x00000000);
422 nv_mthd(dev, 0x9097, 0x282c, 0x00000000);
423 nv_mthd(dev, 0x9097, 0x2830, 0x00000000);
424 nv_mthd(dev, 0x9097, 0x2834, 0x00000000);
425 nv_mthd(dev, 0x9097, 0x2838, 0x00000000);
426 nv_mthd(dev, 0x9097, 0x283c, 0x00000000);
427 nv_mthd(dev, 0x9097, 0x2840, 0x00000000);
428 nv_mthd(dev, 0x9097, 0x2844, 0x00000000);
429 nv_mthd(dev, 0x9097, 0x2848, 0x00000000);
430 nv_mthd(dev, 0x9097, 0x284c, 0x00000000);
431 nv_mthd(dev, 0x9097, 0x2850, 0x00000000);
432 nv_mthd(dev, 0x9097, 0x2854, 0x00000000);
433 nv_mthd(dev, 0x9097, 0x2858, 0x00000000);
434 nv_mthd(dev, 0x9097, 0x285c, 0x00000000);
435 nv_mthd(dev, 0x9097, 0x2860, 0x00000000);
436 nv_mthd(dev, 0x9097, 0x2864, 0x00000000);
437 nv_mthd(dev, 0x9097, 0x2868, 0x00000000);
438 nv_mthd(dev, 0x9097, 0x286c, 0x00000000);
439 nv_mthd(dev, 0x9097, 0x2870, 0x00000000);
440 nv_mthd(dev, 0x9097, 0x2874, 0x00000000);
441 nv_mthd(dev, 0x9097, 0x2878, 0x00000000);
442 nv_mthd(dev, 0x9097, 0x287c, 0x00000000);
443 nv_mthd(dev, 0x9097, 0x2880, 0x00000000);
444 nv_mthd(dev, 0x9097, 0x2884, 0x00000000);
445 nv_mthd(dev, 0x9097, 0x2888, 0x00000000);
446 nv_mthd(dev, 0x9097, 0x288c, 0x00000000);
447 nv_mthd(dev, 0x9097, 0x2890, 0x00000000);
448 nv_mthd(dev, 0x9097, 0x2894, 0x00000000);
449 nv_mthd(dev, 0x9097, 0x2898, 0x00000000);
450 nv_mthd(dev, 0x9097, 0x289c, 0x00000000);
451 nv_mthd(dev, 0x9097, 0x28a0, 0x00000000);
452 nv_mthd(dev, 0x9097, 0x28a4, 0x00000000);
453 nv_mthd(dev, 0x9097, 0x28a8, 0x00000000);
454 nv_mthd(dev, 0x9097, 0x28ac, 0x00000000);
455 nv_mthd(dev, 0x9097, 0x28b0, 0x00000000);
456 nv_mthd(dev, 0x9097, 0x28b4, 0x00000000);
457 nv_mthd(dev, 0x9097, 0x28b8, 0x00000000);
458 nv_mthd(dev, 0x9097, 0x28bc, 0x00000000);
459 nv_mthd(dev, 0x9097, 0x28c0, 0x00000000);
460 nv_mthd(dev, 0x9097, 0x28c4, 0x00000000);
461 nv_mthd(dev, 0x9097, 0x28c8, 0x00000000);
462 nv_mthd(dev, 0x9097, 0x28cc, 0x00000000);
463 nv_mthd(dev, 0x9097, 0x28d0, 0x00000000);
464 nv_mthd(dev, 0x9097, 0x28d4, 0x00000000);
465 nv_mthd(dev, 0x9097, 0x28d8, 0x00000000);
466 nv_mthd(dev, 0x9097, 0x28dc, 0x00000000);
467 nv_mthd(dev, 0x9097, 0x28e0, 0x00000000);
468 nv_mthd(dev, 0x9097, 0x28e4, 0x00000000);
469 nv_mthd(dev, 0x9097, 0x28e8, 0x00000000);
470 nv_mthd(dev, 0x9097, 0x28ec, 0x00000000);
471 nv_mthd(dev, 0x9097, 0x28f0, 0x00000000);
472 nv_mthd(dev, 0x9097, 0x28f4, 0x00000000);
473 nv_mthd(dev, 0x9097, 0x28f8, 0x00000000);
474 nv_mthd(dev, 0x9097, 0x28fc, 0x00000000);
475 nv_mthd(dev, 0x9097, 0x2900, 0x00000000);
476 nv_mthd(dev, 0x9097, 0x2904, 0x00000000);
477 nv_mthd(dev, 0x9097, 0x2908, 0x00000000);
478 nv_mthd(dev, 0x9097, 0x290c, 0x00000000);
479 nv_mthd(dev, 0x9097, 0x2910, 0x00000000);
480 nv_mthd(dev, 0x9097, 0x2914, 0x00000000);
481 nv_mthd(dev, 0x9097, 0x2918, 0x00000000);
482 nv_mthd(dev, 0x9097, 0x291c, 0x00000000);
483 nv_mthd(dev, 0x9097, 0x2920, 0x00000000);
484 nv_mthd(dev, 0x9097, 0x2924, 0x00000000);
485 nv_mthd(dev, 0x9097, 0x2928, 0x00000000);
486 nv_mthd(dev, 0x9097, 0x292c, 0x00000000);
487 nv_mthd(dev, 0x9097, 0x2930, 0x00000000);
488 nv_mthd(dev, 0x9097, 0x2934, 0x00000000);
489 nv_mthd(dev, 0x9097, 0x2938, 0x00000000);
490 nv_mthd(dev, 0x9097, 0x293c, 0x00000000);
491 nv_mthd(dev, 0x9097, 0x2940, 0x00000000);
492 nv_mthd(dev, 0x9097, 0x2944, 0x00000000);
493 nv_mthd(dev, 0x9097, 0x2948, 0x00000000);
494 nv_mthd(dev, 0x9097, 0x294c, 0x00000000);
495 nv_mthd(dev, 0x9097, 0x2950, 0x00000000);
496 nv_mthd(dev, 0x9097, 0x2954, 0x00000000);
497 nv_mthd(dev, 0x9097, 0x2958, 0x00000000);
498 nv_mthd(dev, 0x9097, 0x295c, 0x00000000);
499 nv_mthd(dev, 0x9097, 0x2960, 0x00000000);
500 nv_mthd(dev, 0x9097, 0x2964, 0x00000000);
501 nv_mthd(dev, 0x9097, 0x2968, 0x00000000);
502 nv_mthd(dev, 0x9097, 0x296c, 0x00000000);
503 nv_mthd(dev, 0x9097, 0x2970, 0x00000000);
504 nv_mthd(dev, 0x9097, 0x2974, 0x00000000);
505 nv_mthd(dev, 0x9097, 0x2978, 0x00000000);
506 nv_mthd(dev, 0x9097, 0x297c, 0x00000000);
507 nv_mthd(dev, 0x9097, 0x2980, 0x00000000);
508 nv_mthd(dev, 0x9097, 0x2984, 0x00000000);
509 nv_mthd(dev, 0x9097, 0x2988, 0x00000000);
510 nv_mthd(dev, 0x9097, 0x298c, 0x00000000);
511 nv_mthd(dev, 0x9097, 0x2990, 0x00000000);
512 nv_mthd(dev, 0x9097, 0x2994, 0x00000000);
513 nv_mthd(dev, 0x9097, 0x2998, 0x00000000);
514 nv_mthd(dev, 0x9097, 0x299c, 0x00000000);
515 nv_mthd(dev, 0x9097, 0x29a0, 0x00000000);
516 nv_mthd(dev, 0x9097, 0x29a4, 0x00000000);
517 nv_mthd(dev, 0x9097, 0x29a8, 0x00000000);
518 nv_mthd(dev, 0x9097, 0x29ac, 0x00000000);
519 nv_mthd(dev, 0x9097, 0x29b0, 0x00000000);
520 nv_mthd(dev, 0x9097, 0x29b4, 0x00000000);
521 nv_mthd(dev, 0x9097, 0x29b8, 0x00000000);
522 nv_mthd(dev, 0x9097, 0x29bc, 0x00000000);
523 nv_mthd(dev, 0x9097, 0x29c0, 0x00000000);
524 nv_mthd(dev, 0x9097, 0x29c4, 0x00000000);
525 nv_mthd(dev, 0x9097, 0x29c8, 0x00000000);
526 nv_mthd(dev, 0x9097, 0x29cc, 0x00000000);
527 nv_mthd(dev, 0x9097, 0x29d0, 0x00000000);
528 nv_mthd(dev, 0x9097, 0x29d4, 0x00000000);
529 nv_mthd(dev, 0x9097, 0x29d8, 0x00000000);
530 nv_mthd(dev, 0x9097, 0x29dc, 0x00000000);
531 nv_mthd(dev, 0x9097, 0x29e0, 0x00000000);
532 nv_mthd(dev, 0x9097, 0x29e4, 0x00000000);
533 nv_mthd(dev, 0x9097, 0x29e8, 0x00000000);
534 nv_mthd(dev, 0x9097, 0x29ec, 0x00000000);
535 nv_mthd(dev, 0x9097, 0x29f0, 0x00000000);
536 nv_mthd(dev, 0x9097, 0x29f4, 0x00000000);
537 nv_mthd(dev, 0x9097, 0x29f8, 0x00000000);
538 nv_mthd(dev, 0x9097, 0x29fc, 0x00000000);
539 nv_mthd(dev, 0x9097, 0x0a00, 0x00000000);
540 nv_mthd(dev, 0x9097, 0x0a20, 0x00000000);
541 nv_mthd(dev, 0x9097, 0x0a40, 0x00000000);
542 nv_mthd(dev, 0x9097, 0x0a60, 0x00000000);
543 nv_mthd(dev, 0x9097, 0x0a80, 0x00000000);
544 nv_mthd(dev, 0x9097, 0x0aa0, 0x00000000);
545 nv_mthd(dev, 0x9097, 0x0ac0, 0x00000000);
546 nv_mthd(dev, 0x9097, 0x0ae0, 0x00000000);
547 nv_mthd(dev, 0x9097, 0x0b00, 0x00000000);
548 nv_mthd(dev, 0x9097, 0x0b20, 0x00000000);
549 nv_mthd(dev, 0x9097, 0x0b40, 0x00000000);
550 nv_mthd(dev, 0x9097, 0x0b60, 0x00000000);
551 nv_mthd(dev, 0x9097, 0x0b80, 0x00000000);
552 nv_mthd(dev, 0x9097, 0x0ba0, 0x00000000);
553 nv_mthd(dev, 0x9097, 0x0bc0, 0x00000000);
554 nv_mthd(dev, 0x9097, 0x0be0, 0x00000000);
555 nv_mthd(dev, 0x9097, 0x0a04, 0x00000000);
556 nv_mthd(dev, 0x9097, 0x0a24, 0x00000000);
557 nv_mthd(dev, 0x9097, 0x0a44, 0x00000000);
558 nv_mthd(dev, 0x9097, 0x0a64, 0x00000000);
559 nv_mthd(dev, 0x9097, 0x0a84, 0x00000000);
560 nv_mthd(dev, 0x9097, 0x0aa4, 0x00000000);
561 nv_mthd(dev, 0x9097, 0x0ac4, 0x00000000);
562 nv_mthd(dev, 0x9097, 0x0ae4, 0x00000000);
563 nv_mthd(dev, 0x9097, 0x0b04, 0x00000000);
564 nv_mthd(dev, 0x9097, 0x0b24, 0x00000000);
565 nv_mthd(dev, 0x9097, 0x0b44, 0x00000000);
566 nv_mthd(dev, 0x9097, 0x0b64, 0x00000000);
567 nv_mthd(dev, 0x9097, 0x0b84, 0x00000000);
568 nv_mthd(dev, 0x9097, 0x0ba4, 0x00000000);
569 nv_mthd(dev, 0x9097, 0x0bc4, 0x00000000);
570 nv_mthd(dev, 0x9097, 0x0be4, 0x00000000);
571 nv_mthd(dev, 0x9097, 0x0a08, 0x00000000);
572 nv_mthd(dev, 0x9097, 0x0a28, 0x00000000);
573 nv_mthd(dev, 0x9097, 0x0a48, 0x00000000);
574 nv_mthd(dev, 0x9097, 0x0a68, 0x00000000);
575 nv_mthd(dev, 0x9097, 0x0a88, 0x00000000);
576 nv_mthd(dev, 0x9097, 0x0aa8, 0x00000000);
577 nv_mthd(dev, 0x9097, 0x0ac8, 0x00000000);
578 nv_mthd(dev, 0x9097, 0x0ae8, 0x00000000);
579 nv_mthd(dev, 0x9097, 0x0b08, 0x00000000);
580 nv_mthd(dev, 0x9097, 0x0b28, 0x00000000);
581 nv_mthd(dev, 0x9097, 0x0b48, 0x00000000);
582 nv_mthd(dev, 0x9097, 0x0b68, 0x00000000);
583 nv_mthd(dev, 0x9097, 0x0b88, 0x00000000);
584 nv_mthd(dev, 0x9097, 0x0ba8, 0x00000000);
585 nv_mthd(dev, 0x9097, 0x0bc8, 0x00000000);
586 nv_mthd(dev, 0x9097, 0x0be8, 0x00000000);
587 nv_mthd(dev, 0x9097, 0x0a0c, 0x00000000);
588 nv_mthd(dev, 0x9097, 0x0a2c, 0x00000000);
589 nv_mthd(dev, 0x9097, 0x0a4c, 0x00000000);
590 nv_mthd(dev, 0x9097, 0x0a6c, 0x00000000);
591 nv_mthd(dev, 0x9097, 0x0a8c, 0x00000000);
592 nv_mthd(dev, 0x9097, 0x0aac, 0x00000000);
593 nv_mthd(dev, 0x9097, 0x0acc, 0x00000000);
594 nv_mthd(dev, 0x9097, 0x0aec, 0x00000000);
595 nv_mthd(dev, 0x9097, 0x0b0c, 0x00000000);
596 nv_mthd(dev, 0x9097, 0x0b2c, 0x00000000);
597 nv_mthd(dev, 0x9097, 0x0b4c, 0x00000000);
598 nv_mthd(dev, 0x9097, 0x0b6c, 0x00000000);
599 nv_mthd(dev, 0x9097, 0x0b8c, 0x00000000);
600 nv_mthd(dev, 0x9097, 0x0bac, 0x00000000);
601 nv_mthd(dev, 0x9097, 0x0bcc, 0x00000000);
602 nv_mthd(dev, 0x9097, 0x0bec, 0x00000000);
603 nv_mthd(dev, 0x9097, 0x0a10, 0x00000000);
604 nv_mthd(dev, 0x9097, 0x0a30, 0x00000000);
605 nv_mthd(dev, 0x9097, 0x0a50, 0x00000000);
606 nv_mthd(dev, 0x9097, 0x0a70, 0x00000000);
607 nv_mthd(dev, 0x9097, 0x0a90, 0x00000000);
608 nv_mthd(dev, 0x9097, 0x0ab0, 0x00000000);
609 nv_mthd(dev, 0x9097, 0x0ad0, 0x00000000);
610 nv_mthd(dev, 0x9097, 0x0af0, 0x00000000);
611 nv_mthd(dev, 0x9097, 0x0b10, 0x00000000);
612 nv_mthd(dev, 0x9097, 0x0b30, 0x00000000);
613 nv_mthd(dev, 0x9097, 0x0b50, 0x00000000);
614 nv_mthd(dev, 0x9097, 0x0b70, 0x00000000);
615 nv_mthd(dev, 0x9097, 0x0b90, 0x00000000);
616 nv_mthd(dev, 0x9097, 0x0bb0, 0x00000000);
617 nv_mthd(dev, 0x9097, 0x0bd0, 0x00000000);
618 nv_mthd(dev, 0x9097, 0x0bf0, 0x00000000);
619 nv_mthd(dev, 0x9097, 0x0a14, 0x00000000);
620 nv_mthd(dev, 0x9097, 0x0a34, 0x00000000);
621 nv_mthd(dev, 0x9097, 0x0a54, 0x00000000);
622 nv_mthd(dev, 0x9097, 0x0a74, 0x00000000);
623 nv_mthd(dev, 0x9097, 0x0a94, 0x00000000);
624 nv_mthd(dev, 0x9097, 0x0ab4, 0x00000000);
625 nv_mthd(dev, 0x9097, 0x0ad4, 0x00000000);
626 nv_mthd(dev, 0x9097, 0x0af4, 0x00000000);
627 nv_mthd(dev, 0x9097, 0x0b14, 0x00000000);
628 nv_mthd(dev, 0x9097, 0x0b34, 0x00000000);
629 nv_mthd(dev, 0x9097, 0x0b54, 0x00000000);
630 nv_mthd(dev, 0x9097, 0x0b74, 0x00000000);
631 nv_mthd(dev, 0x9097, 0x0b94, 0x00000000);
632 nv_mthd(dev, 0x9097, 0x0bb4, 0x00000000);
633 nv_mthd(dev, 0x9097, 0x0bd4, 0x00000000);
634 nv_mthd(dev, 0x9097, 0x0bf4, 0x00000000);
635 nv_mthd(dev, 0x9097, 0x0c00, 0x00000000);
636 nv_mthd(dev, 0x9097, 0x0c10, 0x00000000);
637 nv_mthd(dev, 0x9097, 0x0c20, 0x00000000);
638 nv_mthd(dev, 0x9097, 0x0c30, 0x00000000);
639 nv_mthd(dev, 0x9097, 0x0c40, 0x00000000);
640 nv_mthd(dev, 0x9097, 0x0c50, 0x00000000);
641 nv_mthd(dev, 0x9097, 0x0c60, 0x00000000);
642 nv_mthd(dev, 0x9097, 0x0c70, 0x00000000);
643 nv_mthd(dev, 0x9097, 0x0c80, 0x00000000);
644 nv_mthd(dev, 0x9097, 0x0c90, 0x00000000);
645 nv_mthd(dev, 0x9097, 0x0ca0, 0x00000000);
646 nv_mthd(dev, 0x9097, 0x0cb0, 0x00000000);
647 nv_mthd(dev, 0x9097, 0x0cc0, 0x00000000);
648 nv_mthd(dev, 0x9097, 0x0cd0, 0x00000000);
649 nv_mthd(dev, 0x9097, 0x0ce0, 0x00000000);
650 nv_mthd(dev, 0x9097, 0x0cf0, 0x00000000);
651 nv_mthd(dev, 0x9097, 0x0c04, 0x00000000);
652 nv_mthd(dev, 0x9097, 0x0c14, 0x00000000);
653 nv_mthd(dev, 0x9097, 0x0c24, 0x00000000);
654 nv_mthd(dev, 0x9097, 0x0c34, 0x00000000);
655 nv_mthd(dev, 0x9097, 0x0c44, 0x00000000);
656 nv_mthd(dev, 0x9097, 0x0c54, 0x00000000);
657 nv_mthd(dev, 0x9097, 0x0c64, 0x00000000);
658 nv_mthd(dev, 0x9097, 0x0c74, 0x00000000);
659 nv_mthd(dev, 0x9097, 0x0c84, 0x00000000);
660 nv_mthd(dev, 0x9097, 0x0c94, 0x00000000);
661 nv_mthd(dev, 0x9097, 0x0ca4, 0x00000000);
662 nv_mthd(dev, 0x9097, 0x0cb4, 0x00000000);
663 nv_mthd(dev, 0x9097, 0x0cc4, 0x00000000);
664 nv_mthd(dev, 0x9097, 0x0cd4, 0x00000000);
665 nv_mthd(dev, 0x9097, 0x0ce4, 0x00000000);
666 nv_mthd(dev, 0x9097, 0x0cf4, 0x00000000);
667 nv_mthd(dev, 0x9097, 0x0c08, 0x00000000);
668 nv_mthd(dev, 0x9097, 0x0c18, 0x00000000);
669 nv_mthd(dev, 0x9097, 0x0c28, 0x00000000);
670 nv_mthd(dev, 0x9097, 0x0c38, 0x00000000);
671 nv_mthd(dev, 0x9097, 0x0c48, 0x00000000);
672 nv_mthd(dev, 0x9097, 0x0c58, 0x00000000);
673 nv_mthd(dev, 0x9097, 0x0c68, 0x00000000);
674 nv_mthd(dev, 0x9097, 0x0c78, 0x00000000);
675 nv_mthd(dev, 0x9097, 0x0c88, 0x00000000);
676 nv_mthd(dev, 0x9097, 0x0c98, 0x00000000);
677 nv_mthd(dev, 0x9097, 0x0ca8, 0x00000000);
678 nv_mthd(dev, 0x9097, 0x0cb8, 0x00000000);
679 nv_mthd(dev, 0x9097, 0x0cc8, 0x00000000);
680 nv_mthd(dev, 0x9097, 0x0cd8, 0x00000000);
681 nv_mthd(dev, 0x9097, 0x0ce8, 0x00000000);
682 nv_mthd(dev, 0x9097, 0x0cf8, 0x00000000);
683 nv_mthd(dev, 0x9097, 0x0c0c, 0x3f800000);
684 nv_mthd(dev, 0x9097, 0x0c1c, 0x3f800000);
685 nv_mthd(dev, 0x9097, 0x0c2c, 0x3f800000);
686 nv_mthd(dev, 0x9097, 0x0c3c, 0x3f800000);
687 nv_mthd(dev, 0x9097, 0x0c4c, 0x3f800000);
688 nv_mthd(dev, 0x9097, 0x0c5c, 0x3f800000);
689 nv_mthd(dev, 0x9097, 0x0c6c, 0x3f800000);
690 nv_mthd(dev, 0x9097, 0x0c7c, 0x3f800000);
691 nv_mthd(dev, 0x9097, 0x0c8c, 0x3f800000);
692 nv_mthd(dev, 0x9097, 0x0c9c, 0x3f800000);
693 nv_mthd(dev, 0x9097, 0x0cac, 0x3f800000);
694 nv_mthd(dev, 0x9097, 0x0cbc, 0x3f800000);
695 nv_mthd(dev, 0x9097, 0x0ccc, 0x3f800000);
696 nv_mthd(dev, 0x9097, 0x0cdc, 0x3f800000);
697 nv_mthd(dev, 0x9097, 0x0cec, 0x3f800000);
698 nv_mthd(dev, 0x9097, 0x0cfc, 0x3f800000);
699 nv_mthd(dev, 0x9097, 0x0d00, 0xffff0000);
700 nv_mthd(dev, 0x9097, 0x0d08, 0xffff0000);
701 nv_mthd(dev, 0x9097, 0x0d10, 0xffff0000);
702 nv_mthd(dev, 0x9097, 0x0d18, 0xffff0000);
703 nv_mthd(dev, 0x9097, 0x0d20, 0xffff0000);
704 nv_mthd(dev, 0x9097, 0x0d28, 0xffff0000);
705 nv_mthd(dev, 0x9097, 0x0d30, 0xffff0000);
706 nv_mthd(dev, 0x9097, 0x0d38, 0xffff0000);
707 nv_mthd(dev, 0x9097, 0x0d04, 0xffff0000);
708 nv_mthd(dev, 0x9097, 0x0d0c, 0xffff0000);
709 nv_mthd(dev, 0x9097, 0x0d14, 0xffff0000);
710 nv_mthd(dev, 0x9097, 0x0d1c, 0xffff0000);
711 nv_mthd(dev, 0x9097, 0x0d24, 0xffff0000);
712 nv_mthd(dev, 0x9097, 0x0d2c, 0xffff0000);
713 nv_mthd(dev, 0x9097, 0x0d34, 0xffff0000);
714 nv_mthd(dev, 0x9097, 0x0d3c, 0xffff0000);
715 nv_mthd(dev, 0x9097, 0x0e00, 0x00000000);
716 nv_mthd(dev, 0x9097, 0x0e10, 0x00000000);
717 nv_mthd(dev, 0x9097, 0x0e20, 0x00000000);
718 nv_mthd(dev, 0x9097, 0x0e30, 0x00000000);
719 nv_mthd(dev, 0x9097, 0x0e40, 0x00000000);
720 nv_mthd(dev, 0x9097, 0x0e50, 0x00000000);
721 nv_mthd(dev, 0x9097, 0x0e60, 0x00000000);
722 nv_mthd(dev, 0x9097, 0x0e70, 0x00000000);
723 nv_mthd(dev, 0x9097, 0x0e80, 0x00000000);
724 nv_mthd(dev, 0x9097, 0x0e90, 0x00000000);
725 nv_mthd(dev, 0x9097, 0x0ea0, 0x00000000);
726 nv_mthd(dev, 0x9097, 0x0eb0, 0x00000000);
727 nv_mthd(dev, 0x9097, 0x0ec0, 0x00000000);
728 nv_mthd(dev, 0x9097, 0x0ed0, 0x00000000);
729 nv_mthd(dev, 0x9097, 0x0ee0, 0x00000000);
730 nv_mthd(dev, 0x9097, 0x0ef0, 0x00000000);
731 nv_mthd(dev, 0x9097, 0x0e04, 0xffff0000);
732 nv_mthd(dev, 0x9097, 0x0e14, 0xffff0000);
733 nv_mthd(dev, 0x9097, 0x0e24, 0xffff0000);
734 nv_mthd(dev, 0x9097, 0x0e34, 0xffff0000);
735 nv_mthd(dev, 0x9097, 0x0e44, 0xffff0000);
736 nv_mthd(dev, 0x9097, 0x0e54, 0xffff0000);
737 nv_mthd(dev, 0x9097, 0x0e64, 0xffff0000);
738 nv_mthd(dev, 0x9097, 0x0e74, 0xffff0000);
739 nv_mthd(dev, 0x9097, 0x0e84, 0xffff0000);
740 nv_mthd(dev, 0x9097, 0x0e94, 0xffff0000);
741 nv_mthd(dev, 0x9097, 0x0ea4, 0xffff0000);
742 nv_mthd(dev, 0x9097, 0x0eb4, 0xffff0000);
743 nv_mthd(dev, 0x9097, 0x0ec4, 0xffff0000);
744 nv_mthd(dev, 0x9097, 0x0ed4, 0xffff0000);
745 nv_mthd(dev, 0x9097, 0x0ee4, 0xffff0000);
746 nv_mthd(dev, 0x9097, 0x0ef4, 0xffff0000);
747 nv_mthd(dev, 0x9097, 0x0e08, 0xffff0000);
748 nv_mthd(dev, 0x9097, 0x0e18, 0xffff0000);
749 nv_mthd(dev, 0x9097, 0x0e28, 0xffff0000);
750 nv_mthd(dev, 0x9097, 0x0e38, 0xffff0000);
751 nv_mthd(dev, 0x9097, 0x0e48, 0xffff0000);
752 nv_mthd(dev, 0x9097, 0x0e58, 0xffff0000);
753 nv_mthd(dev, 0x9097, 0x0e68, 0xffff0000);
754 nv_mthd(dev, 0x9097, 0x0e78, 0xffff0000);
755 nv_mthd(dev, 0x9097, 0x0e88, 0xffff0000);
756 nv_mthd(dev, 0x9097, 0x0e98, 0xffff0000);
757 nv_mthd(dev, 0x9097, 0x0ea8, 0xffff0000);
758 nv_mthd(dev, 0x9097, 0x0eb8, 0xffff0000);
759 nv_mthd(dev, 0x9097, 0x0ec8, 0xffff0000);
760 nv_mthd(dev, 0x9097, 0x0ed8, 0xffff0000);
761 nv_mthd(dev, 0x9097, 0x0ee8, 0xffff0000);
762 nv_mthd(dev, 0x9097, 0x0ef8, 0xffff0000);
763 nv_mthd(dev, 0x9097, 0x0d40, 0x00000000);
764 nv_mthd(dev, 0x9097, 0x0d48, 0x00000000);
765 nv_mthd(dev, 0x9097, 0x0d50, 0x00000000);
766 nv_mthd(dev, 0x9097, 0x0d58, 0x00000000);
767 nv_mthd(dev, 0x9097, 0x0d44, 0x00000000);
768 nv_mthd(dev, 0x9097, 0x0d4c, 0x00000000);
769 nv_mthd(dev, 0x9097, 0x0d54, 0x00000000);
770 nv_mthd(dev, 0x9097, 0x0d5c, 0x00000000);
771 nv_mthd(dev, 0x9097, 0x1e00, 0x00000001);
772 nv_mthd(dev, 0x9097, 0x1e20, 0x00000001);
773 nv_mthd(dev, 0x9097, 0x1e40, 0x00000001);
774 nv_mthd(dev, 0x9097, 0x1e60, 0x00000001);
775 nv_mthd(dev, 0x9097, 0x1e80, 0x00000001);
776 nv_mthd(dev, 0x9097, 0x1ea0, 0x00000001);
777 nv_mthd(dev, 0x9097, 0x1ec0, 0x00000001);
778 nv_mthd(dev, 0x9097, 0x1ee0, 0x00000001);
779 nv_mthd(dev, 0x9097, 0x1e04, 0x00000001);
780 nv_mthd(dev, 0x9097, 0x1e24, 0x00000001);
781 nv_mthd(dev, 0x9097, 0x1e44, 0x00000001);
782 nv_mthd(dev, 0x9097, 0x1e64, 0x00000001);
783 nv_mthd(dev, 0x9097, 0x1e84, 0x00000001);
784 nv_mthd(dev, 0x9097, 0x1ea4, 0x00000001);
785 nv_mthd(dev, 0x9097, 0x1ec4, 0x00000001);
786 nv_mthd(dev, 0x9097, 0x1ee4, 0x00000001);
787 nv_mthd(dev, 0x9097, 0x1e08, 0x00000002);
788 nv_mthd(dev, 0x9097, 0x1e28, 0x00000002);
789 nv_mthd(dev, 0x9097, 0x1e48, 0x00000002);
790 nv_mthd(dev, 0x9097, 0x1e68, 0x00000002);
791 nv_mthd(dev, 0x9097, 0x1e88, 0x00000002);
792 nv_mthd(dev, 0x9097, 0x1ea8, 0x00000002);
793 nv_mthd(dev, 0x9097, 0x1ec8, 0x00000002);
794 nv_mthd(dev, 0x9097, 0x1ee8, 0x00000002);
795 nv_mthd(dev, 0x9097, 0x1e0c, 0x00000001);
796 nv_mthd(dev, 0x9097, 0x1e2c, 0x00000001);
797 nv_mthd(dev, 0x9097, 0x1e4c, 0x00000001);
798 nv_mthd(dev, 0x9097, 0x1e6c, 0x00000001);
799 nv_mthd(dev, 0x9097, 0x1e8c, 0x00000001);
800 nv_mthd(dev, 0x9097, 0x1eac, 0x00000001);
801 nv_mthd(dev, 0x9097, 0x1ecc, 0x00000001);
802 nv_mthd(dev, 0x9097, 0x1eec, 0x00000001);
803 nv_mthd(dev, 0x9097, 0x1e10, 0x00000001);
804 nv_mthd(dev, 0x9097, 0x1e30, 0x00000001);
805 nv_mthd(dev, 0x9097, 0x1e50, 0x00000001);
806 nv_mthd(dev, 0x9097, 0x1e70, 0x00000001);
807 nv_mthd(dev, 0x9097, 0x1e90, 0x00000001);
808 nv_mthd(dev, 0x9097, 0x1eb0, 0x00000001);
809 nv_mthd(dev, 0x9097, 0x1ed0, 0x00000001);
810 nv_mthd(dev, 0x9097, 0x1ef0, 0x00000001);
811 nv_mthd(dev, 0x9097, 0x1e14, 0x00000002);
812 nv_mthd(dev, 0x9097, 0x1e34, 0x00000002);
813 nv_mthd(dev, 0x9097, 0x1e54, 0x00000002);
814 nv_mthd(dev, 0x9097, 0x1e74, 0x00000002);
815 nv_mthd(dev, 0x9097, 0x1e94, 0x00000002);
816 nv_mthd(dev, 0x9097, 0x1eb4, 0x00000002);
817 nv_mthd(dev, 0x9097, 0x1ed4, 0x00000002);
818 nv_mthd(dev, 0x9097, 0x1ef4, 0x00000002);
819 nv_mthd(dev, 0x9097, 0x1e18, 0x00000001);
820 nv_mthd(dev, 0x9097, 0x1e38, 0x00000001);
821 nv_mthd(dev, 0x9097, 0x1e58, 0x00000001);
822 nv_mthd(dev, 0x9097, 0x1e78, 0x00000001);
823 nv_mthd(dev, 0x9097, 0x1e98, 0x00000001);
824 nv_mthd(dev, 0x9097, 0x1eb8, 0x00000001);
825 nv_mthd(dev, 0x9097, 0x1ed8, 0x00000001);
826 nv_mthd(dev, 0x9097, 0x1ef8, 0x00000001);
827 nv_mthd(dev, 0x9097, 0x3400, 0x00000000);
828 nv_mthd(dev, 0x9097, 0x3404, 0x00000000);
829 nv_mthd(dev, 0x9097, 0x3408, 0x00000000);
830 nv_mthd(dev, 0x9097, 0x340c, 0x00000000);
831 nv_mthd(dev, 0x9097, 0x3410, 0x00000000);
832 nv_mthd(dev, 0x9097, 0x3414, 0x00000000);
833 nv_mthd(dev, 0x9097, 0x3418, 0x00000000);
834 nv_mthd(dev, 0x9097, 0x341c, 0x00000000);
835 nv_mthd(dev, 0x9097, 0x3420, 0x00000000);
836 nv_mthd(dev, 0x9097, 0x3424, 0x00000000);
837 nv_mthd(dev, 0x9097, 0x3428, 0x00000000);
838 nv_mthd(dev, 0x9097, 0x342c, 0x00000000);
839 nv_mthd(dev, 0x9097, 0x3430, 0x00000000);
840 nv_mthd(dev, 0x9097, 0x3434, 0x00000000);
841 nv_mthd(dev, 0x9097, 0x3438, 0x00000000);
842 nv_mthd(dev, 0x9097, 0x343c, 0x00000000);
843 nv_mthd(dev, 0x9097, 0x3440, 0x00000000);
844 nv_mthd(dev, 0x9097, 0x3444, 0x00000000);
845 nv_mthd(dev, 0x9097, 0x3448, 0x00000000);
846 nv_mthd(dev, 0x9097, 0x344c, 0x00000000);
847 nv_mthd(dev, 0x9097, 0x3450, 0x00000000);
848 nv_mthd(dev, 0x9097, 0x3454, 0x00000000);
849 nv_mthd(dev, 0x9097, 0x3458, 0x00000000);
850 nv_mthd(dev, 0x9097, 0x345c, 0x00000000);
851 nv_mthd(dev, 0x9097, 0x3460, 0x00000000);
852 nv_mthd(dev, 0x9097, 0x3464, 0x00000000);
853 nv_mthd(dev, 0x9097, 0x3468, 0x00000000);
854 nv_mthd(dev, 0x9097, 0x346c, 0x00000000);
855 nv_mthd(dev, 0x9097, 0x3470, 0x00000000);
856 nv_mthd(dev, 0x9097, 0x3474, 0x00000000);
857 nv_mthd(dev, 0x9097, 0x3478, 0x00000000);
858 nv_mthd(dev, 0x9097, 0x347c, 0x00000000);
859 nv_mthd(dev, 0x9097, 0x3480, 0x00000000);
860 nv_mthd(dev, 0x9097, 0x3484, 0x00000000);
861 nv_mthd(dev, 0x9097, 0x3488, 0x00000000);
862 nv_mthd(dev, 0x9097, 0x348c, 0x00000000);
863 nv_mthd(dev, 0x9097, 0x3490, 0x00000000);
864 nv_mthd(dev, 0x9097, 0x3494, 0x00000000);
865 nv_mthd(dev, 0x9097, 0x3498, 0x00000000);
866 nv_mthd(dev, 0x9097, 0x349c, 0x00000000);
867 nv_mthd(dev, 0x9097, 0x34a0, 0x00000000);
868 nv_mthd(dev, 0x9097, 0x34a4, 0x00000000);
869 nv_mthd(dev, 0x9097, 0x34a8, 0x00000000);
870 nv_mthd(dev, 0x9097, 0x34ac, 0x00000000);
871 nv_mthd(dev, 0x9097, 0x34b0, 0x00000000);
872 nv_mthd(dev, 0x9097, 0x34b4, 0x00000000);
873 nv_mthd(dev, 0x9097, 0x34b8, 0x00000000);
874 nv_mthd(dev, 0x9097, 0x34bc, 0x00000000);
875 nv_mthd(dev, 0x9097, 0x34c0, 0x00000000);
876 nv_mthd(dev, 0x9097, 0x34c4, 0x00000000);
877 nv_mthd(dev, 0x9097, 0x34c8, 0x00000000);
878 nv_mthd(dev, 0x9097, 0x34cc, 0x00000000);
879 nv_mthd(dev, 0x9097, 0x34d0, 0x00000000);
880 nv_mthd(dev, 0x9097, 0x34d4, 0x00000000);
881 nv_mthd(dev, 0x9097, 0x34d8, 0x00000000);
882 nv_mthd(dev, 0x9097, 0x34dc, 0x00000000);
883 nv_mthd(dev, 0x9097, 0x34e0, 0x00000000);
884 nv_mthd(dev, 0x9097, 0x34e4, 0x00000000);
885 nv_mthd(dev, 0x9097, 0x34e8, 0x00000000);
886 nv_mthd(dev, 0x9097, 0x34ec, 0x00000000);
887 nv_mthd(dev, 0x9097, 0x34f0, 0x00000000);
888 nv_mthd(dev, 0x9097, 0x34f4, 0x00000000);
889 nv_mthd(dev, 0x9097, 0x34f8, 0x00000000);
890 nv_mthd(dev, 0x9097, 0x34fc, 0x00000000);
891 nv_mthd(dev, 0x9097, 0x3500, 0x00000000);
892 nv_mthd(dev, 0x9097, 0x3504, 0x00000000);
893 nv_mthd(dev, 0x9097, 0x3508, 0x00000000);
894 nv_mthd(dev, 0x9097, 0x350c, 0x00000000);
895 nv_mthd(dev, 0x9097, 0x3510, 0x00000000);
896 nv_mthd(dev, 0x9097, 0x3514, 0x00000000);
897 nv_mthd(dev, 0x9097, 0x3518, 0x00000000);
898 nv_mthd(dev, 0x9097, 0x351c, 0x00000000);
899 nv_mthd(dev, 0x9097, 0x3520, 0x00000000);
900 nv_mthd(dev, 0x9097, 0x3524, 0x00000000);
901 nv_mthd(dev, 0x9097, 0x3528, 0x00000000);
902 nv_mthd(dev, 0x9097, 0x352c, 0x00000000);
903 nv_mthd(dev, 0x9097, 0x3530, 0x00000000);
904 nv_mthd(dev, 0x9097, 0x3534, 0x00000000);
905 nv_mthd(dev, 0x9097, 0x3538, 0x00000000);
906 nv_mthd(dev, 0x9097, 0x353c, 0x00000000);
907 nv_mthd(dev, 0x9097, 0x3540, 0x00000000);
908 nv_mthd(dev, 0x9097, 0x3544, 0x00000000);
909 nv_mthd(dev, 0x9097, 0x3548, 0x00000000);
910 nv_mthd(dev, 0x9097, 0x354c, 0x00000000);
911 nv_mthd(dev, 0x9097, 0x3550, 0x00000000);
912 nv_mthd(dev, 0x9097, 0x3554, 0x00000000);
913 nv_mthd(dev, 0x9097, 0x3558, 0x00000000);
914 nv_mthd(dev, 0x9097, 0x355c, 0x00000000);
915 nv_mthd(dev, 0x9097, 0x3560, 0x00000000);
916 nv_mthd(dev, 0x9097, 0x3564, 0x00000000);
917 nv_mthd(dev, 0x9097, 0x3568, 0x00000000);
918 nv_mthd(dev, 0x9097, 0x356c, 0x00000000);
919 nv_mthd(dev, 0x9097, 0x3570, 0x00000000);
920 nv_mthd(dev, 0x9097, 0x3574, 0x00000000);
921 nv_mthd(dev, 0x9097, 0x3578, 0x00000000);
922 nv_mthd(dev, 0x9097, 0x357c, 0x00000000);
923 nv_mthd(dev, 0x9097, 0x3580, 0x00000000);
924 nv_mthd(dev, 0x9097, 0x3584, 0x00000000);
925 nv_mthd(dev, 0x9097, 0x3588, 0x00000000);
926 nv_mthd(dev, 0x9097, 0x358c, 0x00000000);
927 nv_mthd(dev, 0x9097, 0x3590, 0x00000000);
928 nv_mthd(dev, 0x9097, 0x3594, 0x00000000);
929 nv_mthd(dev, 0x9097, 0x3598, 0x00000000);
930 nv_mthd(dev, 0x9097, 0x359c, 0x00000000);
931 nv_mthd(dev, 0x9097, 0x35a0, 0x00000000);
932 nv_mthd(dev, 0x9097, 0x35a4, 0x00000000);
933 nv_mthd(dev, 0x9097, 0x35a8, 0x00000000);
934 nv_mthd(dev, 0x9097, 0x35ac, 0x00000000);
935 nv_mthd(dev, 0x9097, 0x35b0, 0x00000000);
936 nv_mthd(dev, 0x9097, 0x35b4, 0x00000000);
937 nv_mthd(dev, 0x9097, 0x35b8, 0x00000000);
938 nv_mthd(dev, 0x9097, 0x35bc, 0x00000000);
939 nv_mthd(dev, 0x9097, 0x35c0, 0x00000000);
940 nv_mthd(dev, 0x9097, 0x35c4, 0x00000000);
941 nv_mthd(dev, 0x9097, 0x35c8, 0x00000000);
942 nv_mthd(dev, 0x9097, 0x35cc, 0x00000000);
943 nv_mthd(dev, 0x9097, 0x35d0, 0x00000000);
944 nv_mthd(dev, 0x9097, 0x35d4, 0x00000000);
945 nv_mthd(dev, 0x9097, 0x35d8, 0x00000000);
946 nv_mthd(dev, 0x9097, 0x35dc, 0x00000000);
947 nv_mthd(dev, 0x9097, 0x35e0, 0x00000000);
948 nv_mthd(dev, 0x9097, 0x35e4, 0x00000000);
949 nv_mthd(dev, 0x9097, 0x35e8, 0x00000000);
950 nv_mthd(dev, 0x9097, 0x35ec, 0x00000000);
951 nv_mthd(dev, 0x9097, 0x35f0, 0x00000000);
952 nv_mthd(dev, 0x9097, 0x35f4, 0x00000000);
953 nv_mthd(dev, 0x9097, 0x35f8, 0x00000000);
954 nv_mthd(dev, 0x9097, 0x35fc, 0x00000000);
955 nv_mthd(dev, 0x9097, 0x030c, 0x00000001);
956 nv_mthd(dev, 0x9097, 0x1944, 0x00000000);
957 nv_mthd(dev, 0x9097, 0x1514, 0x00000000);
958 nv_mthd(dev, 0x9097, 0x0d68, 0x0000ffff);
959 nv_mthd(dev, 0x9097, 0x121c, 0x0fac6881);
960 nv_mthd(dev, 0x9097, 0x0fac, 0x00000001);
961 nv_mthd(dev, 0x9097, 0x1538, 0x00000001);
962 nv_mthd(dev, 0x9097, 0x0fe0, 0x00000000);
963 nv_mthd(dev, 0x9097, 0x0fe4, 0x00000000);
964 nv_mthd(dev, 0x9097, 0x0fe8, 0x00000014);
965 nv_mthd(dev, 0x9097, 0x0fec, 0x00000040);
966 nv_mthd(dev, 0x9097, 0x0ff0, 0x00000000);
967 nv_mthd(dev, 0x9097, 0x179c, 0x00000000);
968 nv_mthd(dev, 0x9097, 0x1228, 0x00000400);
969 nv_mthd(dev, 0x9097, 0x122c, 0x00000300);
970 nv_mthd(dev, 0x9097, 0x1230, 0x00010001);
971 nv_mthd(dev, 0x9097, 0x07f8, 0x00000000);
972 nv_mthd(dev, 0x9097, 0x15b4, 0x00000001);
973 nv_mthd(dev, 0x9097, 0x15cc, 0x00000000);
974 nv_mthd(dev, 0x9097, 0x1534, 0x00000000);
975 nv_mthd(dev, 0x9097, 0x0fb0, 0x00000000);
976 nv_mthd(dev, 0x9097, 0x15d0, 0x00000000);
977 nv_mthd(dev, 0x9097, 0x153c, 0x00000000);
978 nv_mthd(dev, 0x9097, 0x16b4, 0x00000003);
979 nv_mthd(dev, 0x9097, 0x0fbc, 0x0000ffff);
980 nv_mthd(dev, 0x9097, 0x0fc0, 0x0000ffff);
981 nv_mthd(dev, 0x9097, 0x0fc4, 0x0000ffff);
982 nv_mthd(dev, 0x9097, 0x0fc8, 0x0000ffff);
983 nv_mthd(dev, 0x9097, 0x0df8, 0x00000000);
984 nv_mthd(dev, 0x9097, 0x0dfc, 0x00000000);
985 nv_mthd(dev, 0x9097, 0x1948, 0x00000000);
986 nv_mthd(dev, 0x9097, 0x1970, 0x00000001);
987 nv_mthd(dev, 0x9097, 0x161c, 0x000009f0);
988 nv_mthd(dev, 0x9097, 0x0dcc, 0x00000010);
989 nv_mthd(dev, 0x9097, 0x163c, 0x00000000);
990 nv_mthd(dev, 0x9097, 0x15e4, 0x00000000);
991 nv_mthd(dev, 0x9097, 0x1160, 0x25e00040);
992 nv_mthd(dev, 0x9097, 0x1164, 0x25e00040);
993 nv_mthd(dev, 0x9097, 0x1168, 0x25e00040);
994 nv_mthd(dev, 0x9097, 0x116c, 0x25e00040);
995 nv_mthd(dev, 0x9097, 0x1170, 0x25e00040);
996 nv_mthd(dev, 0x9097, 0x1174, 0x25e00040);
997 nv_mthd(dev, 0x9097, 0x1178, 0x25e00040);
998 nv_mthd(dev, 0x9097, 0x117c, 0x25e00040);
999 nv_mthd(dev, 0x9097, 0x1180, 0x25e00040);
1000 nv_mthd(dev, 0x9097, 0x1184, 0x25e00040);
1001 nv_mthd(dev, 0x9097, 0x1188, 0x25e00040);
1002 nv_mthd(dev, 0x9097, 0x118c, 0x25e00040);
1003 nv_mthd(dev, 0x9097, 0x1190, 0x25e00040);
1004 nv_mthd(dev, 0x9097, 0x1194, 0x25e00040);
1005 nv_mthd(dev, 0x9097, 0x1198, 0x25e00040);
1006 nv_mthd(dev, 0x9097, 0x119c, 0x25e00040);
1007 nv_mthd(dev, 0x9097, 0x11a0, 0x25e00040);
1008 nv_mthd(dev, 0x9097, 0x11a4, 0x25e00040);
1009 nv_mthd(dev, 0x9097, 0x11a8, 0x25e00040);
1010 nv_mthd(dev, 0x9097, 0x11ac, 0x25e00040);
1011 nv_mthd(dev, 0x9097, 0x11b0, 0x25e00040);
1012 nv_mthd(dev, 0x9097, 0x11b4, 0x25e00040);
1013 nv_mthd(dev, 0x9097, 0x11b8, 0x25e00040);
1014 nv_mthd(dev, 0x9097, 0x11bc, 0x25e00040);
1015 nv_mthd(dev, 0x9097, 0x11c0, 0x25e00040);
1016 nv_mthd(dev, 0x9097, 0x11c4, 0x25e00040);
1017 nv_mthd(dev, 0x9097, 0x11c8, 0x25e00040);
1018 nv_mthd(dev, 0x9097, 0x11cc, 0x25e00040);
1019 nv_mthd(dev, 0x9097, 0x11d0, 0x25e00040);
1020 nv_mthd(dev, 0x9097, 0x11d4, 0x25e00040);
1021 nv_mthd(dev, 0x9097, 0x11d8, 0x25e00040);
1022 nv_mthd(dev, 0x9097, 0x11dc, 0x25e00040);
1023 nv_mthd(dev, 0x9097, 0x1880, 0x00000000);
1024 nv_mthd(dev, 0x9097, 0x1884, 0x00000000);
1025 nv_mthd(dev, 0x9097, 0x1888, 0x00000000);
1026 nv_mthd(dev, 0x9097, 0x188c, 0x00000000);
1027 nv_mthd(dev, 0x9097, 0x1890, 0x00000000);
1028 nv_mthd(dev, 0x9097, 0x1894, 0x00000000);
1029 nv_mthd(dev, 0x9097, 0x1898, 0x00000000);
1030 nv_mthd(dev, 0x9097, 0x189c, 0x00000000);
1031 nv_mthd(dev, 0x9097, 0x18a0, 0x00000000);
1032 nv_mthd(dev, 0x9097, 0x18a4, 0x00000000);
1033 nv_mthd(dev, 0x9097, 0x18a8, 0x00000000);
1034 nv_mthd(dev, 0x9097, 0x18ac, 0x00000000);
1035 nv_mthd(dev, 0x9097, 0x18b0, 0x00000000);
1036 nv_mthd(dev, 0x9097, 0x18b4, 0x00000000);
1037 nv_mthd(dev, 0x9097, 0x18b8, 0x00000000);
1038 nv_mthd(dev, 0x9097, 0x18bc, 0x00000000);
1039 nv_mthd(dev, 0x9097, 0x18c0, 0x00000000);
1040 nv_mthd(dev, 0x9097, 0x18c4, 0x00000000);
1041 nv_mthd(dev, 0x9097, 0x18c8, 0x00000000);
1042 nv_mthd(dev, 0x9097, 0x18cc, 0x00000000);
1043 nv_mthd(dev, 0x9097, 0x18d0, 0x00000000);
1044 nv_mthd(dev, 0x9097, 0x18d4, 0x00000000);
1045 nv_mthd(dev, 0x9097, 0x18d8, 0x00000000);
1046 nv_mthd(dev, 0x9097, 0x18dc, 0x00000000);
1047 nv_mthd(dev, 0x9097, 0x18e0, 0x00000000);
1048 nv_mthd(dev, 0x9097, 0x18e4, 0x00000000);
1049 nv_mthd(dev, 0x9097, 0x18e8, 0x00000000);
1050 nv_mthd(dev, 0x9097, 0x18ec, 0x00000000);
1051 nv_mthd(dev, 0x9097, 0x18f0, 0x00000000);
1052 nv_mthd(dev, 0x9097, 0x18f4, 0x00000000);
1053 nv_mthd(dev, 0x9097, 0x18f8, 0x00000000);
1054 nv_mthd(dev, 0x9097, 0x18fc, 0x00000000);
1055 nv_mthd(dev, 0x9097, 0x0f84, 0x00000000);
1056 nv_mthd(dev, 0x9097, 0x0f88, 0x00000000);
1057 nv_mthd(dev, 0x9097, 0x17c8, 0x00000000);
1058 nv_mthd(dev, 0x9097, 0x17cc, 0x00000000);
1059 nv_mthd(dev, 0x9097, 0x17d0, 0x000000ff);
1060 nv_mthd(dev, 0x9097, 0x17d4, 0xffffffff);
1061 nv_mthd(dev, 0x9097, 0x17d8, 0x00000002);
1062 nv_mthd(dev, 0x9097, 0x17dc, 0x00000000);
1063 nv_mthd(dev, 0x9097, 0x15f4, 0x00000000);
1064 nv_mthd(dev, 0x9097, 0x15f8, 0x00000000);
1065 nv_mthd(dev, 0x9097, 0x1434, 0x00000000);
1066 nv_mthd(dev, 0x9097, 0x1438, 0x00000000);
1067 nv_mthd(dev, 0x9097, 0x0d74, 0x00000000);
1068 nv_mthd(dev, 0x9097, 0x0dec, 0x00000001);
1069 nv_mthd(dev, 0x9097, 0x13a4, 0x00000000);
1070 nv_mthd(dev, 0x9097, 0x1318, 0x00000001);
1071 nv_mthd(dev, 0x9097, 0x1644, 0x00000000);
1072 nv_mthd(dev, 0x9097, 0x0748, 0x00000000);
1073 nv_mthd(dev, 0x9097, 0x0de8, 0x00000000);
1074 nv_mthd(dev, 0x9097, 0x1648, 0x00000000);
1075 nv_mthd(dev, 0x9097, 0x12a4, 0x00000000);
1076 nv_mthd(dev, 0x9097, 0x1120, 0x00000000);
1077 nv_mthd(dev, 0x9097, 0x1124, 0x00000000);
1078 nv_mthd(dev, 0x9097, 0x1128, 0x00000000);
1079 nv_mthd(dev, 0x9097, 0x112c, 0x00000000);
1080 nv_mthd(dev, 0x9097, 0x1118, 0x00000000);
1081 nv_mthd(dev, 0x9097, 0x164c, 0x00000000);
1082 nv_mthd(dev, 0x9097, 0x1658, 0x00000000);
1083 nv_mthd(dev, 0x9097, 0x1910, 0x00000290);
1084 nv_mthd(dev, 0x9097, 0x1518, 0x00000000);
1085 nv_mthd(dev, 0x9097, 0x165c, 0x00000001);
1086 nv_mthd(dev, 0x9097, 0x1520, 0x00000000);
1087 nv_mthd(dev, 0x9097, 0x1604, 0x00000000);
1088 nv_mthd(dev, 0x9097, 0x1570, 0x00000000);
1089 nv_mthd(dev, 0x9097, 0x13b0, 0x3f800000);
1090 nv_mthd(dev, 0x9097, 0x13b4, 0x3f800000);
1091 nv_mthd(dev, 0x9097, 0x020c, 0x00000000);
1092 nv_mthd(dev, 0x9097, 0x1670, 0x30201000);
1093 nv_mthd(dev, 0x9097, 0x1674, 0x70605040);
1094 nv_mthd(dev, 0x9097, 0x1678, 0xb8a89888);
1095 nv_mthd(dev, 0x9097, 0x167c, 0xf8e8d8c8);
1096 nv_mthd(dev, 0x9097, 0x166c, 0x00000000);
1097 nv_mthd(dev, 0x9097, 0x1680, 0x00ffff00);
1098 nv_mthd(dev, 0x9097, 0x12d0, 0x00000003);
1099 nv_mthd(dev, 0x9097, 0x12d4, 0x00000002);
1100 nv_mthd(dev, 0x9097, 0x1684, 0x00000000);
1101 nv_mthd(dev, 0x9097, 0x1688, 0x00000000);
1102 nv_mthd(dev, 0x9097, 0x0dac, 0x00001b02);
1103 nv_mthd(dev, 0x9097, 0x0db0, 0x00001b02);
1104 nv_mthd(dev, 0x9097, 0x0db4, 0x00000000);
1105 nv_mthd(dev, 0x9097, 0x168c, 0x00000000);
1106 nv_mthd(dev, 0x9097, 0x15bc, 0x00000000);
1107 nv_mthd(dev, 0x9097, 0x156c, 0x00000000);
1108 nv_mthd(dev, 0x9097, 0x187c, 0x00000000);
1109 nv_mthd(dev, 0x9097, 0x1110, 0x00000001);
1110 nv_mthd(dev, 0x9097, 0x0dc0, 0x00000000);
1111 nv_mthd(dev, 0x9097, 0x0dc4, 0x00000000);
1112 nv_mthd(dev, 0x9097, 0x0dc8, 0x00000000);
1113 nv_mthd(dev, 0x9097, 0x1234, 0x00000000);
1114 nv_mthd(dev, 0x9097, 0x1690, 0x00000000);
1115 nv_mthd(dev, 0x9097, 0x12ac, 0x00000001);
1116 nv_mthd(dev, 0x9097, 0x02c4, 0x00000000);
1117 nv_mthd(dev, 0x9097, 0x0790, 0x00000000);
1118 nv_mthd(dev, 0x9097, 0x0794, 0x00000000);
1119 nv_mthd(dev, 0x9097, 0x0798, 0x00000000);
1120 nv_mthd(dev, 0x9097, 0x079c, 0x00000000);
1121 nv_mthd(dev, 0x9097, 0x07a0, 0x00000000);
1122 nv_mthd(dev, 0x9097, 0x077c, 0x00000000);
1123 nv_mthd(dev, 0x9097, 0x1000, 0x00000010);
1124 nv_mthd(dev, 0x9097, 0x10fc, 0x00000000);
1125 nv_mthd(dev, 0x9097, 0x1290, 0x00000000);
1126 nv_mthd(dev, 0x9097, 0x0218, 0x00000010);
1127 nv_mthd(dev, 0x9097, 0x12d8, 0x00000000);
1128 nv_mthd(dev, 0x9097, 0x12dc, 0x00000010);
1129 nv_mthd(dev, 0x9097, 0x0d94, 0x00000001);
1130 nv_mthd(dev, 0x9097, 0x155c, 0x00000000);
1131 nv_mthd(dev, 0x9097, 0x1560, 0x00000000);
1132 nv_mthd(dev, 0x9097, 0x1564, 0x00001fff);
1133 nv_mthd(dev, 0x9097, 0x1574, 0x00000000);
1134 nv_mthd(dev, 0x9097, 0x1578, 0x00000000);
1135 nv_mthd(dev, 0x9097, 0x157c, 0x003fffff);
1136 nv_mthd(dev, 0x9097, 0x1354, 0x00000000);
1137 nv_mthd(dev, 0x9097, 0x1664, 0x00000000);
1138 nv_mthd(dev, 0x9097, 0x1610, 0x00000012);
1139 nv_mthd(dev, 0x9097, 0x1608, 0x00000000);
1140 nv_mthd(dev, 0x9097, 0x160c, 0x00000000);
1141 nv_mthd(dev, 0x9097, 0x162c, 0x00000003);
1142 nv_mthd(dev, 0x9097, 0x0210, 0x00000000);
1143 nv_mthd(dev, 0x9097, 0x0320, 0x00000000);
1144 nv_mthd(dev, 0x9097, 0x0324, 0x3f800000);
1145 nv_mthd(dev, 0x9097, 0x0328, 0x3f800000);
1146 nv_mthd(dev, 0x9097, 0x032c, 0x3f800000);
1147 nv_mthd(dev, 0x9097, 0x0330, 0x3f800000);
1148 nv_mthd(dev, 0x9097, 0x0334, 0x3f800000);
1149 nv_mthd(dev, 0x9097, 0x0338, 0x3f800000);
1150 nv_mthd(dev, 0x9097, 0x0750, 0x00000000);
1151 nv_mthd(dev, 0x9097, 0x0760, 0x39291909);
1152 nv_mthd(dev, 0x9097, 0x0764, 0x79695949);
1153 nv_mthd(dev, 0x9097, 0x0768, 0xb9a99989);
1154 nv_mthd(dev, 0x9097, 0x076c, 0xf9e9d9c9);
1155 nv_mthd(dev, 0x9097, 0x0770, 0x30201000);
1156 nv_mthd(dev, 0x9097, 0x0774, 0x70605040);
1157 nv_mthd(dev, 0x9097, 0x0778, 0x00009080);
1158 nv_mthd(dev, 0x9097, 0x0780, 0x39291909);
1159 nv_mthd(dev, 0x9097, 0x0784, 0x79695949);
1160 nv_mthd(dev, 0x9097, 0x0788, 0xb9a99989);
1161 nv_mthd(dev, 0x9097, 0x078c, 0xf9e9d9c9);
1162 nv_mthd(dev, 0x9097, 0x07d0, 0x30201000);
1163 nv_mthd(dev, 0x9097, 0x07d4, 0x70605040);
1164 nv_mthd(dev, 0x9097, 0x07d8, 0x00009080);
1165 nv_mthd(dev, 0x9097, 0x037c, 0x00000001);
1166 nv_mthd(dev, 0x9097, 0x0740, 0x00000000);
1167 nv_mthd(dev, 0x9097, 0x0744, 0x00000000);
1168 nv_mthd(dev, 0x9097, 0x2600, 0x00000000);
1169 nv_mthd(dev, 0x9097, 0x1918, 0x00000000);
1170 nv_mthd(dev, 0x9097, 0x191c, 0x00000900);
1171 nv_mthd(dev, 0x9097, 0x1920, 0x00000405);
1172 nv_mthd(dev, 0x9097, 0x1308, 0x00000001);
1173 nv_mthd(dev, 0x9097, 0x1924, 0x00000000);
1174 nv_mthd(dev, 0x9097, 0x13ac, 0x00000000);
1175 nv_mthd(dev, 0x9097, 0x192c, 0x00000001);
1176 nv_mthd(dev, 0x9097, 0x193c, 0x00002c1c);
1177 nv_mthd(dev, 0x9097, 0x0d7c, 0x00000000);
1178 nv_mthd(dev, 0x9097, 0x0f8c, 0x00000000);
1179 nv_mthd(dev, 0x9097, 0x02c0, 0x00000001);
1180 nv_mthd(dev, 0x9097, 0x1510, 0x00000000);
1181 nv_mthd(dev, 0x9097, 0x1940, 0x00000000);
1182 nv_mthd(dev, 0x9097, 0x0ff4, 0x00000000);
1183 nv_mthd(dev, 0x9097, 0x0ff8, 0x00000000);
1184 nv_mthd(dev, 0x9097, 0x194c, 0x00000000);
1185 nv_mthd(dev, 0x9097, 0x1950, 0x00000000);
1186 nv_mthd(dev, 0x9097, 0x1968, 0x00000000);
1187 nv_mthd(dev, 0x9097, 0x1590, 0x0000003f);
1188 nv_mthd(dev, 0x9097, 0x07e8, 0x00000000);
1189 nv_mthd(dev, 0x9097, 0x07ec, 0x00000000);
1190 nv_mthd(dev, 0x9097, 0x07f0, 0x00000000);
1191 nv_mthd(dev, 0x9097, 0x07f4, 0x00000000);
1192 nv_mthd(dev, 0x9097, 0x196c, 0x00000011);
1193 nv_mthd(dev, 0x9097, 0x197c, 0x00000000);
1194 nv_mthd(dev, 0x9097, 0x0fcc, 0x00000000);
1195 nv_mthd(dev, 0x9097, 0x0fd0, 0x00000000);
1196 nv_mthd(dev, 0x9097, 0x02d8, 0x00000040);
1197 nv_mthd(dev, 0x9097, 0x1980, 0x00000080);
1198 nv_mthd(dev, 0x9097, 0x1504, 0x00000080);
1199 nv_mthd(dev, 0x9097, 0x1984, 0x00000000);
1200 nv_mthd(dev, 0x9097, 0x0300, 0x00000001);
1201 nv_mthd(dev, 0x9097, 0x13a8, 0x00000000);
1202 nv_mthd(dev, 0x9097, 0x12ec, 0x00000000);
1203 nv_mthd(dev, 0x9097, 0x1310, 0x00000000);
1204 nv_mthd(dev, 0x9097, 0x1314, 0x00000001);
1205 nv_mthd(dev, 0x9097, 0x1380, 0x00000000);
1206 nv_mthd(dev, 0x9097, 0x1384, 0x00000001);
1207 nv_mthd(dev, 0x9097, 0x1388, 0x00000001);
1208 nv_mthd(dev, 0x9097, 0x138c, 0x00000001);
1209 nv_mthd(dev, 0x9097, 0x1390, 0x00000001);
1210 nv_mthd(dev, 0x9097, 0x1394, 0x00000000);
1211 nv_mthd(dev, 0x9097, 0x139c, 0x00000000);
1212 nv_mthd(dev, 0x9097, 0x1398, 0x00000000);
1213 nv_mthd(dev, 0x9097, 0x1594, 0x00000000);
1214 nv_mthd(dev, 0x9097, 0x1598, 0x00000001);
1215 nv_mthd(dev, 0x9097, 0x159c, 0x00000001);
1216 nv_mthd(dev, 0x9097, 0x15a0, 0x00000001);
1217 nv_mthd(dev, 0x9097, 0x15a4, 0x00000001);
1218 nv_mthd(dev, 0x9097, 0x0f54, 0x00000000);
1219 nv_mthd(dev, 0x9097, 0x0f58, 0x00000000);
1220 nv_mthd(dev, 0x9097, 0x0f5c, 0x00000000);
1221 nv_mthd(dev, 0x9097, 0x19bc, 0x00000000);
1222 nv_mthd(dev, 0x9097, 0x0f9c, 0x00000000);
1223 nv_mthd(dev, 0x9097, 0x0fa0, 0x00000000);
1224 nv_mthd(dev, 0x9097, 0x12cc, 0x00000000);
1225 nv_mthd(dev, 0x9097, 0x12e8, 0x00000000);
1226 nv_mthd(dev, 0x9097, 0x130c, 0x00000001);
1227 nv_mthd(dev, 0x9097, 0x1360, 0x00000000);
1228 nv_mthd(dev, 0x9097, 0x1364, 0x00000000);
1229 nv_mthd(dev, 0x9097, 0x1368, 0x00000000);
1230 nv_mthd(dev, 0x9097, 0x136c, 0x00000000);
1231 nv_mthd(dev, 0x9097, 0x1370, 0x00000000);
1232 nv_mthd(dev, 0x9097, 0x1374, 0x00000000);
1233 nv_mthd(dev, 0x9097, 0x1378, 0x00000000);
1234 nv_mthd(dev, 0x9097, 0x137c, 0x00000000);
1235 nv_mthd(dev, 0x9097, 0x133c, 0x00000001);
1236 nv_mthd(dev, 0x9097, 0x1340, 0x00000001);
1237 nv_mthd(dev, 0x9097, 0x1344, 0x00000002);
1238 nv_mthd(dev, 0x9097, 0x1348, 0x00000001);
1239 nv_mthd(dev, 0x9097, 0x134c, 0x00000001);
1240 nv_mthd(dev, 0x9097, 0x1350, 0x00000002);
1241 nv_mthd(dev, 0x9097, 0x1358, 0x00000001);
1242 nv_mthd(dev, 0x9097, 0x12e4, 0x00000000);
1243 nv_mthd(dev, 0x9097, 0x131c, 0x00000000);
1244 nv_mthd(dev, 0x9097, 0x1320, 0x00000000);
1245 nv_mthd(dev, 0x9097, 0x1324, 0x00000000);
1246 nv_mthd(dev, 0x9097, 0x1328, 0x00000000);
1247 nv_mthd(dev, 0x9097, 0x19c0, 0x00000000);
1248 nv_mthd(dev, 0x9097, 0x1140, 0x00000000);
1249 nv_mthd(dev, 0x9097, 0x19c4, 0x00000000);
1250 nv_mthd(dev, 0x9097, 0x19c8, 0x00001500);
1251 nv_mthd(dev, 0x9097, 0x135c, 0x00000000);
1252 nv_mthd(dev, 0x9097, 0x0f90, 0x00000000);
1253 nv_mthd(dev, 0x9097, 0x19e0, 0x00000001);
1254 nv_mthd(dev, 0x9097, 0x19e4, 0x00000001);
1255 nv_mthd(dev, 0x9097, 0x19e8, 0x00000001);
1256 nv_mthd(dev, 0x9097, 0x19ec, 0x00000001);
1257 nv_mthd(dev, 0x9097, 0x19f0, 0x00000001);
1258 nv_mthd(dev, 0x9097, 0x19f4, 0x00000001);
1259 nv_mthd(dev, 0x9097, 0x19f8, 0x00000001);
1260 nv_mthd(dev, 0x9097, 0x19fc, 0x00000001);
1261 nv_mthd(dev, 0x9097, 0x19cc, 0x00000001);
1262 nv_mthd(dev, 0x9097, 0x15b8, 0x00000000);
1263 nv_mthd(dev, 0x9097, 0x1a00, 0x00001111);
1264 nv_mthd(dev, 0x9097, 0x1a04, 0x00000000);
1265 nv_mthd(dev, 0x9097, 0x1a08, 0x00000000);
1266 nv_mthd(dev, 0x9097, 0x1a0c, 0x00000000);
1267 nv_mthd(dev, 0x9097, 0x1a10, 0x00000000);
1268 nv_mthd(dev, 0x9097, 0x1a14, 0x00000000);
1269 nv_mthd(dev, 0x9097, 0x1a18, 0x00000000);
1270 nv_mthd(dev, 0x9097, 0x1a1c, 0x00000000);
1271 nv_mthd(dev, 0x9097, 0x0d6c, 0xffff0000);
1272 nv_mthd(dev, 0x9097, 0x0d70, 0xffff0000);
1273 nv_mthd(dev, 0x9097, 0x10f8, 0x00001010);
1274 nv_mthd(dev, 0x9097, 0x0d80, 0x00000000);
1275 nv_mthd(dev, 0x9097, 0x0d84, 0x00000000);
1276 nv_mthd(dev, 0x9097, 0x0d88, 0x00000000);
1277 nv_mthd(dev, 0x9097, 0x0d8c, 0x00000000);
1278 nv_mthd(dev, 0x9097, 0x0d90, 0x00000000);
1279 nv_mthd(dev, 0x9097, 0x0da0, 0x00000000);
1280 nv_mthd(dev, 0x9097, 0x1508, 0x80000000);
1281 nv_mthd(dev, 0x9097, 0x150c, 0x40000000);
1282 nv_mthd(dev, 0x9097, 0x1668, 0x00000000);
1283 nv_mthd(dev, 0x9097, 0x0318, 0x00000008);
1284 nv_mthd(dev, 0x9097, 0x031c, 0x00000008);
1285 nv_mthd(dev, 0x9097, 0x0d9c, 0x00000001);
1286 nv_mthd(dev, 0x9097, 0x07dc, 0x00000000);
1287 nv_mthd(dev, 0x9097, 0x074c, 0x00000055);
1288 nv_mthd(dev, 0x9097, 0x1420, 0x00000003);
1289 nv_mthd(dev, 0x9097, 0x17bc, 0x00000000);
1290 nv_mthd(dev, 0x9097, 0x17c0, 0x00000000);
1291 nv_mthd(dev, 0x9097, 0x17c4, 0x00000001);
1292 nv_mthd(dev, 0x9097, 0x1008, 0x00000008);
1293 nv_mthd(dev, 0x9097, 0x100c, 0x00000040);
1294 nv_mthd(dev, 0x9097, 0x1010, 0x0000012c);
1295 nv_mthd(dev, 0x9097, 0x0d60, 0x00000040);
1296 nv_mthd(dev, 0x9097, 0x075c, 0x00000003);
1297 nv_mthd(dev, 0x9097, 0x1018, 0x00000020);
1298 nv_mthd(dev, 0x9097, 0x101c, 0x00000001);
1299 nv_mthd(dev, 0x9097, 0x1020, 0x00000020);
1300 nv_mthd(dev, 0x9097, 0x1024, 0x00000001);
1301 nv_mthd(dev, 0x9097, 0x1444, 0x00000000);
1302 nv_mthd(dev, 0x9097, 0x1448, 0x00000000);
1303 nv_mthd(dev, 0x9097, 0x144c, 0x00000000);
1304 nv_mthd(dev, 0x9097, 0x0360, 0x20164010);
1305 nv_mthd(dev, 0x9097, 0x0364, 0x00000020);
1306 nv_mthd(dev, 0x9097, 0x0368, 0x00000000);
1307 nv_mthd(dev, 0x9097, 0x0de4, 0x00000000);
1308 nv_mthd(dev, 0x9097, 0x0204, 0x00000006);
1309 nv_mthd(dev, 0x9097, 0x0208, 0x00000000);
1310 nv_mthd(dev, 0x9097, 0x02cc, 0x003fffff);
1311 nv_mthd(dev, 0x9097, 0x02d0, 0x00000c48);
1312 nv_mthd(dev, 0x9097, 0x1220, 0x00000005);
1313 nv_mthd(dev, 0x9097, 0x0fdc, 0x00000000);
1314 nv_mthd(dev, 0x9097, 0x0f98, 0x00300008);
1315 nv_mthd(dev, 0x9097, 0x1284, 0x04000080);
1316 nv_mthd(dev, 0x9097, 0x1450, 0x00300008);
1317 nv_mthd(dev, 0x9097, 0x1454, 0x04000080);
1318 nv_mthd(dev, 0x9097, 0x0214, 0x00000000);
1319 /* in trace, right after 0x90c0, not here */
1320 nv_mthd(dev, 0x9097, 0x3410, 0x80002006);
1321}
1322
1323static void
1324nvc0_grctx_generate_902d(struct drm_device *dev)
1325{
1326 nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
1327 nv_mthd(dev, 0x902d, 0x0204, 0x00000001);
1328 nv_mthd(dev, 0x902d, 0x0208, 0x00000020);
1329 nv_mthd(dev, 0x902d, 0x020c, 0x00000001);
1330 nv_mthd(dev, 0x902d, 0x0210, 0x00000000);
1331 nv_mthd(dev, 0x902d, 0x0214, 0x00000080);
1332 nv_mthd(dev, 0x902d, 0x0218, 0x00000100);
1333 nv_mthd(dev, 0x902d, 0x021c, 0x00000100);
1334 nv_mthd(dev, 0x902d, 0x0220, 0x00000000);
1335 nv_mthd(dev, 0x902d, 0x0224, 0x00000000);
1336 nv_mthd(dev, 0x902d, 0x0230, 0x000000cf);
1337 nv_mthd(dev, 0x902d, 0x0234, 0x00000001);
1338 nv_mthd(dev, 0x902d, 0x0238, 0x00000020);
1339 nv_mthd(dev, 0x902d, 0x023c, 0x00000001);
1340 nv_mthd(dev, 0x902d, 0x0244, 0x00000080);
1341 nv_mthd(dev, 0x902d, 0x0248, 0x00000100);
1342 nv_mthd(dev, 0x902d, 0x024c, 0x00000100);
1343}
1344
1345static void
1346nvc0_grctx_generate_9039(struct drm_device *dev)
1347{
1348 nv_mthd(dev, 0x9039, 0x030c, 0x00000000);
1349 nv_mthd(dev, 0x9039, 0x0310, 0x00000000);
1350 nv_mthd(dev, 0x9039, 0x0314, 0x00000000);
1351 nv_mthd(dev, 0x9039, 0x0320, 0x00000000);
1352 nv_mthd(dev, 0x9039, 0x0238, 0x00000000);
1353 nv_mthd(dev, 0x9039, 0x023c, 0x00000000);
1354 nv_mthd(dev, 0x9039, 0x0318, 0x00000000);
1355 nv_mthd(dev, 0x9039, 0x031c, 0x00000000);
1356}
1357
1358static void
1359nvc0_grctx_generate_90c0(struct drm_device *dev)
1360{
1361 nv_mthd(dev, 0x90c0, 0x270c, 0x00000000);
1362 nv_mthd(dev, 0x90c0, 0x272c, 0x00000000);
1363 nv_mthd(dev, 0x90c0, 0x274c, 0x00000000);
1364 nv_mthd(dev, 0x90c0, 0x276c, 0x00000000);
1365 nv_mthd(dev, 0x90c0, 0x278c, 0x00000000);
1366 nv_mthd(dev, 0x90c0, 0x27ac, 0x00000000);
1367 nv_mthd(dev, 0x90c0, 0x27cc, 0x00000000);
1368 nv_mthd(dev, 0x90c0, 0x27ec, 0x00000000);
1369 nv_mthd(dev, 0x90c0, 0x030c, 0x00000001);
1370 nv_mthd(dev, 0x90c0, 0x1944, 0x00000000);
1371 nv_mthd(dev, 0x90c0, 0x0758, 0x00000100);
1372 nv_mthd(dev, 0x90c0, 0x02c4, 0x00000000);
1373 nv_mthd(dev, 0x90c0, 0x0790, 0x00000000);
1374 nv_mthd(dev, 0x90c0, 0x0794, 0x00000000);
1375 nv_mthd(dev, 0x90c0, 0x0798, 0x00000000);
1376 nv_mthd(dev, 0x90c0, 0x079c, 0x00000000);
1377 nv_mthd(dev, 0x90c0, 0x07a0, 0x00000000);
1378 nv_mthd(dev, 0x90c0, 0x077c, 0x00000000);
1379 nv_mthd(dev, 0x90c0, 0x0204, 0x00000000);
1380 nv_mthd(dev, 0x90c0, 0x0208, 0x00000000);
1381 nv_mthd(dev, 0x90c0, 0x020c, 0x00000000);
1382 nv_mthd(dev, 0x90c0, 0x0214, 0x00000000);
1383 nv_mthd(dev, 0x90c0, 0x024c, 0x00000000);
1384 nv_mthd(dev, 0x90c0, 0x0d94, 0x00000001);
1385 nv_mthd(dev, 0x90c0, 0x1608, 0x00000000);
1386 nv_mthd(dev, 0x90c0, 0x160c, 0x00000000);
1387 nv_mthd(dev, 0x90c0, 0x1664, 0x00000000);
1388}
1389
1390static void
1391nvc0_grctx_generate_dispatch(struct drm_device *dev)
1392{
1393 int i;
1394
1395 nv_wr32(dev, 0x404004, 0x00000000);
1396 nv_wr32(dev, 0x404008, 0x00000000);
1397 nv_wr32(dev, 0x40400c, 0x00000000);
1398 nv_wr32(dev, 0x404010, 0x00000000);
1399 nv_wr32(dev, 0x404014, 0x00000000);
1400 nv_wr32(dev, 0x404018, 0x00000000);
1401 nv_wr32(dev, 0x40401c, 0x00000000);
1402 nv_wr32(dev, 0x404020, 0x00000000);
1403 nv_wr32(dev, 0x404024, 0x00000000);
1404 nv_wr32(dev, 0x404028, 0x00000000);
1405 nv_wr32(dev, 0x40402c, 0x00000000);
1406 nv_wr32(dev, 0x404044, 0x00000000);
1407 nv_wr32(dev, 0x404094, 0x00000000);
1408 nv_wr32(dev, 0x404098, 0x00000000);
1409 nv_wr32(dev, 0x40409c, 0x00000000);
1410 nv_wr32(dev, 0x4040a0, 0x00000000);
1411 nv_wr32(dev, 0x4040a4, 0x00000000);
1412 nv_wr32(dev, 0x4040a8, 0x00000000);
1413 nv_wr32(dev, 0x4040ac, 0x00000000);
1414 nv_wr32(dev, 0x4040b0, 0x00000000);
1415 nv_wr32(dev, 0x4040b4, 0x00000000);
1416 nv_wr32(dev, 0x4040b8, 0x00000000);
1417 nv_wr32(dev, 0x4040bc, 0x00000000);
1418 nv_wr32(dev, 0x4040c0, 0x00000000);
1419 nv_wr32(dev, 0x4040c4, 0x00000000);
1420 nv_wr32(dev, 0x4040c8, 0xf0000087);
1421 nv_wr32(dev, 0x4040d4, 0x00000000);
1422 nv_wr32(dev, 0x4040d8, 0x00000000);
1423 nv_wr32(dev, 0x4040dc, 0x00000000);
1424 nv_wr32(dev, 0x4040e0, 0x00000000);
1425 nv_wr32(dev, 0x4040e4, 0x00000000);
1426 nv_wr32(dev, 0x4040e8, 0x00001000);
1427 nv_wr32(dev, 0x4040f8, 0x00000000);
1428 nv_wr32(dev, 0x404130, 0x00000000);
1429 nv_wr32(dev, 0x404134, 0x00000000);
1430 nv_wr32(dev, 0x404138, 0x20000040);
1431 nv_wr32(dev, 0x404150, 0x0000002e);
1432 nv_wr32(dev, 0x404154, 0x00000400);
1433 nv_wr32(dev, 0x404158, 0x00000200);
1434 nv_wr32(dev, 0x404164, 0x00000055);
1435 nv_wr32(dev, 0x404168, 0x00000000);
1436 nv_wr32(dev, 0x404174, 0x00000000);
1437 nv_wr32(dev, 0x404178, 0x00000000);
1438 nv_wr32(dev, 0x40417c, 0x00000000);
1439 for (i = 0; i < 8; i++)
1440 nv_wr32(dev, 0x404200 + (i * 4), 0x00000000); /* subc */
1441}
1442
1443static void
1444nvc0_grctx_generate_macro(struct drm_device *dev)
1445{
1446 nv_wr32(dev, 0x404404, 0x00000000);
1447 nv_wr32(dev, 0x404408, 0x00000000);
1448 nv_wr32(dev, 0x40440c, 0x00000000);
1449 nv_wr32(dev, 0x404410, 0x00000000);
1450 nv_wr32(dev, 0x404414, 0x00000000);
1451 nv_wr32(dev, 0x404418, 0x00000000);
1452 nv_wr32(dev, 0x40441c, 0x00000000);
1453 nv_wr32(dev, 0x404420, 0x00000000);
1454 nv_wr32(dev, 0x404424, 0x00000000);
1455 nv_wr32(dev, 0x404428, 0x00000000);
1456 nv_wr32(dev, 0x40442c, 0x00000000);
1457 nv_wr32(dev, 0x404430, 0x00000000);
1458 nv_wr32(dev, 0x404434, 0x00000000);
1459 nv_wr32(dev, 0x404438, 0x00000000);
1460 nv_wr32(dev, 0x404460, 0x00000000);
1461 nv_wr32(dev, 0x404464, 0x00000000);
1462 nv_wr32(dev, 0x404468, 0x00ffffff);
1463 nv_wr32(dev, 0x40446c, 0x00000000);
1464 nv_wr32(dev, 0x404480, 0x00000001);
1465 nv_wr32(dev, 0x404498, 0x00000001);
1466}
1467
1468static void
1469nvc0_grctx_generate_m2mf(struct drm_device *dev)
1470{
1471 nv_wr32(dev, 0x404604, 0x00000015);
1472 nv_wr32(dev, 0x404608, 0x00000000);
1473 nv_wr32(dev, 0x40460c, 0x00002e00);
1474 nv_wr32(dev, 0x404610, 0x00000100);
1475 nv_wr32(dev, 0x404618, 0x00000000);
1476 nv_wr32(dev, 0x40461c, 0x00000000);
1477 nv_wr32(dev, 0x404620, 0x00000000);
1478 nv_wr32(dev, 0x404624, 0x00000000);
1479 nv_wr32(dev, 0x404628, 0x00000000);
1480 nv_wr32(dev, 0x40462c, 0x00000000);
1481 nv_wr32(dev, 0x404630, 0x00000000);
1482 nv_wr32(dev, 0x404634, 0x00000000);
1483 nv_wr32(dev, 0x404638, 0x00000004);
1484 nv_wr32(dev, 0x40463c, 0x00000000);
1485 nv_wr32(dev, 0x404640, 0x00000000);
1486 nv_wr32(dev, 0x404644, 0x00000000);
1487 nv_wr32(dev, 0x404648, 0x00000000);
1488 nv_wr32(dev, 0x40464c, 0x00000000);
1489 nv_wr32(dev, 0x404650, 0x00000000);
1490 nv_wr32(dev, 0x404654, 0x00000000);
1491 nv_wr32(dev, 0x404658, 0x00000000);
1492 nv_wr32(dev, 0x40465c, 0x007f0100);
1493 nv_wr32(dev, 0x404660, 0x00000000);
1494 nv_wr32(dev, 0x404664, 0x00000000);
1495 nv_wr32(dev, 0x404668, 0x00000000);
1496 nv_wr32(dev, 0x40466c, 0x00000000);
1497 nv_wr32(dev, 0x404670, 0x00000000);
1498 nv_wr32(dev, 0x404674, 0x00000000);
1499 nv_wr32(dev, 0x404678, 0x00000000);
1500 nv_wr32(dev, 0x40467c, 0x00000002);
1501 nv_wr32(dev, 0x404680, 0x00000000);
1502 nv_wr32(dev, 0x404684, 0x00000000);
1503 nv_wr32(dev, 0x404688, 0x00000000);
1504 nv_wr32(dev, 0x40468c, 0x00000000);
1505 nv_wr32(dev, 0x404690, 0x00000000);
1506 nv_wr32(dev, 0x404694, 0x00000000);
1507 nv_wr32(dev, 0x404698, 0x00000000);
1508 nv_wr32(dev, 0x40469c, 0x00000000);
1509 nv_wr32(dev, 0x4046a0, 0x007f0080);
1510 nv_wr32(dev, 0x4046a4, 0x00000000);
1511 nv_wr32(dev, 0x4046a8, 0x00000000);
1512 nv_wr32(dev, 0x4046ac, 0x00000000);
1513 nv_wr32(dev, 0x4046b0, 0x00000000);
1514 nv_wr32(dev, 0x4046b4, 0x00000000);
1515 nv_wr32(dev, 0x4046b8, 0x00000000);
1516 nv_wr32(dev, 0x4046bc, 0x00000000);
1517 nv_wr32(dev, 0x4046c0, 0x00000000);
1518 nv_wr32(dev, 0x4046c4, 0x00000000);
1519 nv_wr32(dev, 0x4046c8, 0x00000000);
1520 nv_wr32(dev, 0x4046cc, 0x00000000);
1521 nv_wr32(dev, 0x4046d0, 0x00000000);
1522 nv_wr32(dev, 0x4046d4, 0x00000000);
1523 nv_wr32(dev, 0x4046d8, 0x00000000);
1524 nv_wr32(dev, 0x4046dc, 0x00000000);
1525 nv_wr32(dev, 0x4046e0, 0x00000000);
1526 nv_wr32(dev, 0x4046e4, 0x00000000);
1527 nv_wr32(dev, 0x4046e8, 0x00000000);
1528 nv_wr32(dev, 0x4046f0, 0x00000000);
1529 nv_wr32(dev, 0x4046f4, 0x00000000);
1530}
1531
1532static void
1533nvc0_grctx_generate_unk47xx(struct drm_device *dev)
1534{
1535 nv_wr32(dev, 0x404700, 0x00000000);
1536 nv_wr32(dev, 0x404704, 0x00000000);
1537 nv_wr32(dev, 0x404708, 0x00000000);
1538 nv_wr32(dev, 0x40470c, 0x00000000);
1539 nv_wr32(dev, 0x404710, 0x00000000);
1540 nv_wr32(dev, 0x404714, 0x00000000);
1541 nv_wr32(dev, 0x404718, 0x00000000);
1542 nv_wr32(dev, 0x40471c, 0x00000000);
1543 nv_wr32(dev, 0x404720, 0x00000000);
1544 nv_wr32(dev, 0x404724, 0x00000000);
1545 nv_wr32(dev, 0x404728, 0x00000000);
1546 nv_wr32(dev, 0x40472c, 0x00000000);
1547 nv_wr32(dev, 0x404730, 0x00000000);
1548 nv_wr32(dev, 0x404734, 0x00000100);
1549 nv_wr32(dev, 0x404738, 0x00000000);
1550 nv_wr32(dev, 0x40473c, 0x00000000);
1551 nv_wr32(dev, 0x404740, 0x00000000);
1552 nv_wr32(dev, 0x404744, 0x00000000);
1553 nv_wr32(dev, 0x404748, 0x00000000);
1554 nv_wr32(dev, 0x40474c, 0x00000000);
1555 nv_wr32(dev, 0x404750, 0x00000000);
1556 nv_wr32(dev, 0x404754, 0x00000000);
1557}
1558
1559static void
1560nvc0_grctx_generate_shaders(struct drm_device *dev)
1561{
1562 nv_wr32(dev, 0x405800, 0x078000bf);
1563 nv_wr32(dev, 0x405830, 0x02180000);
1564 nv_wr32(dev, 0x405834, 0x00000000);
1565 nv_wr32(dev, 0x405838, 0x00000000);
1566 nv_wr32(dev, 0x405854, 0x00000000);
1567 nv_wr32(dev, 0x405870, 0x00000001);
1568 nv_wr32(dev, 0x405874, 0x00000001);
1569 nv_wr32(dev, 0x405878, 0x00000001);
1570 nv_wr32(dev, 0x40587c, 0x00000001);
1571 nv_wr32(dev, 0x405a00, 0x00000000);
1572 nv_wr32(dev, 0x405a04, 0x00000000);
1573 nv_wr32(dev, 0x405a18, 0x00000000);
1574}
1575
1576static void
1577nvc0_grctx_generate_unk60xx(struct drm_device *dev)
1578{
1579 nv_wr32(dev, 0x406020, 0x000103c1);
1580 nv_wr32(dev, 0x406028, 0x00000001);
1581 nv_wr32(dev, 0x40602c, 0x00000001);
1582 nv_wr32(dev, 0x406030, 0x00000001);
1583 nv_wr32(dev, 0x406034, 0x00000001);
1584}
1585
1586static void
1587nvc0_grctx_generate_unk64xx(struct drm_device *dev)
1588{
1589 nv_wr32(dev, 0x4064a8, 0x00000000);
1590 nv_wr32(dev, 0x4064ac, 0x00003fff);
1591 nv_wr32(dev, 0x4064b4, 0x00000000);
1592 nv_wr32(dev, 0x4064b8, 0x00000000);
1593}
1594
1595static void
1596nvc0_grctx_generate_tpbus(struct drm_device *dev)
1597{
1598 nv_wr32(dev, 0x407804, 0x00000023);
1599 nv_wr32(dev, 0x40780c, 0x0a418820);
1600 nv_wr32(dev, 0x407810, 0x062080e6);
1601 nv_wr32(dev, 0x407814, 0x020398a4);
1602 nv_wr32(dev, 0x407818, 0x0e629062);
1603 nv_wr32(dev, 0x40781c, 0x0a418820);
1604 nv_wr32(dev, 0x407820, 0x000000e6);
1605 nv_wr32(dev, 0x4078bc, 0x00000103);
1606}
1607
1608static void
1609nvc0_grctx_generate_ccache(struct drm_device *dev)
1610{
1611 nv_wr32(dev, 0x408000, 0x00000000);
1612 nv_wr32(dev, 0x408004, 0x00000000);
1613 nv_wr32(dev, 0x408008, 0x00000018);
1614 nv_wr32(dev, 0x40800c, 0x00000000);
1615 nv_wr32(dev, 0x408010, 0x00000000);
1616 nv_wr32(dev, 0x408014, 0x00000069);
1617 nv_wr32(dev, 0x408018, 0xe100e100);
1618 nv_wr32(dev, 0x408064, 0x00000000);
1619}
1620
1621static void
1622nvc0_grctx_generate_rop(struct drm_device *dev)
1623{
1624 struct drm_nouveau_private *dev_priv = dev->dev_private;
1625
1626 // ROPC_BROADCAST
1627 nv_wr32(dev, 0x408800, 0x02802a3c);
1628 nv_wr32(dev, 0x408804, 0x00000040);
1629 nv_wr32(dev, 0x408808, 0x0003e00d);
1630 switch (dev_priv->chipset) {
1631 case 0xc0:
1632 nv_wr32(dev, 0x408900, 0x0080b801);
1633 break;
1634 case 0xc3:
1635 case 0xc4:
1636 nv_wr32(dev, 0x408900, 0x3080b801);
1637 break;
1638 }
1639 nv_wr32(dev, 0x408904, 0x02000001);
1640 nv_wr32(dev, 0x408908, 0x00c80929);
1641 nv_wr32(dev, 0x40890c, 0x00000000);
1642 nv_wr32(dev, 0x408980, 0x0000011d);
1643}
1644
1645static void
1646nvc0_grctx_generate_gpc(struct drm_device *dev)
1647{
1648 int i;
1649
1650 // GPC_BROADCAST
1651 nv_wr32(dev, 0x418380, 0x00000016);
1652 nv_wr32(dev, 0x418400, 0x38004e00);
1653 nv_wr32(dev, 0x418404, 0x71e0ffff);
1654 nv_wr32(dev, 0x418408, 0x00000000);
1655 nv_wr32(dev, 0x41840c, 0x00001008);
1656 nv_wr32(dev, 0x418410, 0x0fff0fff);
1657 nv_wr32(dev, 0x418414, 0x00200fff);
1658 nv_wr32(dev, 0x418450, 0x00000000);
1659 nv_wr32(dev, 0x418454, 0x00000000);
1660 nv_wr32(dev, 0x418458, 0x00000000);
1661 nv_wr32(dev, 0x41845c, 0x00000000);
1662 nv_wr32(dev, 0x418460, 0x00000000);
1663 nv_wr32(dev, 0x418464, 0x00000000);
1664 nv_wr32(dev, 0x418468, 0x00000001);
1665 nv_wr32(dev, 0x41846c, 0x00000000);
1666 nv_wr32(dev, 0x418470, 0x00000000);
1667 nv_wr32(dev, 0x418600, 0x0000001f);
1668 nv_wr32(dev, 0x418684, 0x0000000f);
1669 nv_wr32(dev, 0x418700, 0x00000002);
1670 nv_wr32(dev, 0x418704, 0x00000080);
1671 nv_wr32(dev, 0x418708, 0x00000000);
1672 nv_wr32(dev, 0x41870c, 0x07c80000);
1673 nv_wr32(dev, 0x418710, 0x00000000);
1674 nv_wr32(dev, 0x418800, 0x0006860a);
1675 nv_wr32(dev, 0x418808, 0x00000000);
1676 nv_wr32(dev, 0x41880c, 0x00000000);
1677 nv_wr32(dev, 0x418810, 0x00000000);
1678 nv_wr32(dev, 0x418828, 0x00008442);
1679 nv_wr32(dev, 0x418830, 0x00000001);
1680 nv_wr32(dev, 0x4188d8, 0x00000008);
1681 nv_wr32(dev, 0x4188e0, 0x01000000);
1682 nv_wr32(dev, 0x4188e8, 0x00000000);
1683 nv_wr32(dev, 0x4188ec, 0x00000000);
1684 nv_wr32(dev, 0x4188f0, 0x00000000);
1685 nv_wr32(dev, 0x4188f4, 0x00000000);
1686 nv_wr32(dev, 0x4188f8, 0x00000000);
1687 nv_wr32(dev, 0x4188fc, 0x00100000);
1688 nv_wr32(dev, 0x41891c, 0x00ff00ff);
1689 nv_wr32(dev, 0x418924, 0x00000000);
1690 nv_wr32(dev, 0x418928, 0x00ffff00);
1691 nv_wr32(dev, 0x41892c, 0x0000ff00);
1692 for (i = 0; i < 8; i++) {
1693 nv_wr32(dev, 0x418a00 + (i * 0x20), 0x00000000);
1694 nv_wr32(dev, 0x418a04 + (i * 0x20), 0x00000000);
1695 nv_wr32(dev, 0x418a08 + (i * 0x20), 0x00000000);
1696 nv_wr32(dev, 0x418a0c + (i * 0x20), 0x00010000);
1697 nv_wr32(dev, 0x418a10 + (i * 0x20), 0x00000000);
1698 nv_wr32(dev, 0x418a14 + (i * 0x20), 0x00000000);
1699 nv_wr32(dev, 0x418a18 + (i * 0x20), 0x00000000);
1700 }
1701 nv_wr32(dev, 0x418b00, 0x00000000);
1702 nv_wr32(dev, 0x418b08, 0x0a418820);
1703 nv_wr32(dev, 0x418b0c, 0x062080e6);
1704 nv_wr32(dev, 0x418b10, 0x020398a4);
1705 nv_wr32(dev, 0x418b14, 0x0e629062);
1706 nv_wr32(dev, 0x418b18, 0x0a418820);
1707 nv_wr32(dev, 0x418b1c, 0x000000e6);
1708 nv_wr32(dev, 0x418bb8, 0x00000103);
1709 nv_wr32(dev, 0x418c08, 0x00000001);
1710 nv_wr32(dev, 0x418c10, 0x00000000);
1711 nv_wr32(dev, 0x418c14, 0x00000000);
1712 nv_wr32(dev, 0x418c18, 0x00000000);
1713 nv_wr32(dev, 0x418c1c, 0x00000000);
1714 nv_wr32(dev, 0x418c20, 0x00000000);
1715 nv_wr32(dev, 0x418c24, 0x00000000);
1716 nv_wr32(dev, 0x418c28, 0x00000000);
1717 nv_wr32(dev, 0x418c2c, 0x00000000);
1718 nv_wr32(dev, 0x418c80, 0x20200004);
1719 nv_wr32(dev, 0x418c8c, 0x00000001);
1720 nv_wr32(dev, 0x419000, 0x00000780);
1721 nv_wr32(dev, 0x419004, 0x00000000);
1722 nv_wr32(dev, 0x419008, 0x00000000);
1723 nv_wr32(dev, 0x419014, 0x00000004);
1724}
1725
1726static void
1727nvc0_grctx_generate_tp(struct drm_device *dev)
1728{
1729 struct drm_nouveau_private *dev_priv = dev->dev_private;
1730
1731 // GPC_BROADCAST.TP_BROADCAST
1732 nv_wr32(dev, 0x419848, 0x00000000);
1733 nv_wr32(dev, 0x419864, 0x0000012a);
1734 nv_wr32(dev, 0x419888, 0x00000000);
1735 nv_wr32(dev, 0x419a00, 0x000001f0);
1736 nv_wr32(dev, 0x419a04, 0x00000001);
1737 nv_wr32(dev, 0x419a08, 0x00000023);
1738 nv_wr32(dev, 0x419a0c, 0x00020000);
1739 nv_wr32(dev, 0x419a10, 0x00000000);
1740 nv_wr32(dev, 0x419a14, 0x00000200);
1741 nv_wr32(dev, 0x419a1c, 0x00000000);
1742 nv_wr32(dev, 0x419a20, 0x00000800);
1743 if (dev_priv->chipset != 0xc0)
1744 nv_wr32(dev, 0x00419ac4, 0x0007f440); // 0xc3
1745 nv_wr32(dev, 0x419b00, 0x0a418820);
1746 nv_wr32(dev, 0x419b04, 0x062080e6);
1747 nv_wr32(dev, 0x419b08, 0x020398a4);
1748 nv_wr32(dev, 0x419b0c, 0x0e629062);
1749 nv_wr32(dev, 0x419b10, 0x0a418820);
1750 nv_wr32(dev, 0x419b14, 0x000000e6);
1751 nv_wr32(dev, 0x419bd0, 0x00900103);
1752 nv_wr32(dev, 0x419be0, 0x00000001);
1753 nv_wr32(dev, 0x419be4, 0x00000000);
1754 nv_wr32(dev, 0x419c00, 0x00000002);
1755 nv_wr32(dev, 0x419c04, 0x00000006);
1756 nv_wr32(dev, 0x419c08, 0x00000002);
1757 nv_wr32(dev, 0x419c20, 0x00000000);
1758 nv_wr32(dev, 0x419cbc, 0x28137606);
1759 nv_wr32(dev, 0x419ce8, 0x00000000);
1760 nv_wr32(dev, 0x419cf4, 0x00000183);
1761 nv_wr32(dev, 0x419d20, 0x02180000);
1762 nv_wr32(dev, 0x419d24, 0x00001fff);
1763 nv_wr32(dev, 0x419e04, 0x00000000);
1764 nv_wr32(dev, 0x419e08, 0x00000000);
1765 nv_wr32(dev, 0x419e0c, 0x00000000);
1766 nv_wr32(dev, 0x419e10, 0x00000002);
1767 nv_wr32(dev, 0x419e44, 0x001beff2);
1768 nv_wr32(dev, 0x419e48, 0x00000000);
1769 nv_wr32(dev, 0x419e4c, 0x0000000f);
1770 nv_wr32(dev, 0x419e50, 0x00000000);
1771 nv_wr32(dev, 0x419e54, 0x00000000);
1772 nv_wr32(dev, 0x419e58, 0x00000000);
1773 nv_wr32(dev, 0x419e5c, 0x00000000);
1774 nv_wr32(dev, 0x419e60, 0x00000000);
1775 nv_wr32(dev, 0x419e64, 0x00000000);
1776 nv_wr32(dev, 0x419e68, 0x00000000);
1777 nv_wr32(dev, 0x419e6c, 0x00000000);
1778 nv_wr32(dev, 0x419e70, 0x00000000);
1779 nv_wr32(dev, 0x419e74, 0x00000000);
1780 nv_wr32(dev, 0x419e78, 0x00000000);
1781 nv_wr32(dev, 0x419e7c, 0x00000000);
1782 nv_wr32(dev, 0x419e80, 0x00000000);
1783 nv_wr32(dev, 0x419e84, 0x00000000);
1784 nv_wr32(dev, 0x419e88, 0x00000000);
1785 nv_wr32(dev, 0x419e8c, 0x00000000);
1786 nv_wr32(dev, 0x419e90, 0x00000000);
1787 nv_wr32(dev, 0x419e98, 0x00000000);
1788 if (dev_priv->chipset != 0xc0)
1789 nv_wr32(dev, 0x419ee0, 0x00011110);
1790 nv_wr32(dev, 0x419f50, 0x00000000);
1791 nv_wr32(dev, 0x419f54, 0x00000000);
1792 if (dev_priv->chipset != 0xc0)
1793 nv_wr32(dev, 0x419f58, 0x00000000);
1794}
1795
1796int
1797nvc0_grctx_generate(struct nouveau_channel *chan)
1798{
1799 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
1800 struct nvc0_graph_priv *priv = dev_priv->engine.graph.priv;
1801 struct nvc0_graph_chan *grch = chan->pgraph_ctx;
1802 struct drm_device *dev = chan->dev;
1803 int i, gpc, tp, id;
1804 u32 r000260, tmp;
1805
1806 r000260 = nv_rd32(dev, 0x000260);
1807 nv_wr32(dev, 0x000260, r000260 & ~1);
1808 nv_wr32(dev, 0x400208, 0x00000000);
1809
1810 nvc0_grctx_generate_dispatch(dev);
1811 nvc0_grctx_generate_macro(dev);
1812 nvc0_grctx_generate_m2mf(dev);
1813 nvc0_grctx_generate_unk47xx(dev);
1814 nvc0_grctx_generate_shaders(dev);
1815 nvc0_grctx_generate_unk60xx(dev);
1816 nvc0_grctx_generate_unk64xx(dev);
1817 nvc0_grctx_generate_tpbus(dev);
1818 nvc0_grctx_generate_ccache(dev);
1819 nvc0_grctx_generate_rop(dev);
1820 nvc0_grctx_generate_gpc(dev);
1821 nvc0_grctx_generate_tp(dev);
1822
1823 nv_wr32(dev, 0x404154, 0x00000000);
1824
1825 /* fuc "mmio list" writes */
1826 for (i = 0; i < grch->mmio_nr * 8; i += 8) {
1827 u32 reg = nv_ro32(grch->mmio, i + 0);
1828 nv_wr32(dev, reg, nv_ro32(grch->mmio, i + 4));
1829 }
1830
1831 for (tp = 0, id = 0; tp < 4; tp++) {
1832 for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
1833 if (tp <= priv->tp_nr[gpc]) {
1834 nv_wr32(dev, TP_UNIT(gpc, tp, 0x698), id);
1835 nv_wr32(dev, TP_UNIT(gpc, tp, 0x4e8), id);
1836 nv_wr32(dev, GPC_UNIT(gpc, 0x0c10 + tp * 4), id);
1837 nv_wr32(dev, TP_UNIT(gpc, tp, 0x088), id);
1838 id++;
1839 }
1840
1841 nv_wr32(dev, GPC_UNIT(gpc, 0x0c08), priv->tp_nr[gpc]);
1842 nv_wr32(dev, GPC_UNIT(gpc, 0x0c8c), priv->tp_nr[gpc]);
1843 }
1844 }
1845
1846 tmp = 0;
1847 for (i = 0; i < priv->gpc_nr; i++)
1848 tmp |= priv->tp_nr[i] << (i * 4);
1849 nv_wr32(dev, 0x406028, tmp);
1850 nv_wr32(dev, 0x405870, tmp);
1851
1852 nv_wr32(dev, 0x40602c, 0x00000000);
1853 nv_wr32(dev, 0x405874, 0x00000000);
1854 nv_wr32(dev, 0x406030, 0x00000000);
1855 nv_wr32(dev, 0x405878, 0x00000000);
1856 nv_wr32(dev, 0x406034, 0x00000000);
1857 nv_wr32(dev, 0x40587c, 0x00000000);
1858
1859 if (1) {
1860 const u8 chipset_tp_max[] = { 16, 0, 0, 4, 8 };
1861 u8 max = chipset_tp_max[dev_priv->chipset & 0x0f];
1862 u8 tpnr[GPC_MAX];
1863 u8 data[32];
1864
1865 memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
1866 memset(data, 0x1f, sizeof(data));
1867
1868 gpc = -1;
1869 for (tp = 0; tp < priv->tp_total; tp++) {
1870 do {
1871 gpc = (gpc + 1) % priv->gpc_nr;
1872 } while (!tpnr[gpc]);
1873 tpnr[gpc]--;
1874 data[tp] = gpc;
1875 }
1876
1877 for (i = 0; i < max / 4; i++)
1878 nv_wr32(dev, 0x4060a8 + (i * 4), ((u32 *)data)[i]);
1879 }
1880
1881 if (1) {
1882 u32 data[6] = {}, data2[2] = {};
1883 u8 tpnr[GPC_MAX];
1884 u8 shift, ntpcv;
1885
1886 /* calculate first set of magics */
1887 memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
1888
1889 for (tp = 0; tp < priv->tp_total; tp++) {
1890 do {
1891 gpc = (gpc + 1) % priv->gpc_nr;
1892 } while (!tpnr[gpc]);
1893 tpnr[gpc]--;
1894
1895 data[tp / 6] |= gpc << ((tp % 6) * 5);
1896 }
1897
1898 for (; tp < 32; tp++)
1899 data[tp / 6] |= 7 << ((tp % 6) * 5);
1900
1901 /* and the second... */
1902 shift = 0;
1903 ntpcv = priv->tp_total;
1904 while (!(ntpcv & (1 << 4))) {
1905 ntpcv <<= 1;
1906 shift++;
1907 }
1908
1909 data2[0] = (ntpcv << 16);
1910 data2[0] |= (shift << 21);
1911 data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
1912 for (i = 1; i < 7; i++)
1913 data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
1914
1915 // GPC_BROADCAST
1916 nv_wr32(dev, 0x418bb8, (priv->tp_total << 8) |
1917 priv->magic_not_rop_nr);
1918 for (i = 0; i < 6; i++)
1919 nv_wr32(dev, 0x418b08 + (i * 4), data[i]);
1920
1921 // GPC_BROADCAST.TP_BROADCAST
1922 nv_wr32(dev, 0x419bd0, (priv->tp_total << 8) |
1923 priv->magic_not_rop_nr |
1924 data2[0]);
1925 nv_wr32(dev, 0x419be4, data2[1]);
1926 for (i = 0; i < 6; i++)
1927 nv_wr32(dev, 0x419b00 + (i * 4), data[i]);
1928
1929 // UNK78xx
1930 nv_wr32(dev, 0x4078bc, (priv->tp_total << 8) |
1931 priv->magic_not_rop_nr);
1932 for (i = 0; i < 6; i++)
1933 nv_wr32(dev, 0x40780c + (i * 4), data[i]);
1934 }
1935
1936 if (1) {
1937 u32 tp_mask = 0, tp_set = 0;
1938 u8 tpnr[GPC_MAX];
1939
1940 memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
1941 for (gpc = 0; gpc < priv->gpc_nr; gpc++)
1942 tp_mask |= ((1 << priv->tp_nr[gpc]) - 1) << (gpc * 8);
1943
1944 gpc = -1;
1945 for (i = 0, gpc = -1; i < 32; i++) {
1946 int ltp = i * (priv->tp_total - 1) / 32;
1947
1948 do {
1949 gpc = (gpc + 1) % priv->gpc_nr;
1950 } while (!tpnr[gpc]);
1951 tp = priv->tp_nr[gpc] - tpnr[gpc]--;
1952
1953 tp_set |= 1 << ((gpc * 8) + tp);
1954
1955 do {
1956 nv_wr32(dev, 0x406800 + (i * 0x20), tp_set);
1957 tp_set ^= tp_mask;
1958 nv_wr32(dev, 0x406c00 + (i * 0x20), tp_set);
1959 tp_set ^= tp_mask;
1960 } while (ltp == (++i * (priv->tp_total - 1) / 32));
1961 i--;
1962 }
1963 }
1964
1965 nv_wr32(dev, 0x400208, 0x80000000);
1966
1967 nv_icmd(dev, 0x00001000, 0x00000004);
1968 nv_icmd(dev, 0x000000a9, 0x0000ffff);
1969 nv_icmd(dev, 0x00000038, 0x0fac6881);
1970 nv_icmd(dev, 0x0000003d, 0x00000001);
1971 nv_icmd(dev, 0x000000e8, 0x00000400);
1972 nv_icmd(dev, 0x000000e9, 0x00000400);
1973 nv_icmd(dev, 0x000000ea, 0x00000400);
1974 nv_icmd(dev, 0x000000eb, 0x00000400);
1975 nv_icmd(dev, 0x000000ec, 0x00000400);
1976 nv_icmd(dev, 0x000000ed, 0x00000400);
1977 nv_icmd(dev, 0x000000ee, 0x00000400);
1978 nv_icmd(dev, 0x000000ef, 0x00000400);
1979 nv_icmd(dev, 0x00000078, 0x00000300);
1980 nv_icmd(dev, 0x00000079, 0x00000300);
1981 nv_icmd(dev, 0x0000007a, 0x00000300);
1982 nv_icmd(dev, 0x0000007b, 0x00000300);
1983 nv_icmd(dev, 0x0000007c, 0x00000300);
1984 nv_icmd(dev, 0x0000007d, 0x00000300);
1985 nv_icmd(dev, 0x0000007e, 0x00000300);
1986 nv_icmd(dev, 0x0000007f, 0x00000300);
1987 nv_icmd(dev, 0x00000050, 0x00000011);
1988 nv_icmd(dev, 0x00000058, 0x00000008);
1989 nv_icmd(dev, 0x00000059, 0x00000008);
1990 nv_icmd(dev, 0x0000005a, 0x00000008);
1991 nv_icmd(dev, 0x0000005b, 0x00000008);
1992 nv_icmd(dev, 0x0000005c, 0x00000008);
1993 nv_icmd(dev, 0x0000005d, 0x00000008);
1994 nv_icmd(dev, 0x0000005e, 0x00000008);
1995 nv_icmd(dev, 0x0000005f, 0x00000008);
1996 nv_icmd(dev, 0x00000208, 0x00000001);
1997 nv_icmd(dev, 0x00000209, 0x00000001);
1998 nv_icmd(dev, 0x0000020a, 0x00000001);
1999 nv_icmd(dev, 0x0000020b, 0x00000001);
2000 nv_icmd(dev, 0x0000020c, 0x00000001);
2001 nv_icmd(dev, 0x0000020d, 0x00000001);
2002 nv_icmd(dev, 0x0000020e, 0x00000001);
2003 nv_icmd(dev, 0x0000020f, 0x00000001);
2004 nv_icmd(dev, 0x00000081, 0x00000001);
2005 nv_icmd(dev, 0x00000085, 0x00000004);
2006 nv_icmd(dev, 0x00000088, 0x00000400);
2007 nv_icmd(dev, 0x00000090, 0x00000300);
2008 nv_icmd(dev, 0x00000098, 0x00001001);
2009 nv_icmd(dev, 0x000000e3, 0x00000001);
2010 nv_icmd(dev, 0x000000da, 0x00000001);
2011 nv_icmd(dev, 0x000000f8, 0x00000003);
2012 nv_icmd(dev, 0x000000fa, 0x00000001);
2013 nv_icmd(dev, 0x0000009f, 0x0000ffff);
2014 nv_icmd(dev, 0x000000a0, 0x0000ffff);
2015 nv_icmd(dev, 0x000000a1, 0x0000ffff);
2016 nv_icmd(dev, 0x000000a2, 0x0000ffff);
2017 nv_icmd(dev, 0x000000b1, 0x00000001);
2018 nv_icmd(dev, 0x000000b2, 0x00000000);
2019 nv_icmd(dev, 0x000000b3, 0x00000000);
2020 nv_icmd(dev, 0x000000b4, 0x00000000);
2021 nv_icmd(dev, 0x000000b5, 0x00000000);
2022 nv_icmd(dev, 0x000000b6, 0x00000000);
2023 nv_icmd(dev, 0x000000b7, 0x00000000);
2024 nv_icmd(dev, 0x000000b8, 0x00000000);
2025 nv_icmd(dev, 0x000000b9, 0x00000000);
2026 nv_icmd(dev, 0x000000ba, 0x00000000);
2027 nv_icmd(dev, 0x000000bb, 0x00000000);
2028 nv_icmd(dev, 0x000000bc, 0x00000000);
2029 nv_icmd(dev, 0x000000bd, 0x00000000);
2030 nv_icmd(dev, 0x000000be, 0x00000000);
2031 nv_icmd(dev, 0x000000bf, 0x00000000);
2032 nv_icmd(dev, 0x000000c0, 0x00000000);
2033 nv_icmd(dev, 0x000000c1, 0x00000000);
2034 nv_icmd(dev, 0x000000c2, 0x00000000);
2035 nv_icmd(dev, 0x000000c3, 0x00000000);
2036 nv_icmd(dev, 0x000000c4, 0x00000000);
2037 nv_icmd(dev, 0x000000c5, 0x00000000);
2038 nv_icmd(dev, 0x000000c6, 0x00000000);
2039 nv_icmd(dev, 0x000000c7, 0x00000000);
2040 nv_icmd(dev, 0x000000c8, 0x00000000);
2041 nv_icmd(dev, 0x000000c9, 0x00000000);
2042 nv_icmd(dev, 0x000000ca, 0x00000000);
2043 nv_icmd(dev, 0x000000cb, 0x00000000);
2044 nv_icmd(dev, 0x000000cc, 0x00000000);
2045 nv_icmd(dev, 0x000000cd, 0x00000000);
2046 nv_icmd(dev, 0x000000ce, 0x00000000);
2047 nv_icmd(dev, 0x000000cf, 0x00000000);
2048 nv_icmd(dev, 0x000000d0, 0x00000000);
2049 nv_icmd(dev, 0x000000d1, 0x00000000);
2050 nv_icmd(dev, 0x000000d2, 0x00000000);
2051 nv_icmd(dev, 0x000000d3, 0x00000000);
2052 nv_icmd(dev, 0x000000d4, 0x00000000);
2053 nv_icmd(dev, 0x000000d5, 0x00000000);
2054 nv_icmd(dev, 0x000000d6, 0x00000000);
2055 nv_icmd(dev, 0x000000d7, 0x00000000);
2056 nv_icmd(dev, 0x000000d8, 0x00000000);
2057 nv_icmd(dev, 0x000000d9, 0x00000000);
2058 nv_icmd(dev, 0x00000210, 0x00000040);
2059 nv_icmd(dev, 0x00000211, 0x00000040);
2060 nv_icmd(dev, 0x00000212, 0x00000040);
2061 nv_icmd(dev, 0x00000213, 0x00000040);
2062 nv_icmd(dev, 0x00000214, 0x00000040);
2063 nv_icmd(dev, 0x00000215, 0x00000040);
2064 nv_icmd(dev, 0x00000216, 0x00000040);
2065 nv_icmd(dev, 0x00000217, 0x00000040);
2066 nv_icmd(dev, 0x00000218, 0x0000c080);
2067 nv_icmd(dev, 0x00000219, 0x0000c080);
2068 nv_icmd(dev, 0x0000021a, 0x0000c080);
2069 nv_icmd(dev, 0x0000021b, 0x0000c080);
2070 nv_icmd(dev, 0x0000021c, 0x0000c080);
2071 nv_icmd(dev, 0x0000021d, 0x0000c080);
2072 nv_icmd(dev, 0x0000021e, 0x0000c080);
2073 nv_icmd(dev, 0x0000021f, 0x0000c080);
2074 nv_icmd(dev, 0x000000ad, 0x0000013e);
2075 nv_icmd(dev, 0x000000e1, 0x00000010);
2076 nv_icmd(dev, 0x00000290, 0x00000000);
2077 nv_icmd(dev, 0x00000291, 0x00000000);
2078 nv_icmd(dev, 0x00000292, 0x00000000);
2079 nv_icmd(dev, 0x00000293, 0x00000000);
2080 nv_icmd(dev, 0x00000294, 0x00000000);
2081 nv_icmd(dev, 0x00000295, 0x00000000);
2082 nv_icmd(dev, 0x00000296, 0x00000000);
2083 nv_icmd(dev, 0x00000297, 0x00000000);
2084 nv_icmd(dev, 0x00000298, 0x00000000);
2085 nv_icmd(dev, 0x00000299, 0x00000000);
2086 nv_icmd(dev, 0x0000029a, 0x00000000);
2087 nv_icmd(dev, 0x0000029b, 0x00000000);
2088 nv_icmd(dev, 0x0000029c, 0x00000000);
2089 nv_icmd(dev, 0x0000029d, 0x00000000);
2090 nv_icmd(dev, 0x0000029e, 0x00000000);
2091 nv_icmd(dev, 0x0000029f, 0x00000000);
2092 nv_icmd(dev, 0x000003b0, 0x00000000);
2093 nv_icmd(dev, 0x000003b1, 0x00000000);
2094 nv_icmd(dev, 0x000003b2, 0x00000000);
2095 nv_icmd(dev, 0x000003b3, 0x00000000);
2096 nv_icmd(dev, 0x000003b4, 0x00000000);
2097 nv_icmd(dev, 0x000003b5, 0x00000000);
2098 nv_icmd(dev, 0x000003b6, 0x00000000);
2099 nv_icmd(dev, 0x000003b7, 0x00000000);
2100 nv_icmd(dev, 0x000003b8, 0x00000000);
2101 nv_icmd(dev, 0x000003b9, 0x00000000);
2102 nv_icmd(dev, 0x000003ba, 0x00000000);
2103 nv_icmd(dev, 0x000003bb, 0x00000000);
2104 nv_icmd(dev, 0x000003bc, 0x00000000);
2105 nv_icmd(dev, 0x000003bd, 0x00000000);
2106 nv_icmd(dev, 0x000003be, 0x00000000);
2107 nv_icmd(dev, 0x000003bf, 0x00000000);
2108 nv_icmd(dev, 0x000002a0, 0x00000000);
2109 nv_icmd(dev, 0x000002a1, 0x00000000);
2110 nv_icmd(dev, 0x000002a2, 0x00000000);
2111 nv_icmd(dev, 0x000002a3, 0x00000000);
2112 nv_icmd(dev, 0x000002a4, 0x00000000);
2113 nv_icmd(dev, 0x000002a5, 0x00000000);
2114 nv_icmd(dev, 0x000002a6, 0x00000000);
2115 nv_icmd(dev, 0x000002a7, 0x00000000);
2116 nv_icmd(dev, 0x000002a8, 0x00000000);
2117 nv_icmd(dev, 0x000002a9, 0x00000000);
2118 nv_icmd(dev, 0x000002aa, 0x00000000);
2119 nv_icmd(dev, 0x000002ab, 0x00000000);
2120 nv_icmd(dev, 0x000002ac, 0x00000000);
2121 nv_icmd(dev, 0x000002ad, 0x00000000);
2122 nv_icmd(dev, 0x000002ae, 0x00000000);
2123 nv_icmd(dev, 0x000002af, 0x00000000);
2124 nv_icmd(dev, 0x00000420, 0x00000000);
2125 nv_icmd(dev, 0x00000421, 0x00000000);
2126 nv_icmd(dev, 0x00000422, 0x00000000);
2127 nv_icmd(dev, 0x00000423, 0x00000000);
2128 nv_icmd(dev, 0x00000424, 0x00000000);
2129 nv_icmd(dev, 0x00000425, 0x00000000);
2130 nv_icmd(dev, 0x00000426, 0x00000000);
2131 nv_icmd(dev, 0x00000427, 0x00000000);
2132 nv_icmd(dev, 0x00000428, 0x00000000);
2133 nv_icmd(dev, 0x00000429, 0x00000000);
2134 nv_icmd(dev, 0x0000042a, 0x00000000);
2135 nv_icmd(dev, 0x0000042b, 0x00000000);
2136 nv_icmd(dev, 0x0000042c, 0x00000000);
2137 nv_icmd(dev, 0x0000042d, 0x00000000);
2138 nv_icmd(dev, 0x0000042e, 0x00000000);
2139 nv_icmd(dev, 0x0000042f, 0x00000000);
2140 nv_icmd(dev, 0x000002b0, 0x00000000);
2141 nv_icmd(dev, 0x000002b1, 0x00000000);
2142 nv_icmd(dev, 0x000002b2, 0x00000000);
2143 nv_icmd(dev, 0x000002b3, 0x00000000);
2144 nv_icmd(dev, 0x000002b4, 0x00000000);
2145 nv_icmd(dev, 0x000002b5, 0x00000000);
2146 nv_icmd(dev, 0x000002b6, 0x00000000);
2147 nv_icmd(dev, 0x000002b7, 0x00000000);
2148 nv_icmd(dev, 0x000002b8, 0x00000000);
2149 nv_icmd(dev, 0x000002b9, 0x00000000);
2150 nv_icmd(dev, 0x000002ba, 0x00000000);
2151 nv_icmd(dev, 0x000002bb, 0x00000000);
2152 nv_icmd(dev, 0x000002bc, 0x00000000);
2153 nv_icmd(dev, 0x000002bd, 0x00000000);
2154 nv_icmd(dev, 0x000002be, 0x00000000);
2155 nv_icmd(dev, 0x000002bf, 0x00000000);
2156 nv_icmd(dev, 0x00000430, 0x00000000);
2157 nv_icmd(dev, 0x00000431, 0x00000000);
2158 nv_icmd(dev, 0x00000432, 0x00000000);
2159 nv_icmd(dev, 0x00000433, 0x00000000);
2160 nv_icmd(dev, 0x00000434, 0x00000000);
2161 nv_icmd(dev, 0x00000435, 0x00000000);
2162 nv_icmd(dev, 0x00000436, 0x00000000);
2163 nv_icmd(dev, 0x00000437, 0x00000000);
2164 nv_icmd(dev, 0x00000438, 0x00000000);
2165 nv_icmd(dev, 0x00000439, 0x00000000);
2166 nv_icmd(dev, 0x0000043a, 0x00000000);
2167 nv_icmd(dev, 0x0000043b, 0x00000000);
2168 nv_icmd(dev, 0x0000043c, 0x00000000);
2169 nv_icmd(dev, 0x0000043d, 0x00000000);
2170 nv_icmd(dev, 0x0000043e, 0x00000000);
2171 nv_icmd(dev, 0x0000043f, 0x00000000);
2172 nv_icmd(dev, 0x000002c0, 0x00000000);
2173 nv_icmd(dev, 0x000002c1, 0x00000000);
2174 nv_icmd(dev, 0x000002c2, 0x00000000);
2175 nv_icmd(dev, 0x000002c3, 0x00000000);
2176 nv_icmd(dev, 0x000002c4, 0x00000000);
2177 nv_icmd(dev, 0x000002c5, 0x00000000);
2178 nv_icmd(dev, 0x000002c6, 0x00000000);
2179 nv_icmd(dev, 0x000002c7, 0x00000000);
2180 nv_icmd(dev, 0x000002c8, 0x00000000);
2181 nv_icmd(dev, 0x000002c9, 0x00000000);
2182 nv_icmd(dev, 0x000002ca, 0x00000000);
2183 nv_icmd(dev, 0x000002cb, 0x00000000);
2184 nv_icmd(dev, 0x000002cc, 0x00000000);
2185 nv_icmd(dev, 0x000002cd, 0x00000000);
2186 nv_icmd(dev, 0x000002ce, 0x00000000);
2187 nv_icmd(dev, 0x000002cf, 0x00000000);
2188 nv_icmd(dev, 0x000004d0, 0x00000000);
2189 nv_icmd(dev, 0x000004d1, 0x00000000);
2190 nv_icmd(dev, 0x000004d2, 0x00000000);
2191 nv_icmd(dev, 0x000004d3, 0x00000000);
2192 nv_icmd(dev, 0x000004d4, 0x00000000);
2193 nv_icmd(dev, 0x000004d5, 0x00000000);
2194 nv_icmd(dev, 0x000004d6, 0x00000000);
2195 nv_icmd(dev, 0x000004d7, 0x00000000);
2196 nv_icmd(dev, 0x000004d8, 0x00000000);
2197 nv_icmd(dev, 0x000004d9, 0x00000000);
2198 nv_icmd(dev, 0x000004da, 0x00000000);
2199 nv_icmd(dev, 0x000004db, 0x00000000);
2200 nv_icmd(dev, 0x000004dc, 0x00000000);
2201 nv_icmd(dev, 0x000004dd, 0x00000000);
2202 nv_icmd(dev, 0x000004de, 0x00000000);
2203 nv_icmd(dev, 0x000004df, 0x00000000);
2204 nv_icmd(dev, 0x00000720, 0x00000000);
2205 nv_icmd(dev, 0x00000721, 0x00000000);
2206 nv_icmd(dev, 0x00000722, 0x00000000);
2207 nv_icmd(dev, 0x00000723, 0x00000000);
2208 nv_icmd(dev, 0x00000724, 0x00000000);
2209 nv_icmd(dev, 0x00000725, 0x00000000);
2210 nv_icmd(dev, 0x00000726, 0x00000000);
2211 nv_icmd(dev, 0x00000727, 0x00000000);
2212 nv_icmd(dev, 0x00000728, 0x00000000);
2213 nv_icmd(dev, 0x00000729, 0x00000000);
2214 nv_icmd(dev, 0x0000072a, 0x00000000);
2215 nv_icmd(dev, 0x0000072b, 0x00000000);
2216 nv_icmd(dev, 0x0000072c, 0x00000000);
2217 nv_icmd(dev, 0x0000072d, 0x00000000);
2218 nv_icmd(dev, 0x0000072e, 0x00000000);
2219 nv_icmd(dev, 0x0000072f, 0x00000000);
2220 nv_icmd(dev, 0x000008c0, 0x00000000);
2221 nv_icmd(dev, 0x000008c1, 0x00000000);
2222 nv_icmd(dev, 0x000008c2, 0x00000000);
2223 nv_icmd(dev, 0x000008c3, 0x00000000);
2224 nv_icmd(dev, 0x000008c4, 0x00000000);
2225 nv_icmd(dev, 0x000008c5, 0x00000000);
2226 nv_icmd(dev, 0x000008c6, 0x00000000);
2227 nv_icmd(dev, 0x000008c7, 0x00000000);
2228 nv_icmd(dev, 0x000008c8, 0x00000000);
2229 nv_icmd(dev, 0x000008c9, 0x00000000);
2230 nv_icmd(dev, 0x000008ca, 0x00000000);
2231 nv_icmd(dev, 0x000008cb, 0x00000000);
2232 nv_icmd(dev, 0x000008cc, 0x00000000);
2233 nv_icmd(dev, 0x000008cd, 0x00000000);
2234 nv_icmd(dev, 0x000008ce, 0x00000000);
2235 nv_icmd(dev, 0x000008cf, 0x00000000);
2236 nv_icmd(dev, 0x00000890, 0x00000000);
2237 nv_icmd(dev, 0x00000891, 0x00000000);
2238 nv_icmd(dev, 0x00000892, 0x00000000);
2239 nv_icmd(dev, 0x00000893, 0x00000000);
2240 nv_icmd(dev, 0x00000894, 0x00000000);
2241 nv_icmd(dev, 0x00000895, 0x00000000);
2242 nv_icmd(dev, 0x00000896, 0x00000000);
2243 nv_icmd(dev, 0x00000897, 0x00000000);
2244 nv_icmd(dev, 0x00000898, 0x00000000);
2245 nv_icmd(dev, 0x00000899, 0x00000000);
2246 nv_icmd(dev, 0x0000089a, 0x00000000);
2247 nv_icmd(dev, 0x0000089b, 0x00000000);
2248 nv_icmd(dev, 0x0000089c, 0x00000000);
2249 nv_icmd(dev, 0x0000089d, 0x00000000);
2250 nv_icmd(dev, 0x0000089e, 0x00000000);
2251 nv_icmd(dev, 0x0000089f, 0x00000000);
2252 nv_icmd(dev, 0x000008e0, 0x00000000);
2253 nv_icmd(dev, 0x000008e1, 0x00000000);
2254 nv_icmd(dev, 0x000008e2, 0x00000000);
2255 nv_icmd(dev, 0x000008e3, 0x00000000);
2256 nv_icmd(dev, 0x000008e4, 0x00000000);
2257 nv_icmd(dev, 0x000008e5, 0x00000000);
2258 nv_icmd(dev, 0x000008e6, 0x00000000);
2259 nv_icmd(dev, 0x000008e7, 0x00000000);
2260 nv_icmd(dev, 0x000008e8, 0x00000000);
2261 nv_icmd(dev, 0x000008e9, 0x00000000);
2262 nv_icmd(dev, 0x000008ea, 0x00000000);
2263 nv_icmd(dev, 0x000008eb, 0x00000000);
2264 nv_icmd(dev, 0x000008ec, 0x00000000);
2265 nv_icmd(dev, 0x000008ed, 0x00000000);
2266 nv_icmd(dev, 0x000008ee, 0x00000000);
2267 nv_icmd(dev, 0x000008ef, 0x00000000);
2268 nv_icmd(dev, 0x000008a0, 0x00000000);
2269 nv_icmd(dev, 0x000008a1, 0x00000000);
2270 nv_icmd(dev, 0x000008a2, 0x00000000);
2271 nv_icmd(dev, 0x000008a3, 0x00000000);
2272 nv_icmd(dev, 0x000008a4, 0x00000000);
2273 nv_icmd(dev, 0x000008a5, 0x00000000);
2274 nv_icmd(dev, 0x000008a6, 0x00000000);
2275 nv_icmd(dev, 0x000008a7, 0x00000000);
2276 nv_icmd(dev, 0x000008a8, 0x00000000);
2277 nv_icmd(dev, 0x000008a9, 0x00000000);
2278 nv_icmd(dev, 0x000008aa, 0x00000000);
2279 nv_icmd(dev, 0x000008ab, 0x00000000);
2280 nv_icmd(dev, 0x000008ac, 0x00000000);
2281 nv_icmd(dev, 0x000008ad, 0x00000000);
2282 nv_icmd(dev, 0x000008ae, 0x00000000);
2283 nv_icmd(dev, 0x000008af, 0x00000000);
2284 nv_icmd(dev, 0x000008f0, 0x00000000);
2285 nv_icmd(dev, 0x000008f1, 0x00000000);
2286 nv_icmd(dev, 0x000008f2, 0x00000000);
2287 nv_icmd(dev, 0x000008f3, 0x00000000);
2288 nv_icmd(dev, 0x000008f4, 0x00000000);
2289 nv_icmd(dev, 0x000008f5, 0x00000000);
2290 nv_icmd(dev, 0x000008f6, 0x00000000);
2291 nv_icmd(dev, 0x000008f7, 0x00000000);
2292 nv_icmd(dev, 0x000008f8, 0x00000000);
2293 nv_icmd(dev, 0x000008f9, 0x00000000);
2294 nv_icmd(dev, 0x000008fa, 0x00000000);
2295 nv_icmd(dev, 0x000008fb, 0x00000000);
2296 nv_icmd(dev, 0x000008fc, 0x00000000);
2297 nv_icmd(dev, 0x000008fd, 0x00000000);
2298 nv_icmd(dev, 0x000008fe, 0x00000000);
2299 nv_icmd(dev, 0x000008ff, 0x00000000);
2300 nv_icmd(dev, 0x0000094c, 0x000000ff);
2301 nv_icmd(dev, 0x0000094d, 0xffffffff);
2302 nv_icmd(dev, 0x0000094e, 0x00000002);
2303 nv_icmd(dev, 0x000002ec, 0x00000001);
2304 nv_icmd(dev, 0x00000303, 0x00000001);
2305 nv_icmd(dev, 0x000002e6, 0x00000001);
2306 nv_icmd(dev, 0x00000466, 0x00000052);
2307 nv_icmd(dev, 0x00000301, 0x3f800000);
2308 nv_icmd(dev, 0x00000304, 0x30201000);
2309 nv_icmd(dev, 0x00000305, 0x70605040);
2310 nv_icmd(dev, 0x00000306, 0xb8a89888);
2311 nv_icmd(dev, 0x00000307, 0xf8e8d8c8);
2312 nv_icmd(dev, 0x0000030a, 0x00ffff00);
2313 nv_icmd(dev, 0x0000030b, 0x0000001a);
2314 nv_icmd(dev, 0x0000030c, 0x00000001);
2315 nv_icmd(dev, 0x00000318, 0x00000001);
2316 nv_icmd(dev, 0x00000340, 0x00000000);
2317 nv_icmd(dev, 0x00000375, 0x00000001);
2318 nv_icmd(dev, 0x00000351, 0x00000100);
2319 nv_icmd(dev, 0x0000037d, 0x00000006);
2320 nv_icmd(dev, 0x000003a0, 0x00000002);
2321 nv_icmd(dev, 0x000003aa, 0x00000001);
2322 nv_icmd(dev, 0x000003a9, 0x00000001);
2323 nv_icmd(dev, 0x00000380, 0x00000001);
2324 nv_icmd(dev, 0x00000360, 0x00000040);
2325 nv_icmd(dev, 0x00000366, 0x00000000);
2326 nv_icmd(dev, 0x00000367, 0x00000000);
2327 nv_icmd(dev, 0x00000368, 0x00001fff);
2328 nv_icmd(dev, 0x00000370, 0x00000000);
2329 nv_icmd(dev, 0x00000371, 0x00000000);
2330 nv_icmd(dev, 0x00000372, 0x003fffff);
2331 nv_icmd(dev, 0x0000037a, 0x00000012);
2332 nv_icmd(dev, 0x000005e0, 0x00000022);
2333 nv_icmd(dev, 0x000005e1, 0x00000022);
2334 nv_icmd(dev, 0x000005e2, 0x00000022);
2335 nv_icmd(dev, 0x000005e3, 0x00000022);
2336 nv_icmd(dev, 0x000005e4, 0x00000022);
2337 nv_icmd(dev, 0x00000619, 0x00000003);
2338 nv_icmd(dev, 0x00000811, 0x00000003);
2339 nv_icmd(dev, 0x00000812, 0x00000004);
2340 nv_icmd(dev, 0x00000813, 0x00000006);
2341 nv_icmd(dev, 0x00000814, 0x00000008);
2342 nv_icmd(dev, 0x00000815, 0x0000000b);
2343 nv_icmd(dev, 0x00000800, 0x00000001);
2344 nv_icmd(dev, 0x00000801, 0x00000001);
2345 nv_icmd(dev, 0x00000802, 0x00000001);
2346 nv_icmd(dev, 0x00000803, 0x00000001);
2347 nv_icmd(dev, 0x00000804, 0x00000001);
2348 nv_icmd(dev, 0x00000805, 0x00000001);
2349 nv_icmd(dev, 0x00000632, 0x00000001);
2350 nv_icmd(dev, 0x00000633, 0x00000002);
2351 nv_icmd(dev, 0x00000634, 0x00000003);
2352 nv_icmd(dev, 0x00000635, 0x00000004);
2353 nv_icmd(dev, 0x00000654, 0x3f800000);
2354 nv_icmd(dev, 0x00000657, 0x3f800000);
2355 nv_icmd(dev, 0x00000655, 0x3f800000);
2356 nv_icmd(dev, 0x00000656, 0x3f800000);
2357 nv_icmd(dev, 0x000006cd, 0x3f800000);
2358 nv_icmd(dev, 0x000007f5, 0x3f800000);
2359 nv_icmd(dev, 0x000007dc, 0x39291909);
2360 nv_icmd(dev, 0x000007dd, 0x79695949);
2361 nv_icmd(dev, 0x000007de, 0xb9a99989);
2362 nv_icmd(dev, 0x000007df, 0xf9e9d9c9);
2363 nv_icmd(dev, 0x000007e8, 0x00003210);
2364 nv_icmd(dev, 0x000007e9, 0x00007654);
2365 nv_icmd(dev, 0x000007ea, 0x00000098);
2366 nv_icmd(dev, 0x000007ec, 0x39291909);
2367 nv_icmd(dev, 0x000007ed, 0x79695949);
2368 nv_icmd(dev, 0x000007ee, 0xb9a99989);
2369 nv_icmd(dev, 0x000007ef, 0xf9e9d9c9);
2370 nv_icmd(dev, 0x000007f0, 0x00003210);
2371 nv_icmd(dev, 0x000007f1, 0x00007654);
2372 nv_icmd(dev, 0x000007f2, 0x00000098);
2373 nv_icmd(dev, 0x000005a5, 0x00000001);
2374 nv_icmd(dev, 0x00000980, 0x00000000);
2375 nv_icmd(dev, 0x00000981, 0x00000000);
2376 nv_icmd(dev, 0x00000982, 0x00000000);
2377 nv_icmd(dev, 0x00000983, 0x00000000);
2378 nv_icmd(dev, 0x00000984, 0x00000000);
2379 nv_icmd(dev, 0x00000985, 0x00000000);
2380 nv_icmd(dev, 0x00000986, 0x00000000);
2381 nv_icmd(dev, 0x00000987, 0x00000000);
2382 nv_icmd(dev, 0x00000988, 0x00000000);
2383 nv_icmd(dev, 0x00000989, 0x00000000);
2384 nv_icmd(dev, 0x0000098a, 0x00000000);
2385 nv_icmd(dev, 0x0000098b, 0x00000000);
2386 nv_icmd(dev, 0x0000098c, 0x00000000);
2387 nv_icmd(dev, 0x0000098d, 0x00000000);
2388 nv_icmd(dev, 0x0000098e, 0x00000000);
2389 nv_icmd(dev, 0x0000098f, 0x00000000);
2390 nv_icmd(dev, 0x00000990, 0x00000000);
2391 nv_icmd(dev, 0x00000991, 0x00000000);
2392 nv_icmd(dev, 0x00000992, 0x00000000);
2393 nv_icmd(dev, 0x00000993, 0x00000000);
2394 nv_icmd(dev, 0x00000994, 0x00000000);
2395 nv_icmd(dev, 0x00000995, 0x00000000);
2396 nv_icmd(dev, 0x00000996, 0x00000000);
2397 nv_icmd(dev, 0x00000997, 0x00000000);
2398 nv_icmd(dev, 0x00000998, 0x00000000);
2399 nv_icmd(dev, 0x00000999, 0x00000000);
2400 nv_icmd(dev, 0x0000099a, 0x00000000);
2401 nv_icmd(dev, 0x0000099b, 0x00000000);
2402 nv_icmd(dev, 0x0000099c, 0x00000000);
2403 nv_icmd(dev, 0x0000099d, 0x00000000);
2404 nv_icmd(dev, 0x0000099e, 0x00000000);
2405 nv_icmd(dev, 0x0000099f, 0x00000000);
2406 nv_icmd(dev, 0x000009a0, 0x00000000);
2407 nv_icmd(dev, 0x000009a1, 0x00000000);
2408 nv_icmd(dev, 0x000009a2, 0x00000000);
2409 nv_icmd(dev, 0x000009a3, 0x00000000);
2410 nv_icmd(dev, 0x000009a4, 0x00000000);
2411 nv_icmd(dev, 0x000009a5, 0x00000000);
2412 nv_icmd(dev, 0x000009a6, 0x00000000);
2413 nv_icmd(dev, 0x000009a7, 0x00000000);
2414 nv_icmd(dev, 0x000009a8, 0x00000000);
2415 nv_icmd(dev, 0x000009a9, 0x00000000);
2416 nv_icmd(dev, 0x000009aa, 0x00000000);
2417 nv_icmd(dev, 0x000009ab, 0x00000000);
2418 nv_icmd(dev, 0x000009ac, 0x00000000);
2419 nv_icmd(dev, 0x000009ad, 0x00000000);
2420 nv_icmd(dev, 0x000009ae, 0x00000000);
2421 nv_icmd(dev, 0x000009af, 0x00000000);
2422 nv_icmd(dev, 0x000009b0, 0x00000000);
2423 nv_icmd(dev, 0x000009b1, 0x00000000);
2424 nv_icmd(dev, 0x000009b2, 0x00000000);
2425 nv_icmd(dev, 0x000009b3, 0x00000000);
2426 nv_icmd(dev, 0x000009b4, 0x00000000);
2427 nv_icmd(dev, 0x000009b5, 0x00000000);
2428 nv_icmd(dev, 0x000009b6, 0x00000000);
2429 nv_icmd(dev, 0x000009b7, 0x00000000);
2430 nv_icmd(dev, 0x000009b8, 0x00000000);
2431 nv_icmd(dev, 0x000009b9, 0x00000000);
2432 nv_icmd(dev, 0x000009ba, 0x00000000);
2433 nv_icmd(dev, 0x000009bb, 0x00000000);
2434 nv_icmd(dev, 0x000009bc, 0x00000000);
2435 nv_icmd(dev, 0x000009bd, 0x00000000);
2436 nv_icmd(dev, 0x000009be, 0x00000000);
2437 nv_icmd(dev, 0x000009bf, 0x00000000);
2438 nv_icmd(dev, 0x000009c0, 0x00000000);
2439 nv_icmd(dev, 0x000009c1, 0x00000000);
2440 nv_icmd(dev, 0x000009c2, 0x00000000);
2441 nv_icmd(dev, 0x000009c3, 0x00000000);
2442 nv_icmd(dev, 0x000009c4, 0x00000000);
2443 nv_icmd(dev, 0x000009c5, 0x00000000);
2444 nv_icmd(dev, 0x000009c6, 0x00000000);
2445 nv_icmd(dev, 0x000009c7, 0x00000000);
2446 nv_icmd(dev, 0x000009c8, 0x00000000);
2447 nv_icmd(dev, 0x000009c9, 0x00000000);
2448 nv_icmd(dev, 0x000009ca, 0x00000000);
2449 nv_icmd(dev, 0x000009cb, 0x00000000);
2450 nv_icmd(dev, 0x000009cc, 0x00000000);
2451 nv_icmd(dev, 0x000009cd, 0x00000000);
2452 nv_icmd(dev, 0x000009ce, 0x00000000);
2453 nv_icmd(dev, 0x000009cf, 0x00000000);
2454 nv_icmd(dev, 0x000009d0, 0x00000000);
2455 nv_icmd(dev, 0x000009d1, 0x00000000);
2456 nv_icmd(dev, 0x000009d2, 0x00000000);
2457 nv_icmd(dev, 0x000009d3, 0x00000000);
2458 nv_icmd(dev, 0x000009d4, 0x00000000);
2459 nv_icmd(dev, 0x000009d5, 0x00000000);
2460 nv_icmd(dev, 0x000009d6, 0x00000000);
2461 nv_icmd(dev, 0x000009d7, 0x00000000);
2462 nv_icmd(dev, 0x000009d8, 0x00000000);
2463 nv_icmd(dev, 0x000009d9, 0x00000000);
2464 nv_icmd(dev, 0x000009da, 0x00000000);
2465 nv_icmd(dev, 0x000009db, 0x00000000);
2466 nv_icmd(dev, 0x000009dc, 0x00000000);
2467 nv_icmd(dev, 0x000009dd, 0x00000000);
2468 nv_icmd(dev, 0x000009de, 0x00000000);
2469 nv_icmd(dev, 0x000009df, 0x00000000);
2470 nv_icmd(dev, 0x000009e0, 0x00000000);
2471 nv_icmd(dev, 0x000009e1, 0x00000000);
2472 nv_icmd(dev, 0x000009e2, 0x00000000);
2473 nv_icmd(dev, 0x000009e3, 0x00000000);
2474 nv_icmd(dev, 0x000009e4, 0x00000000);
2475 nv_icmd(dev, 0x000009e5, 0x00000000);
2476 nv_icmd(dev, 0x000009e6, 0x00000000);
2477 nv_icmd(dev, 0x000009e7, 0x00000000);
2478 nv_icmd(dev, 0x000009e8, 0x00000000);
2479 nv_icmd(dev, 0x000009e9, 0x00000000);
2480 nv_icmd(dev, 0x000009ea, 0x00000000);
2481 nv_icmd(dev, 0x000009eb, 0x00000000);
2482 nv_icmd(dev, 0x000009ec, 0x00000000);
2483 nv_icmd(dev, 0x000009ed, 0x00000000);
2484 nv_icmd(dev, 0x000009ee, 0x00000000);
2485 nv_icmd(dev, 0x000009ef, 0x00000000);
2486 nv_icmd(dev, 0x000009f0, 0x00000000);
2487 nv_icmd(dev, 0x000009f1, 0x00000000);
2488 nv_icmd(dev, 0x000009f2, 0x00000000);
2489 nv_icmd(dev, 0x000009f3, 0x00000000);
2490 nv_icmd(dev, 0x000009f4, 0x00000000);
2491 nv_icmd(dev, 0x000009f5, 0x00000000);
2492 nv_icmd(dev, 0x000009f6, 0x00000000);
2493 nv_icmd(dev, 0x000009f7, 0x00000000);
2494 nv_icmd(dev, 0x000009f8, 0x00000000);
2495 nv_icmd(dev, 0x000009f9, 0x00000000);
2496 nv_icmd(dev, 0x000009fa, 0x00000000);
2497 nv_icmd(dev, 0x000009fb, 0x00000000);
2498 nv_icmd(dev, 0x000009fc, 0x00000000);
2499 nv_icmd(dev, 0x000009fd, 0x00000000);
2500 nv_icmd(dev, 0x000009fe, 0x00000000);
2501 nv_icmd(dev, 0x000009ff, 0x00000000);
2502 nv_icmd(dev, 0x00000468, 0x00000004);
2503 nv_icmd(dev, 0x0000046c, 0x00000001);
2504 nv_icmd(dev, 0x00000470, 0x00000000);
2505 nv_icmd(dev, 0x00000471, 0x00000000);
2506 nv_icmd(dev, 0x00000472, 0x00000000);
2507 nv_icmd(dev, 0x00000473, 0x00000000);
2508 nv_icmd(dev, 0x00000474, 0x00000000);
2509 nv_icmd(dev, 0x00000475, 0x00000000);
2510 nv_icmd(dev, 0x00000476, 0x00000000);
2511 nv_icmd(dev, 0x00000477, 0x00000000);
2512 nv_icmd(dev, 0x00000478, 0x00000000);
2513 nv_icmd(dev, 0x00000479, 0x00000000);
2514 nv_icmd(dev, 0x0000047a, 0x00000000);
2515 nv_icmd(dev, 0x0000047b, 0x00000000);
2516 nv_icmd(dev, 0x0000047c, 0x00000000);
2517 nv_icmd(dev, 0x0000047d, 0x00000000);
2518 nv_icmd(dev, 0x0000047e, 0x00000000);
2519 nv_icmd(dev, 0x0000047f, 0x00000000);
2520 nv_icmd(dev, 0x00000480, 0x00000000);
2521 nv_icmd(dev, 0x00000481, 0x00000000);
2522 nv_icmd(dev, 0x00000482, 0x00000000);
2523 nv_icmd(dev, 0x00000483, 0x00000000);
2524 nv_icmd(dev, 0x00000484, 0x00000000);
2525 nv_icmd(dev, 0x00000485, 0x00000000);
2526 nv_icmd(dev, 0x00000486, 0x00000000);
2527 nv_icmd(dev, 0x00000487, 0x00000000);
2528 nv_icmd(dev, 0x00000488, 0x00000000);
2529 nv_icmd(dev, 0x00000489, 0x00000000);
2530 nv_icmd(dev, 0x0000048a, 0x00000000);
2531 nv_icmd(dev, 0x0000048b, 0x00000000);
2532 nv_icmd(dev, 0x0000048c, 0x00000000);
2533 nv_icmd(dev, 0x0000048d, 0x00000000);
2534 nv_icmd(dev, 0x0000048e, 0x00000000);
2535 nv_icmd(dev, 0x0000048f, 0x00000000);
2536 nv_icmd(dev, 0x00000490, 0x00000000);
2537 nv_icmd(dev, 0x00000491, 0x00000000);
2538 nv_icmd(dev, 0x00000492, 0x00000000);
2539 nv_icmd(dev, 0x00000493, 0x00000000);
2540 nv_icmd(dev, 0x00000494, 0x00000000);
2541 nv_icmd(dev, 0x00000495, 0x00000000);
2542 nv_icmd(dev, 0x00000496, 0x00000000);
2543 nv_icmd(dev, 0x00000497, 0x00000000);
2544 nv_icmd(dev, 0x00000498, 0x00000000);
2545 nv_icmd(dev, 0x00000499, 0x00000000);
2546 nv_icmd(dev, 0x0000049a, 0x00000000);
2547 nv_icmd(dev, 0x0000049b, 0x00000000);
2548 nv_icmd(dev, 0x0000049c, 0x00000000);
2549 nv_icmd(dev, 0x0000049d, 0x00000000);
2550 nv_icmd(dev, 0x0000049e, 0x00000000);
2551 nv_icmd(dev, 0x0000049f, 0x00000000);
2552 nv_icmd(dev, 0x000004a0, 0x00000000);
2553 nv_icmd(dev, 0x000004a1, 0x00000000);
2554 nv_icmd(dev, 0x000004a2, 0x00000000);
2555 nv_icmd(dev, 0x000004a3, 0x00000000);
2556 nv_icmd(dev, 0x000004a4, 0x00000000);
2557 nv_icmd(dev, 0x000004a5, 0x00000000);
2558 nv_icmd(dev, 0x000004a6, 0x00000000);
2559 nv_icmd(dev, 0x000004a7, 0x00000000);
2560 nv_icmd(dev, 0x000004a8, 0x00000000);
2561 nv_icmd(dev, 0x000004a9, 0x00000000);
2562 nv_icmd(dev, 0x000004aa, 0x00000000);
2563 nv_icmd(dev, 0x000004ab, 0x00000000);
2564 nv_icmd(dev, 0x000004ac, 0x00000000);
2565 nv_icmd(dev, 0x000004ad, 0x00000000);
2566 nv_icmd(dev, 0x000004ae, 0x00000000);
2567 nv_icmd(dev, 0x000004af, 0x00000000);
2568 nv_icmd(dev, 0x000004b0, 0x00000000);
2569 nv_icmd(dev, 0x000004b1, 0x00000000);
2570 nv_icmd(dev, 0x000004b2, 0x00000000);
2571 nv_icmd(dev, 0x000004b3, 0x00000000);
2572 nv_icmd(dev, 0x000004b4, 0x00000000);
2573 nv_icmd(dev, 0x000004b5, 0x00000000);
2574 nv_icmd(dev, 0x000004b6, 0x00000000);
2575 nv_icmd(dev, 0x000004b7, 0x00000000);
2576 nv_icmd(dev, 0x000004b8, 0x00000000);
2577 nv_icmd(dev, 0x000004b9, 0x00000000);
2578 nv_icmd(dev, 0x000004ba, 0x00000000);
2579 nv_icmd(dev, 0x000004bb, 0x00000000);
2580 nv_icmd(dev, 0x000004bc, 0x00000000);
2581 nv_icmd(dev, 0x000004bd, 0x00000000);
2582 nv_icmd(dev, 0x000004be, 0x00000000);
2583 nv_icmd(dev, 0x000004bf, 0x00000000);
2584 nv_icmd(dev, 0x000004c0, 0x00000000);
2585 nv_icmd(dev, 0x000004c1, 0x00000000);
2586 nv_icmd(dev, 0x000004c2, 0x00000000);
2587 nv_icmd(dev, 0x000004c3, 0x00000000);
2588 nv_icmd(dev, 0x000004c4, 0x00000000);
2589 nv_icmd(dev, 0x000004c5, 0x00000000);
2590 nv_icmd(dev, 0x000004c6, 0x00000000);
2591 nv_icmd(dev, 0x000004c7, 0x00000000);
2592 nv_icmd(dev, 0x000004c8, 0x00000000);
2593 nv_icmd(dev, 0x000004c9, 0x00000000);
2594 nv_icmd(dev, 0x000004ca, 0x00000000);
2595 nv_icmd(dev, 0x000004cb, 0x00000000);
2596 nv_icmd(dev, 0x000004cc, 0x00000000);
2597 nv_icmd(dev, 0x000004cd, 0x00000000);
2598 nv_icmd(dev, 0x000004ce, 0x00000000);
2599 nv_icmd(dev, 0x000004cf, 0x00000000);
2600 nv_icmd(dev, 0x00000510, 0x3f800000);
2601 nv_icmd(dev, 0x00000511, 0x3f800000);
2602 nv_icmd(dev, 0x00000512, 0x3f800000);
2603 nv_icmd(dev, 0x00000513, 0x3f800000);
2604 nv_icmd(dev, 0x00000514, 0x3f800000);
2605 nv_icmd(dev, 0x00000515, 0x3f800000);
2606 nv_icmd(dev, 0x00000516, 0x3f800000);
2607 nv_icmd(dev, 0x00000517, 0x3f800000);
2608 nv_icmd(dev, 0x00000518, 0x3f800000);
2609 nv_icmd(dev, 0x00000519, 0x3f800000);
2610 nv_icmd(dev, 0x0000051a, 0x3f800000);
2611 nv_icmd(dev, 0x0000051b, 0x3f800000);
2612 nv_icmd(dev, 0x0000051c, 0x3f800000);
2613 nv_icmd(dev, 0x0000051d, 0x3f800000);
2614 nv_icmd(dev, 0x0000051e, 0x3f800000);
2615 nv_icmd(dev, 0x0000051f, 0x3f800000);
2616 nv_icmd(dev, 0x00000520, 0x000002b6);
2617 nv_icmd(dev, 0x00000529, 0x00000001);
2618 nv_icmd(dev, 0x00000530, 0xffff0000);
2619 nv_icmd(dev, 0x00000531, 0xffff0000);
2620 nv_icmd(dev, 0x00000532, 0xffff0000);
2621 nv_icmd(dev, 0x00000533, 0xffff0000);
2622 nv_icmd(dev, 0x00000534, 0xffff0000);
2623 nv_icmd(dev, 0x00000535, 0xffff0000);
2624 nv_icmd(dev, 0x00000536, 0xffff0000);
2625 nv_icmd(dev, 0x00000537, 0xffff0000);
2626 nv_icmd(dev, 0x00000538, 0xffff0000);
2627 nv_icmd(dev, 0x00000539, 0xffff0000);
2628 nv_icmd(dev, 0x0000053a, 0xffff0000);
2629 nv_icmd(dev, 0x0000053b, 0xffff0000);
2630 nv_icmd(dev, 0x0000053c, 0xffff0000);
2631 nv_icmd(dev, 0x0000053d, 0xffff0000);
2632 nv_icmd(dev, 0x0000053e, 0xffff0000);
2633 nv_icmd(dev, 0x0000053f, 0xffff0000);
2634 nv_icmd(dev, 0x00000585, 0x0000003f);
2635 nv_icmd(dev, 0x00000576, 0x00000003);
2636 nv_icmd(dev, 0x00000586, 0x00000040);
2637 nv_icmd(dev, 0x00000582, 0x00000080);
2638 nv_icmd(dev, 0x00000583, 0x00000080);
2639 nv_icmd(dev, 0x000005c2, 0x00000001);
2640 nv_icmd(dev, 0x00000638, 0x00000001);
2641 nv_icmd(dev, 0x00000639, 0x00000001);
2642 nv_icmd(dev, 0x0000063a, 0x00000002);
2643 nv_icmd(dev, 0x0000063b, 0x00000001);
2644 nv_icmd(dev, 0x0000063c, 0x00000001);
2645 nv_icmd(dev, 0x0000063d, 0x00000002);
2646 nv_icmd(dev, 0x0000063e, 0x00000001);
2647 nv_icmd(dev, 0x000008b8, 0x00000001);
2648 nv_icmd(dev, 0x000008b9, 0x00000001);
2649 nv_icmd(dev, 0x000008ba, 0x00000001);
2650 nv_icmd(dev, 0x000008bb, 0x00000001);
2651 nv_icmd(dev, 0x000008bc, 0x00000001);
2652 nv_icmd(dev, 0x000008bd, 0x00000001);
2653 nv_icmd(dev, 0x000008be, 0x00000001);
2654 nv_icmd(dev, 0x000008bf, 0x00000001);
2655 nv_icmd(dev, 0x00000900, 0x00000001);
2656 nv_icmd(dev, 0x00000901, 0x00000001);
2657 nv_icmd(dev, 0x00000902, 0x00000001);
2658 nv_icmd(dev, 0x00000903, 0x00000001);
2659 nv_icmd(dev, 0x00000904, 0x00000001);
2660 nv_icmd(dev, 0x00000905, 0x00000001);
2661 nv_icmd(dev, 0x00000906, 0x00000001);
2662 nv_icmd(dev, 0x00000907, 0x00000001);
2663 nv_icmd(dev, 0x00000908, 0x00000002);
2664 nv_icmd(dev, 0x00000909, 0x00000002);
2665 nv_icmd(dev, 0x0000090a, 0x00000002);
2666 nv_icmd(dev, 0x0000090b, 0x00000002);
2667 nv_icmd(dev, 0x0000090c, 0x00000002);
2668 nv_icmd(dev, 0x0000090d, 0x00000002);
2669 nv_icmd(dev, 0x0000090e, 0x00000002);
2670 nv_icmd(dev, 0x0000090f, 0x00000002);
2671 nv_icmd(dev, 0x00000910, 0x00000001);
2672 nv_icmd(dev, 0x00000911, 0x00000001);
2673 nv_icmd(dev, 0x00000912, 0x00000001);
2674 nv_icmd(dev, 0x00000913, 0x00000001);
2675 nv_icmd(dev, 0x00000914, 0x00000001);
2676 nv_icmd(dev, 0x00000915, 0x00000001);
2677 nv_icmd(dev, 0x00000916, 0x00000001);
2678 nv_icmd(dev, 0x00000917, 0x00000001);
2679 nv_icmd(dev, 0x00000918, 0x00000001);
2680 nv_icmd(dev, 0x00000919, 0x00000001);
2681 nv_icmd(dev, 0x0000091a, 0x00000001);
2682 nv_icmd(dev, 0x0000091b, 0x00000001);
2683 nv_icmd(dev, 0x0000091c, 0x00000001);
2684 nv_icmd(dev, 0x0000091d, 0x00000001);
2685 nv_icmd(dev, 0x0000091e, 0x00000001);
2686 nv_icmd(dev, 0x0000091f, 0x00000001);
2687 nv_icmd(dev, 0x00000920, 0x00000002);
2688 nv_icmd(dev, 0x00000921, 0x00000002);
2689 nv_icmd(dev, 0x00000922, 0x00000002);
2690 nv_icmd(dev, 0x00000923, 0x00000002);
2691 nv_icmd(dev, 0x00000924, 0x00000002);
2692 nv_icmd(dev, 0x00000925, 0x00000002);
2693 nv_icmd(dev, 0x00000926, 0x00000002);
2694 nv_icmd(dev, 0x00000927, 0x00000002);
2695 nv_icmd(dev, 0x00000928, 0x00000001);
2696 nv_icmd(dev, 0x00000929, 0x00000001);
2697 nv_icmd(dev, 0x0000092a, 0x00000001);
2698 nv_icmd(dev, 0x0000092b, 0x00000001);
2699 nv_icmd(dev, 0x0000092c, 0x00000001);
2700 nv_icmd(dev, 0x0000092d, 0x00000001);
2701 nv_icmd(dev, 0x0000092e, 0x00000001);
2702 nv_icmd(dev, 0x0000092f, 0x00000001);
2703 nv_icmd(dev, 0x00000648, 0x00000001);
2704 nv_icmd(dev, 0x00000649, 0x00000001);
2705 nv_icmd(dev, 0x0000064a, 0x00000001);
2706 nv_icmd(dev, 0x0000064b, 0x00000001);
2707 nv_icmd(dev, 0x0000064c, 0x00000001);
2708 nv_icmd(dev, 0x0000064d, 0x00000001);
2709 nv_icmd(dev, 0x0000064e, 0x00000001);
2710 nv_icmd(dev, 0x0000064f, 0x00000001);
2711 nv_icmd(dev, 0x00000650, 0x00000001);
2712 nv_icmd(dev, 0x00000658, 0x0000000f);
2713 nv_icmd(dev, 0x000007ff, 0x0000000a);
2714 nv_icmd(dev, 0x0000066a, 0x40000000);
2715 nv_icmd(dev, 0x0000066b, 0x10000000);
2716 nv_icmd(dev, 0x0000066c, 0xffff0000);
2717 nv_icmd(dev, 0x0000066d, 0xffff0000);
2718 nv_icmd(dev, 0x000007af, 0x00000008);
2719 nv_icmd(dev, 0x000007b0, 0x00000008);
2720 nv_icmd(dev, 0x000007f6, 0x00000001);
2721 nv_icmd(dev, 0x000006b2, 0x00000055);
2722 nv_icmd(dev, 0x000007ad, 0x00000003);
2723 nv_icmd(dev, 0x00000937, 0x00000001);
2724 nv_icmd(dev, 0x00000971, 0x00000008);
2725 nv_icmd(dev, 0x00000972, 0x00000040);
2726 nv_icmd(dev, 0x00000973, 0x0000012c);
2727 nv_icmd(dev, 0x0000097c, 0x00000040);
2728 nv_icmd(dev, 0x00000979, 0x00000003);
2729 nv_icmd(dev, 0x00000975, 0x00000020);
2730 nv_icmd(dev, 0x00000976, 0x00000001);
2731 nv_icmd(dev, 0x00000977, 0x00000020);
2732 nv_icmd(dev, 0x00000978, 0x00000001);
2733 nv_icmd(dev, 0x00000957, 0x00000003);
2734 nv_icmd(dev, 0x0000095e, 0x20164010);
2735 nv_icmd(dev, 0x0000095f, 0x00000020);
2736 nv_icmd(dev, 0x00000683, 0x00000006);
2737 nv_icmd(dev, 0x00000685, 0x003fffff);
2738 nv_icmd(dev, 0x00000687, 0x00000c48);
2739 nv_icmd(dev, 0x000006a0, 0x00000005);
2740 nv_icmd(dev, 0x00000840, 0x00300008);
2741 nv_icmd(dev, 0x00000841, 0x04000080);
2742 nv_icmd(dev, 0x00000842, 0x00300008);
2743 nv_icmd(dev, 0x00000843, 0x04000080);
2744 nv_icmd(dev, 0x00000818, 0x00000000);
2745 nv_icmd(dev, 0x00000819, 0x00000000);
2746 nv_icmd(dev, 0x0000081a, 0x00000000);
2747 nv_icmd(dev, 0x0000081b, 0x00000000);
2748 nv_icmd(dev, 0x0000081c, 0x00000000);
2749 nv_icmd(dev, 0x0000081d, 0x00000000);
2750 nv_icmd(dev, 0x0000081e, 0x00000000);
2751 nv_icmd(dev, 0x0000081f, 0x00000000);
2752 nv_icmd(dev, 0x00000848, 0x00000000);
2753 nv_icmd(dev, 0x00000849, 0x00000000);
2754 nv_icmd(dev, 0x0000084a, 0x00000000);
2755 nv_icmd(dev, 0x0000084b, 0x00000000);
2756 nv_icmd(dev, 0x0000084c, 0x00000000);
2757 nv_icmd(dev, 0x0000084d, 0x00000000);
2758 nv_icmd(dev, 0x0000084e, 0x00000000);
2759 nv_icmd(dev, 0x0000084f, 0x00000000);
2760 nv_icmd(dev, 0x00000850, 0x00000000);
2761 nv_icmd(dev, 0x00000851, 0x00000000);
2762 nv_icmd(dev, 0x00000852, 0x00000000);
2763 nv_icmd(dev, 0x00000853, 0x00000000);
2764 nv_icmd(dev, 0x00000854, 0x00000000);
2765 nv_icmd(dev, 0x00000855, 0x00000000);
2766 nv_icmd(dev, 0x00000856, 0x00000000);
2767 nv_icmd(dev, 0x00000857, 0x00000000);
2768 nv_icmd(dev, 0x00000738, 0x00000000);
2769 nv_icmd(dev, 0x000006aa, 0x00000001);
2770 nv_icmd(dev, 0x000006ab, 0x00000002);
2771 nv_icmd(dev, 0x000006ac, 0x00000080);
2772 nv_icmd(dev, 0x000006ad, 0x00000100);
2773 nv_icmd(dev, 0x000006ae, 0x00000100);
2774 nv_icmd(dev, 0x000006b1, 0x00000011);
2775 nv_icmd(dev, 0x000006bb, 0x000000cf);
2776 nv_icmd(dev, 0x000006ce, 0x2a712488);
2777 nv_icmd(dev, 0x00000739, 0x4085c000);
2778 nv_icmd(dev, 0x0000073a, 0x00000080);
2779 nv_icmd(dev, 0x00000786, 0x80000100);
2780 nv_icmd(dev, 0x0000073c, 0x00010100);
2781 nv_icmd(dev, 0x0000073d, 0x02800000);
2782 nv_icmd(dev, 0x00000787, 0x000000cf);
2783 nv_icmd(dev, 0x0000078c, 0x00000008);
2784 nv_icmd(dev, 0x00000792, 0x00000001);
2785 nv_icmd(dev, 0x00000794, 0x00000001);
2786 nv_icmd(dev, 0x00000795, 0x00000001);
2787 nv_icmd(dev, 0x00000796, 0x00000001);
2788 nv_icmd(dev, 0x00000797, 0x000000cf);
2789 nv_icmd(dev, 0x00000836, 0x00000001);
2790 nv_icmd(dev, 0x0000079a, 0x00000002);
2791 nv_icmd(dev, 0x00000833, 0x04444480);
2792 nv_icmd(dev, 0x000007a1, 0x00000001);
2793 nv_icmd(dev, 0x000007a3, 0x00000001);
2794 nv_icmd(dev, 0x000007a4, 0x00000001);
2795 nv_icmd(dev, 0x000007a5, 0x00000001);
2796 nv_icmd(dev, 0x00000831, 0x00000004);
2797 nv_icmd(dev, 0x0000080c, 0x00000002);
2798 nv_icmd(dev, 0x0000080d, 0x00000100);
2799 nv_icmd(dev, 0x0000080e, 0x00000100);
2800 nv_icmd(dev, 0x0000080f, 0x00000001);
2801 nv_icmd(dev, 0x00000823, 0x00000002);
2802 nv_icmd(dev, 0x00000824, 0x00000100);
2803 nv_icmd(dev, 0x00000825, 0x00000100);
2804 nv_icmd(dev, 0x00000826, 0x00000001);
2805 nv_icmd(dev, 0x0000095d, 0x00000001);
2806 nv_icmd(dev, 0x0000082b, 0x00000004);
2807 nv_icmd(dev, 0x00000942, 0x00010001);
2808 nv_icmd(dev, 0x00000943, 0x00000001);
2809 nv_icmd(dev, 0x00000944, 0x00000022);
2810 nv_icmd(dev, 0x000007c5, 0x00010001);
2811 nv_icmd(dev, 0x00000834, 0x00000001);
2812 nv_icmd(dev, 0x000007c7, 0x00000001);
2813 nv_icmd(dev, 0x0000c1b0, 0x0000000f);
2814 nv_icmd(dev, 0x0000c1b1, 0x0000000f);
2815 nv_icmd(dev, 0x0000c1b2, 0x0000000f);
2816 nv_icmd(dev, 0x0000c1b3, 0x0000000f);
2817 nv_icmd(dev, 0x0000c1b4, 0x0000000f);
2818 nv_icmd(dev, 0x0000c1b5, 0x0000000f);
2819 nv_icmd(dev, 0x0000c1b6, 0x0000000f);
2820 nv_icmd(dev, 0x0000c1b7, 0x0000000f);
2821 nv_icmd(dev, 0x0000c1b8, 0x0fac6881);
2822 nv_icmd(dev, 0x0000c1b9, 0x00fac688);
2823 nv_icmd(dev, 0x0001e100, 0x00000001);
2824 nv_icmd(dev, 0x00001000, 0x00000002);
2825 nv_icmd(dev, 0x000006aa, 0x00000001);
2826 nv_icmd(dev, 0x000006ad, 0x00000100);
2827 nv_icmd(dev, 0x000006ae, 0x00000100);
2828 nv_icmd(dev, 0x000006b1, 0x00000011);
2829 nv_icmd(dev, 0x0000078c, 0x00000008);
2830 nv_icmd(dev, 0x00000792, 0x00000001);
2831 nv_icmd(dev, 0x00000794, 0x00000001);
2832 nv_icmd(dev, 0x00000795, 0x00000001);
2833 nv_icmd(dev, 0x00000796, 0x00000001);
2834 nv_icmd(dev, 0x00000797, 0x000000cf);
2835 nv_icmd(dev, 0x0000079a, 0x00000002);
2836 nv_icmd(dev, 0x00000833, 0x04444480);
2837 nv_icmd(dev, 0x000007a1, 0x00000001);
2838 nv_icmd(dev, 0x000007a3, 0x00000001);
2839 nv_icmd(dev, 0x000007a4, 0x00000001);
2840 nv_icmd(dev, 0x000007a5, 0x00000001);
2841 nv_icmd(dev, 0x00000831, 0x00000004);
2842 nv_icmd(dev, 0x0001e100, 0x00000001);
2843 nv_icmd(dev, 0x00001000, 0x00000014);
2844 nv_icmd(dev, 0x00000351, 0x00000100);
2845 nv_icmd(dev, 0x00000957, 0x00000003);
2846 nv_icmd(dev, 0x0000095d, 0x00000001);
2847 nv_icmd(dev, 0x0000082b, 0x00000004);
2848 nv_icmd(dev, 0x00000942, 0x00010001);
2849 nv_icmd(dev, 0x00000943, 0x00000001);
2850 nv_icmd(dev, 0x000007c5, 0x00010001);
2851 nv_icmd(dev, 0x00000834, 0x00000001);
2852 nv_icmd(dev, 0x000007c7, 0x00000001);
2853 nv_icmd(dev, 0x0001e100, 0x00000001);
2854 nv_icmd(dev, 0x00001000, 0x00000001);
2855 nv_icmd(dev, 0x0000080c, 0x00000002);
2856 nv_icmd(dev, 0x0000080d, 0x00000100);
2857 nv_icmd(dev, 0x0000080e, 0x00000100);
2858 nv_icmd(dev, 0x0000080f, 0x00000001);
2859 nv_icmd(dev, 0x00000823, 0x00000002);
2860 nv_icmd(dev, 0x00000824, 0x00000100);
2861 nv_icmd(dev, 0x00000825, 0x00000100);
2862 nv_icmd(dev, 0x00000826, 0x00000001);
2863 nv_icmd(dev, 0x0001e100, 0x00000001);
2864 nv_wr32(dev, 0x400208, 0x00000000);
2865 nv_wr32(dev, 0x404154, 0x00000400);
2866
2867 nvc0_grctx_generate_9097(dev);
2868 nvc0_grctx_generate_902d(dev);
2869 nvc0_grctx_generate_9039(dev);
2870 nvc0_grctx_generate_90c0(dev);
2871
2872 nv_wr32(dev, 0x000260, r000260);
2873 return 0;
2874}
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
index 13a0f78a9088..c09091749054 100644
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ b/drivers/gpu/drm/nouveau/nvc0_instmem.c
@@ -25,206 +25,207 @@
25#include "drmP.h" 25#include "drmP.h"
26 26
27#include "nouveau_drv.h" 27#include "nouveau_drv.h"
28#include "nouveau_vm.h"
29
30struct nvc0_instmem_priv {
31 struct nouveau_gpuobj *bar1_pgd;
32 struct nouveau_channel *bar1;
33 struct nouveau_gpuobj *bar3_pgd;
34 struct nouveau_channel *bar3;
35 struct nouveau_gpuobj *chan_pgd;
36};
28 37
29int 38int
30nvc0_instmem_populate(struct drm_device *dev, struct nouveau_gpuobj *gpuobj, 39nvc0_instmem_suspend(struct drm_device *dev)
31 uint32_t *size)
32{ 40{
33 int ret; 41 struct drm_nouveau_private *dev_priv = dev->dev_private;
34
35 *size = ALIGN(*size, 4096);
36 if (*size == 0)
37 return -EINVAL;
38
39 ret = nouveau_bo_new(dev, NULL, *size, 0, TTM_PL_FLAG_VRAM, 0, 0x0000,
40 true, false, &gpuobj->im_backing);
41 if (ret) {
42 NV_ERROR(dev, "error getting PRAMIN backing pages: %d\n", ret);
43 return ret;
44 }
45
46 ret = nouveau_bo_pin(gpuobj->im_backing, TTM_PL_FLAG_VRAM);
47 if (ret) {
48 NV_ERROR(dev, "error pinning PRAMIN backing VRAM: %d\n", ret);
49 nouveau_bo_ref(NULL, &gpuobj->im_backing);
50 return ret;
51 }
52 42
53 gpuobj->vinst = gpuobj->im_backing->bo.mem.start << PAGE_SHIFT; 43 dev_priv->ramin_available = false;
54 return 0; 44 return 0;
55} 45}
56 46
57void 47void
58nvc0_instmem_clear(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 48nvc0_instmem_resume(struct drm_device *dev)
59{ 49{
60 struct drm_nouveau_private *dev_priv = dev->dev_private; 50 struct drm_nouveau_private *dev_priv = dev->dev_private;
51 struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
61 52
62 if (gpuobj && gpuobj->im_backing) { 53 nv_mask(dev, 0x100c80, 0x00000001, 0x00000000);
63 if (gpuobj->im_bound) 54 nv_wr32(dev, 0x001704, 0x80000000 | priv->bar1->ramin->vinst >> 12);
64 dev_priv->engine.instmem.unbind(dev, gpuobj); 55 nv_wr32(dev, 0x001714, 0xc0000000 | priv->bar3->ramin->vinst >> 12);
65 nouveau_bo_unpin(gpuobj->im_backing); 56 dev_priv->ramin_available = true;
66 nouveau_bo_ref(NULL, &gpuobj->im_backing);
67 gpuobj->im_backing = NULL;
68 }
69} 57}
70 58
71int 59static void
72nvc0_instmem_bind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj) 60nvc0_channel_del(struct nouveau_channel **pchan)
73{ 61{
74 struct drm_nouveau_private *dev_priv = dev->dev_private; 62 struct nouveau_channel *chan;
75 uint32_t pte, pte_end; 63
76 uint64_t vram; 64 chan = *pchan;
77 65 *pchan = NULL;
78 if (!gpuobj->im_backing || !gpuobj->im_pramin || gpuobj->im_bound) 66 if (!chan)
79 return -EINVAL; 67 return;
80 68
81 NV_DEBUG(dev, "st=0x%lx sz=0x%lx\n", 69 nouveau_vm_ref(NULL, &chan->vm, NULL);
82 gpuobj->im_pramin->start, gpuobj->im_pramin->size); 70 if (chan->ramin_heap.free_stack.next)
71 drm_mm_takedown(&chan->ramin_heap);
72 nouveau_gpuobj_ref(NULL, &chan->ramin);
73 kfree(chan);
74}
83 75
84 pte = gpuobj->im_pramin->start >> 12; 76static int
85 pte_end = (gpuobj->im_pramin->size >> 12) + pte; 77nvc0_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
86 vram = gpuobj->vinst; 78 struct nouveau_channel **pchan,
79 struct nouveau_gpuobj *pgd, u64 vm_size)
80{
81 struct nouveau_channel *chan;
82 int ret;
87 83
88 NV_DEBUG(dev, "pramin=0x%lx, pte=%d, pte_end=%d\n", 84 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
89 gpuobj->im_pramin->start, pte, pte_end); 85 if (!chan)
90 NV_DEBUG(dev, "first vram page: 0x%010llx\n", gpuobj->vinst); 86 return -ENOMEM;
87 chan->dev = dev;
91 88
92 while (pte < pte_end) { 89 ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
93 nv_wr32(dev, 0x702000 + (pte * 8), (vram >> 8) | 1); 90 if (ret) {
94 nv_wr32(dev, 0x702004 + (pte * 8), 0); 91 nvc0_channel_del(&chan);
95 vram += 4096; 92 return ret;
96 pte++;
97 } 93 }
98 dev_priv->engine.instmem.flush(dev);
99 94
100 if (1) { 95 ret = drm_mm_init(&chan->ramin_heap, 0x1000, size - 0x1000);
101 u32 chan = nv_rd32(dev, 0x1700) << 16; 96 if (ret) {
102 nv_wr32(dev, 0x100cb8, (chan + 0x1000) >> 8); 97 nvc0_channel_del(&chan);
103 nv_wr32(dev, 0x100cbc, 0x80000005); 98 return ret;
104 } 99 }
105 100
106 gpuobj->im_bound = 1; 101 ret = nouveau_vm_ref(vm, &chan->vm, NULL);
107 return 0; 102 if (ret) {
108} 103 nvc0_channel_del(&chan);
109 104 return ret;
110int
111nvc0_instmem_unbind(struct drm_device *dev, struct nouveau_gpuobj *gpuobj)
112{
113 struct drm_nouveau_private *dev_priv = dev->dev_private;
114 uint32_t pte, pte_end;
115
116 if (gpuobj->im_bound == 0)
117 return -EINVAL;
118
119 pte = gpuobj->im_pramin->start >> 12;
120 pte_end = (gpuobj->im_pramin->size >> 12) + pte;
121 while (pte < pte_end) {
122 nv_wr32(dev, 0x702000 + (pte * 8), 0);
123 nv_wr32(dev, 0x702004 + (pte * 8), 0);
124 pte++;
125 } 105 }
126 dev_priv->engine.instmem.flush(dev);
127 106
128 gpuobj->im_bound = 0; 107 nv_wo32(chan->ramin, 0x0200, lower_32_bits(pgd->vinst));
129 return 0; 108 nv_wo32(chan->ramin, 0x0204, upper_32_bits(pgd->vinst));
130} 109 nv_wo32(chan->ramin, 0x0208, lower_32_bits(vm_size - 1));
110 nv_wo32(chan->ramin, 0x020c, upper_32_bits(vm_size - 1));
131 111
132void 112 *pchan = chan;
133nvc0_instmem_flush(struct drm_device *dev) 113 return 0;
134{
135 nv_wr32(dev, 0x070000, 1);
136 if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
137 NV_ERROR(dev, "PRAMIN flush timeout\n");
138} 114}
139 115
140int 116int
141nvc0_instmem_suspend(struct drm_device *dev) 117nvc0_instmem_init(struct drm_device *dev)
142{ 118{
143 struct drm_nouveau_private *dev_priv = dev->dev_private; 119 struct drm_nouveau_private *dev_priv = dev->dev_private;
144 u32 *buf; 120 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
145 int i; 121 struct pci_dev *pdev = dev->pdev;
122 struct nvc0_instmem_priv *priv;
123 struct nouveau_vm *vm = NULL;
124 int ret;
146 125
147 dev_priv->susres.ramin_copy = vmalloc(65536); 126 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
148 if (!dev_priv->susres.ramin_copy) 127 if (!priv)
149 return -ENOMEM; 128 return -ENOMEM;
150 buf = dev_priv->susres.ramin_copy; 129 pinstmem->priv = priv;
151 130
152 for (i = 0; i < 65536; i += 4) 131 /* BAR3 VM */
153 buf[i/4] = nv_rd32(dev, NV04_PRAMIN + i); 132 ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 3), 0,
133 &dev_priv->bar3_vm);
134 if (ret)
135 goto error;
136
137 ret = nouveau_gpuobj_new(dev, NULL,
138 (pci_resource_len(pdev, 3) >> 12) * 8, 0,
139 NVOBJ_FLAG_DONT_MAP |
140 NVOBJ_FLAG_ZERO_ALLOC,
141 &dev_priv->bar3_vm->pgt[0].obj[0]);
142 if (ret)
143 goto error;
144 dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
145
146 nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
147
148 ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
149 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar3_pgd);
150 if (ret)
151 goto error;
152
153 ret = nouveau_vm_ref(dev_priv->bar3_vm, &vm, priv->bar3_pgd);
154 if (ret)
155 goto error;
156 nouveau_vm_ref(NULL, &vm, NULL);
157
158 ret = nvc0_channel_new(dev, 8192, dev_priv->bar3_vm, &priv->bar3,
159 priv->bar3_pgd, pci_resource_len(dev->pdev, 3));
160 if (ret)
161 goto error;
162
163 /* BAR1 VM */
164 ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 1), 0, &vm);
165 if (ret)
166 goto error;
167
168 ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
169 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar1_pgd);
170 if (ret)
171 goto error;
172
173 ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, priv->bar1_pgd);
174 if (ret)
175 goto error;
176 nouveau_vm_ref(NULL, &vm, NULL);
177
178 ret = nvc0_channel_new(dev, 8192, dev_priv->bar1_vm, &priv->bar1,
179 priv->bar1_pgd, pci_resource_len(dev->pdev, 1));
180 if (ret)
181 goto error;
182
183 /* channel vm */
184 ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL, &vm);
185 if (ret)
186 goto error;
187
188 ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096, 0, &priv->chan_pgd);
189 if (ret)
190 goto error;
191
192 nouveau_vm_ref(vm, &dev_priv->chan_vm, priv->chan_pgd);
193 nouveau_vm_ref(NULL, &vm, NULL);
194
195 nvc0_instmem_resume(dev);
154 return 0; 196 return 0;
197error:
198 nvc0_instmem_takedown(dev);
199 return ret;
155} 200}
156 201
157void 202void
158nvc0_instmem_resume(struct drm_device *dev) 203nvc0_instmem_takedown(struct drm_device *dev)
159{ 204{
160 struct drm_nouveau_private *dev_priv = dev->dev_private; 205 struct drm_nouveau_private *dev_priv = dev->dev_private;
161 u32 *buf = dev_priv->susres.ramin_copy; 206 struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
162 u64 chan; 207 struct nouveau_vm *vm = NULL;
163 int i;
164 208
165 chan = dev_priv->vram_size - dev_priv->ramin_rsvd_vram; 209 nvc0_instmem_suspend(dev);
166 nv_wr32(dev, 0x001700, chan >> 16);
167 210
168 for (i = 0; i < 65536; i += 4) 211 nv_wr32(dev, 0x1704, 0x00000000);
169 nv_wr32(dev, NV04_PRAMIN + i, buf[i/4]); 212 nv_wr32(dev, 0x1714, 0x00000000);
170 vfree(dev_priv->susres.ramin_copy);
171 dev_priv->susres.ramin_copy = NULL;
172 213
173 nv_wr32(dev, 0x001714, 0xc0000000 | (chan >> 12)); 214 nouveau_vm_ref(NULL, &dev_priv->chan_vm, priv->chan_pgd);
174} 215 nouveau_gpuobj_ref(NULL, &priv->chan_pgd);
175 216
176int 217 nvc0_channel_del(&priv->bar1);
177nvc0_instmem_init(struct drm_device *dev) 218 nouveau_vm_ref(NULL, &dev_priv->bar1_vm, priv->bar1_pgd);
178{ 219 nouveau_gpuobj_ref(NULL, &priv->bar1_pgd);
179 struct drm_nouveau_private *dev_priv = dev->dev_private;
180 u64 chan, pgt3, imem, lim3 = dev_priv->ramin_size - 1;
181 int ret, i;
182
183 dev_priv->ramin_rsvd_vram = 1 * 1024 * 1024;
184 chan = dev_priv->vram_size - dev_priv->ramin_rsvd_vram;
185 imem = 4096 + 4096 + 32768;
186
187 nv_wr32(dev, 0x001700, chan >> 16);
188
189 /* channel setup */
190 nv_wr32(dev, 0x700200, lower_32_bits(chan + 0x1000));
191 nv_wr32(dev, 0x700204, upper_32_bits(chan + 0x1000));
192 nv_wr32(dev, 0x700208, lower_32_bits(lim3));
193 nv_wr32(dev, 0x70020c, upper_32_bits(lim3));
194
195 /* point pgd -> pgt */
196 nv_wr32(dev, 0x701000, 0);
197 nv_wr32(dev, 0x701004, ((chan + 0x2000) >> 8) | 1);
198
199 /* point pgt -> physical vram for channel */
200 pgt3 = 0x2000;
201 for (i = 0; i < dev_priv->ramin_rsvd_vram; i += 4096, pgt3 += 8) {
202 nv_wr32(dev, 0x700000 + pgt3, ((chan + i) >> 8) | 1);
203 nv_wr32(dev, 0x700004 + pgt3, 0);
204 }
205
206 /* clear rest of pgt */
207 for (; i < dev_priv->ramin_size; i += 4096, pgt3 += 8) {
208 nv_wr32(dev, 0x700000 + pgt3, 0);
209 nv_wr32(dev, 0x700004 + pgt3, 0);
210 }
211
212 /* point bar3 at the channel */
213 nv_wr32(dev, 0x001714, 0xc0000000 | (chan >> 12));
214
215 /* Global PRAMIN heap */
216 ret = drm_mm_init(&dev_priv->ramin_heap, imem,
217 dev_priv->ramin_size - imem);
218 if (ret) {
219 NV_ERROR(dev, "Failed to init RAMIN heap\n");
220 return -ENOMEM;
221 }
222 220
223 return 0; 221 nvc0_channel_del(&priv->bar3);
224} 222 nouveau_vm_ref(dev_priv->bar3_vm, &vm, NULL);
223 nouveau_vm_ref(NULL, &vm, priv->bar3_pgd);
224 nouveau_gpuobj_ref(NULL, &priv->bar3_pgd);
225 nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
226 nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
225 227
226void 228 dev_priv->engine.instmem.priv = NULL;
227nvc0_instmem_takedown(struct drm_device *dev) 229 kfree(priv);
228{
229} 230}
230 231
diff --git a/drivers/gpu/drm/nouveau/nvc0_vm.c b/drivers/gpu/drm/nouveau/nvc0_vm.c
new file mode 100644
index 000000000000..4b9251bb0ff4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_vm.c
@@ -0,0 +1,123 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26
27#include "nouveau_drv.h"
28#include "nouveau_vm.h"
29
30void
31nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 index,
32 struct nouveau_gpuobj *pgt[2])
33{
34 u32 pde[2] = { 0, 0 };
35
36 if (pgt[0])
37 pde[1] = 0x00000001 | (pgt[0]->vinst >> 8);
38 if (pgt[1])
39 pde[0] = 0x00000001 | (pgt[1]->vinst >> 8);
40
41 nv_wo32(pgd, (index * 8) + 0, pde[0]);
42 nv_wo32(pgd, (index * 8) + 4, pde[1]);
43}
44
45static inline u64
46nvc0_vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
47{
48 phys >>= 8;
49
50 phys |= 0x00000001; /* present */
51// if (vma->access & NV_MEM_ACCESS_SYS)
52// phys |= 0x00000002;
53
54 phys |= ((u64)target << 32);
55 phys |= ((u64)memtype << 36);
56
57 return phys;
58}
59
60void
61nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
62 struct nouveau_vram *mem, u32 pte, u32 cnt, u64 phys)
63{
64 u32 next = 1 << (vma->node->type - 8);
65
66 phys = nvc0_vm_addr(vma, phys, mem->memtype, 0);
67 pte <<= 3;
68 while (cnt--) {
69 nv_wo32(pgt, pte + 0, lower_32_bits(phys));
70 nv_wo32(pgt, pte + 4, upper_32_bits(phys));
71 phys += next;
72 pte += 8;
73 }
74}
75
76void
77nvc0_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
78 u32 pte, dma_addr_t *list, u32 cnt)
79{
80 pte <<= 3;
81 while (cnt--) {
82 u64 phys = nvc0_vm_addr(vma, *list++, 0, 5);
83 nv_wo32(pgt, pte + 0, lower_32_bits(phys));
84 nv_wo32(pgt, pte + 4, upper_32_bits(phys));
85 pte += 8;
86 }
87}
88
89void
90nvc0_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
91{
92 pte <<= 3;
93 while (cnt--) {
94 nv_wo32(pgt, pte + 0, 0x00000000);
95 nv_wo32(pgt, pte + 4, 0x00000000);
96 pte += 8;
97 }
98}
99
100void
101nvc0_vm_flush(struct nouveau_vm *vm)
102{
103 struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
104 struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
105 struct drm_device *dev = vm->dev;
106 struct nouveau_vm_pgd *vpgd;
107 u32 r100c80, engine;
108
109 pinstmem->flush(vm->dev);
110
111 if (vm == dev_priv->chan_vm)
112 engine = 1;
113 else
114 engine = 5;
115
116 list_for_each_entry(vpgd, &vm->pgd_list, head) {
117 r100c80 = nv_rd32(dev, 0x100c80);
118 nv_wr32(dev, 0x100cb8, vpgd->obj->vinst >> 8);
119 nv_wr32(dev, 0x100cbc, 0x80000000 | engine);
120 if (!nv_wait(dev, 0x100c80, 0xffffffff, r100c80))
121 NV_ERROR(dev, "vm flush timeout eng %d\n", engine);
122 }
123}
diff --git a/drivers/gpu/drm/nouveau/nvc0_vram.c b/drivers/gpu/drm/nouveau/nvc0_vram.c
new file mode 100644
index 000000000000..858eda5dedd1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvc0_vram.c
@@ -0,0 +1,99 @@
1/*
2 * Copyright 2010 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Ben Skeggs
23 */
24
25#include "drmP.h"
26#include "nouveau_drv.h"
27#include "nouveau_mm.h"
28
29bool
30nvc0_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
31{
32 switch (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) {
33 case 0x0000:
34 case 0xfe00:
35 case 0xdb00:
36 case 0x1100:
37 return true;
38 default:
39 break;
40 }
41
42 return false;
43}
44
45int
46nvc0_vram_new(struct drm_device *dev, u64 size, u32 align, u32 ncmin,
47 u32 type, struct nouveau_vram **pvram)
48{
49 struct drm_nouveau_private *dev_priv = dev->dev_private;
50 struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
51 struct ttm_mem_type_manager *man = &bdev->man[TTM_PL_VRAM];
52 struct nouveau_mm *mm = man->priv;
53 struct nouveau_mm_node *r;
54 struct nouveau_vram *vram;
55 int ret;
56
57 size >>= 12;
58 align >>= 12;
59 ncmin >>= 12;
60
61 vram = kzalloc(sizeof(*vram), GFP_KERNEL);
62 if (!vram)
63 return -ENOMEM;
64
65 INIT_LIST_HEAD(&vram->regions);
66 vram->dev = dev_priv->dev;
67 vram->memtype = type;
68 vram->size = size;
69
70 mutex_lock(&mm->mutex);
71 do {
72 ret = nouveau_mm_get(mm, 1, size, ncmin, align, &r);
73 if (ret) {
74 mutex_unlock(&mm->mutex);
75 nv50_vram_del(dev, &vram);
76 return ret;
77 }
78
79 list_add_tail(&r->rl_entry, &vram->regions);
80 size -= r->length;
81 } while (size);
82 mutex_unlock(&mm->mutex);
83
84 r = list_first_entry(&vram->regions, struct nouveau_mm_node, rl_entry);
85 vram->offset = (u64)r->offset << 12;
86 *pvram = vram;
87 return 0;
88}
89
90int
91nvc0_vram_init(struct drm_device *dev)
92{
93 struct drm_nouveau_private *dev_priv = dev->dev_private;
94
95 dev_priv->vram_size = nv_rd32(dev, 0x10f20c) << 20;
96 dev_priv->vram_size *= nv_rd32(dev, 0x121c74);
97 dev_priv->vram_rblock_size = 4096;
98 return 0;
99}
diff --git a/drivers/gpu/drm/nouveau/nvreg.h b/drivers/gpu/drm/nouveau/nvreg.h
index 881f8a585613..fe0f253089ac 100644
--- a/drivers/gpu/drm/nouveau/nvreg.h
+++ b/drivers/gpu/drm/nouveau/nvreg.h
@@ -153,7 +153,8 @@
153#define NV_PCRTC_START 0x00600800 153#define NV_PCRTC_START 0x00600800
154#define NV_PCRTC_CONFIG 0x00600804 154#define NV_PCRTC_CONFIG 0x00600804
155# define NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA (1 << 0) 155# define NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA (1 << 0)
156# define NV_PCRTC_CONFIG_START_ADDRESS_HSYNC (2 << 0) 156# define NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC (4 << 0)
157# define NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC (2 << 0)
157#define NV_PCRTC_CURSOR_CONFIG 0x00600810 158#define NV_PCRTC_CURSOR_CONFIG 0x00600810
158# define NV_PCRTC_CURSOR_CONFIG_ENABLE_ENABLE (1 << 0) 159# define NV_PCRTC_CURSOR_CONFIG_ENABLE_ENABLE (1 << 0)
159# define NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE (1 << 4) 160# define NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE (1 << 4)
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index 6cae4f2028d2..e47eecfc2df4 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -65,10 +65,13 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
65 rs400.o rs600.o rs690.o rv515.o r520.o r600.o rv770.o radeon_test.o \ 65 rs400.o rs600.o rs690.o rv515.o r520.o r600.o rv770.o radeon_test.o \
66 r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \ 66 r200.o radeon_legacy_tv.o r600_cs.o r600_blit.o r600_blit_shaders.o \
67 r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \ 67 r600_blit_kms.o radeon_pm.o atombios_dp.o r600_audio.o r600_hdmi.o \
68 evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o 68 evergreen.o evergreen_cs.o evergreen_blit_shaders.o evergreen_blit_kms.o \
69 radeon_trace_points.o ni.o
69 70
70radeon-$(CONFIG_COMPAT) += radeon_ioc32.o 71radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
71radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o 72radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
72radeon-$(CONFIG_ACPI) += radeon_acpi.o 73radeon-$(CONFIG_ACPI) += radeon_acpi.o
73 74
74obj-$(CONFIG_DRM_RADEON)+= radeon.o 75obj-$(CONFIG_DRM_RADEON)+= radeon.o
76
77CFLAGS_radeon_trace_points.o := -I$(src) \ No newline at end of file
diff --git a/drivers/gpu/drm/radeon/ObjectID.h b/drivers/gpu/drm/radeon/ObjectID.h
index c714179d1bfa..c61c3fe9fb98 100644
--- a/drivers/gpu/drm/radeon/ObjectID.h
+++ b/drivers/gpu/drm/radeon/ObjectID.h
@@ -37,6 +37,8 @@
37#define GRAPH_OBJECT_TYPE_CONNECTOR 0x3 37#define GRAPH_OBJECT_TYPE_CONNECTOR 0x3
38#define GRAPH_OBJECT_TYPE_ROUTER 0x4 38#define GRAPH_OBJECT_TYPE_ROUTER 0x4
39/* deleted */ 39/* deleted */
40#define GRAPH_OBJECT_TYPE_DISPLAY_PATH 0x6
41#define GRAPH_OBJECT_TYPE_GENERIC 0x7
40 42
41/****************************************************/ 43/****************************************************/
42/* Encoder Object ID Definition */ 44/* Encoder Object ID Definition */
@@ -64,6 +66,9 @@
64#define ENCODER_OBJECT_ID_VT1623 0x10 66#define ENCODER_OBJECT_ID_VT1623 0x10
65#define ENCODER_OBJECT_ID_HDMI_SI1930 0x11 67#define ENCODER_OBJECT_ID_HDMI_SI1930 0x11
66#define ENCODER_OBJECT_ID_HDMI_INTERNAL 0x12 68#define ENCODER_OBJECT_ID_HDMI_INTERNAL 0x12
69#define ENCODER_OBJECT_ID_ALMOND 0x22
70#define ENCODER_OBJECT_ID_TRAVIS 0x23
71#define ENCODER_OBJECT_ID_NUTMEG 0x22
67/* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */ 72/* Kaleidoscope (KLDSCP) Class Display Hardware (internal) */
68#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 0x13 73#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1 0x13
69#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1 0x14 74#define ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1 0x14
@@ -108,6 +113,7 @@
108#define CONNECTOR_OBJECT_ID_DISPLAYPORT 0x13 113#define CONNECTOR_OBJECT_ID_DISPLAYPORT 0x13
109#define CONNECTOR_OBJECT_ID_eDP 0x14 114#define CONNECTOR_OBJECT_ID_eDP 0x14
110#define CONNECTOR_OBJECT_ID_MXM 0x15 115#define CONNECTOR_OBJECT_ID_MXM 0x15
116#define CONNECTOR_OBJECT_ID_LVDS_eDP 0x16
111 117
112/* deleted */ 118/* deleted */
113 119
@@ -124,6 +130,7 @@
124#define GENERIC_OBJECT_ID_GLSYNC 0x01 130#define GENERIC_OBJECT_ID_GLSYNC 0x01
125#define GENERIC_OBJECT_ID_PX2_NON_DRIVABLE 0x02 131#define GENERIC_OBJECT_ID_PX2_NON_DRIVABLE 0x02
126#define GENERIC_OBJECT_ID_MXM_OPM 0x03 132#define GENERIC_OBJECT_ID_MXM_OPM 0x03
133#define GENERIC_OBJECT_ID_STEREO_PIN 0x04 //This object could show up from Misc Object table, it follows ATOM_OBJECT format, and contains one ATOM_OBJECT_GPIO_CNTL_RECORD for the stereo pin
127 134
128/****************************************************/ 135/****************************************************/
129/* Graphics Object ENUM ID Definition */ 136/* Graphics Object ENUM ID Definition */
@@ -360,6 +367,26 @@
360 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ 367 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
361 ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT) 368 ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO << OBJECT_ID_SHIFT)
362 369
370#define ENCODER_ALMOND_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
371 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
372 ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT)
373
374#define ENCODER_ALMOND_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
375 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
376 ENCODER_OBJECT_ID_ALMOND << OBJECT_ID_SHIFT)
377
378#define ENCODER_TRAVIS_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
379 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
380 ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT)
381
382#define ENCODER_TRAVIS_ENUM_ID2 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
383 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
384 ENCODER_OBJECT_ID_TRAVIS << OBJECT_ID_SHIFT)
385
386#define ENCODER_NUTMEG_ENUM_ID1 ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
387 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
388 ENCODER_OBJECT_ID_NUTMEG << OBJECT_ID_SHIFT)
389
363/****************************************************/ 390/****************************************************/
364/* Connector Object ID definition - Shared with BIOS */ 391/* Connector Object ID definition - Shared with BIOS */
365/****************************************************/ 392/****************************************************/
@@ -421,6 +448,14 @@
421 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ 448 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
422 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT) 449 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
423 450
451#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID3 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
452 GRAPH_OBJECT_ENUM_ID3 << ENUM_ID_SHIFT |\
453 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
454
455#define CONNECTOR_SINGLE_LINK_DVI_D_ENUM_ID4 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
456 GRAPH_OBJECT_ENUM_ID4 << ENUM_ID_SHIFT |\
457 CONNECTOR_OBJECT_ID_SINGLE_LINK_DVI_D << OBJECT_ID_SHIFT)
458
424#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ 459#define CONNECTOR_DUAL_LINK_DVI_D_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
425 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ 460 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
426 CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT) 461 CONNECTOR_OBJECT_ID_DUAL_LINK_DVI_D << OBJECT_ID_SHIFT)
@@ -512,6 +547,7 @@
512#define CONNECTOR_7PIN_DIN_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ 547#define CONNECTOR_7PIN_DIN_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
513 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ 548 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
514 CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT) 549 CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT)
550
515#define CONNECTOR_7PIN_DIN_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\ 551#define CONNECTOR_7PIN_DIN_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
516 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\ 552 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
517 CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT) 553 CONNECTOR_OBJECT_ID_7PIN_DIN << OBJECT_ID_SHIFT)
@@ -593,6 +629,14 @@
593 GRAPH_OBJECT_ENUM_ID7 << ENUM_ID_SHIFT |\ 629 GRAPH_OBJECT_ENUM_ID7 << ENUM_ID_SHIFT |\
594 CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DAC 630 CONNECTOR_OBJECT_ID_MXM << OBJECT_ID_SHIFT) //Mapping to MXM_DAC
595 631
632#define CONNECTOR_LVDS_eDP_ENUM_ID1 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
633 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
634 CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT)
635
636#define CONNECTOR_LVDS_eDP_ENUM_ID2 ( GRAPH_OBJECT_TYPE_CONNECTOR << OBJECT_TYPE_SHIFT |\
637 GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
638 CONNECTOR_OBJECT_ID_LVDS_eDP << OBJECT_ID_SHIFT)
639
596/****************************************************/ 640/****************************************************/
597/* Router Object ID definition - Shared with BIOS */ 641/* Router Object ID definition - Shared with BIOS */
598/****************************************************/ 642/****************************************************/
@@ -621,6 +665,10 @@
621 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\ 665 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
622 GENERIC_OBJECT_ID_MXM_OPM << OBJECT_ID_SHIFT) 666 GENERIC_OBJECT_ID_MXM_OPM << OBJECT_ID_SHIFT)
623 667
668#define GENERICOBJECT_STEREO_PIN_ENUM_ID1 (GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
669 GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
670 GENERIC_OBJECT_ID_STEREO_PIN << OBJECT_ID_SHIFT)
671
624/****************************************************/ 672/****************************************************/
625/* Object Cap definition - Shared with BIOS */ 673/* Object Cap definition - Shared with BIOS */
626/****************************************************/ 674/****************************************************/
diff --git a/drivers/gpu/drm/radeon/atom.c b/drivers/gpu/drm/radeon/atom.c
index 05efb5b9f13e..258fa5e7a2d9 100644
--- a/drivers/gpu/drm/radeon/atom.c
+++ b/drivers/gpu/drm/radeon/atom.c
@@ -734,16 +734,16 @@ static void atom_op_jump(atom_exec_context *ctx, int *ptr, int arg)
734static void atom_op_mask(atom_exec_context *ctx, int *ptr, int arg) 734static void atom_op_mask(atom_exec_context *ctx, int *ptr, int arg)
735{ 735{
736 uint8_t attr = U8((*ptr)++); 736 uint8_t attr = U8((*ptr)++);
737 uint32_t dst, src1, src2, saved; 737 uint32_t dst, mask, src, saved;
738 int dptr = *ptr; 738 int dptr = *ptr;
739 SDEBUG(" dst: "); 739 SDEBUG(" dst: ");
740 dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); 740 dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1);
741 SDEBUG(" src1: "); 741 mask = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr);
742 src1 = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr); 742 SDEBUG(" mask: 0x%08x", mask);
743 SDEBUG(" src2: "); 743 SDEBUG(" src: ");
744 src2 = atom_get_src(ctx, attr, ptr); 744 src = atom_get_src(ctx, attr, ptr);
745 dst &= src1; 745 dst &= mask;
746 dst |= src2; 746 dst |= src;
747 SDEBUG(" dst: "); 747 SDEBUG(" dst: ");
748 atom_put_dst(ctx, arg, attr, &dptr, dst, saved); 748 atom_put_dst(ctx, arg, attr, &dptr, dst, saved);
749} 749}
diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h
index fe359a239df3..58a0cd02c0a2 100644
--- a/drivers/gpu/drm/radeon/atombios.h
+++ b/drivers/gpu/drm/radeon/atombios.h
@@ -73,8 +73,18 @@
73#define ATOM_PPLL1 0 73#define ATOM_PPLL1 0
74#define ATOM_PPLL2 1 74#define ATOM_PPLL2 1
75#define ATOM_DCPLL 2 75#define ATOM_DCPLL 2
76#define ATOM_PPLL0 2
77#define ATOM_EXT_PLL1 8
78#define ATOM_EXT_PLL2 9
79#define ATOM_EXT_CLOCK 10
76#define ATOM_PPLL_INVALID 0xFF 80#define ATOM_PPLL_INVALID 0xFF
77 81
82#define ENCODER_REFCLK_SRC_P1PLL 0
83#define ENCODER_REFCLK_SRC_P2PLL 1
84#define ENCODER_REFCLK_SRC_DCPLL 2
85#define ENCODER_REFCLK_SRC_EXTCLK 3
86#define ENCODER_REFCLK_SRC_INVALID 0xFF
87
78#define ATOM_SCALER1 0 88#define ATOM_SCALER1 0
79#define ATOM_SCALER2 1 89#define ATOM_SCALER2 1
80 90
@@ -192,6 +202,9 @@ typedef struct _ATOM_COMMON_TABLE_HEADER
192 /*Image can't be updated, while Driver needs to carry the new table! */ 202 /*Image can't be updated, while Driver needs to carry the new table! */
193}ATOM_COMMON_TABLE_HEADER; 203}ATOM_COMMON_TABLE_HEADER;
194 204
205/****************************************************************************/
206// Structure stores the ROM header.
207/****************************************************************************/
195typedef struct _ATOM_ROM_HEADER 208typedef struct _ATOM_ROM_HEADER
196{ 209{
197 ATOM_COMMON_TABLE_HEADER sHeader; 210 ATOM_COMMON_TABLE_HEADER sHeader;
@@ -221,6 +234,9 @@ typedef struct _ATOM_ROM_HEADER
221 #define USHORT void* 234 #define USHORT void*
222#endif 235#endif
223 236
237/****************************************************************************/
238// Structures used in Command.mtb
239/****************************************************************************/
224typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{ 240typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
225 USHORT ASIC_Init; //Function Table, used by various SW components,latest version 1.1 241 USHORT ASIC_Init; //Function Table, used by various SW components,latest version 1.1
226 USHORT GetDisplaySurfaceSize; //Atomic Table, Used by Bios when enabling HW ICON 242 USHORT GetDisplaySurfaceSize; //Atomic Table, Used by Bios when enabling HW ICON
@@ -312,6 +328,7 @@ typedef struct _ATOM_MASTER_LIST_OF_COMMAND_TABLES{
312#define SetUniphyInstance ASIC_StaticPwrMgtStatusChange 328#define SetUniphyInstance ASIC_StaticPwrMgtStatusChange
313#define HPDInterruptService ReadHWAssistedI2CStatus 329#define HPDInterruptService ReadHWAssistedI2CStatus
314#define EnableVGA_Access GetSCLKOverMCLKRatio 330#define EnableVGA_Access GetSCLKOverMCLKRatio
331#define GetDispObjectInfo EnableYUV
315 332
316typedef struct _ATOM_MASTER_COMMAND_TABLE 333typedef struct _ATOM_MASTER_COMMAND_TABLE
317{ 334{
@@ -357,6 +374,24 @@ typedef struct _ATOM_COMMON_ROM_COMMAND_TABLE_HEADER
357/****************************************************************************/ 374/****************************************************************************/
358#define COMPUTE_MEMORY_PLL_PARAM 1 375#define COMPUTE_MEMORY_PLL_PARAM 1
359#define COMPUTE_ENGINE_PLL_PARAM 2 376#define COMPUTE_ENGINE_PLL_PARAM 2
377#define ADJUST_MC_SETTING_PARAM 3
378
379/****************************************************************************/
380// Structures used by AdjustMemoryControllerTable
381/****************************************************************************/
382typedef struct _ATOM_ADJUST_MEMORY_CLOCK_FREQ
383{
384#if ATOM_BIG_ENDIAN
385 ULONG ulPointerReturnFlag:1; // BYTE_3[7]=1 - Return the pointer to the right Data Block; BYTE_3[7]=0 - Program the right Data Block
386 ULONG ulMemoryModuleNumber:7; // BYTE_3[6:0]
387 ULONG ulClockFreq:24;
388#else
389 ULONG ulClockFreq:24;
390 ULONG ulMemoryModuleNumber:7; // BYTE_3[6:0]
391 ULONG ulPointerReturnFlag:1; // BYTE_3[7]=1 - Return the pointer to the right Data Block; BYTE_3[7]=0 - Program the right Data Block
392#endif
393}ATOM_ADJUST_MEMORY_CLOCK_FREQ;
394#define POINTER_RETURN_FLAG 0x80
360 395
361typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS 396typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS
362{ 397{
@@ -440,6 +475,26 @@ typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4
440#endif 475#endif
441}COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4; 476}COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4;
442 477
478typedef struct _COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5
479{
480 union
481 {
482 ATOM_COMPUTE_CLOCK_FREQ ulClock; //Input Parameter
483 ATOM_S_MPLL_FB_DIVIDER ulFbDiv; //Output Parameter
484 };
485 UCHAR ucRefDiv; //Output Parameter
486 UCHAR ucPostDiv; //Output Parameter
487 union
488 {
489 UCHAR ucCntlFlag; //Output Flags
490 UCHAR ucInputFlag; //Input Flags. ucInputFlag[0] - Strobe(1)/Performance(0) mode
491 };
492 UCHAR ucReserved;
493}COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V5;
494
495// ucInputFlag
496#define ATOM_PLL_INPUT_FLAG_PLL_STROBE_MODE_EN 1 // 1-StrobeMode, 0-PerformanceMode
497
443typedef struct _DYNAMICE_MEMORY_SETTINGS_PARAMETER 498typedef struct _DYNAMICE_MEMORY_SETTINGS_PARAMETER
444{ 499{
445 ATOM_COMPUTE_CLOCK_FREQ ulClock; 500 ATOM_COMPUTE_CLOCK_FREQ ulClock;
@@ -583,6 +638,7 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS
583#define ATOM_ENCODER_CONFIG_DPLINKRATE_MASK 0x01 638#define ATOM_ENCODER_CONFIG_DPLINKRATE_MASK 0x01
584#define ATOM_ENCODER_CONFIG_DPLINKRATE_1_62GHZ 0x00 639#define ATOM_ENCODER_CONFIG_DPLINKRATE_1_62GHZ 0x00
585#define ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ 0x01 640#define ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ 0x01
641#define ATOM_ENCODER_CONFIG_DPLINKRATE_5_40GHZ 0x02
586#define ATOM_ENCODER_CONFIG_LINK_SEL_MASK 0x04 642#define ATOM_ENCODER_CONFIG_LINK_SEL_MASK 0x04
587#define ATOM_ENCODER_CONFIG_LINKA 0x00 643#define ATOM_ENCODER_CONFIG_LINKA 0x00
588#define ATOM_ENCODER_CONFIG_LINKB 0x04 644#define ATOM_ENCODER_CONFIG_LINKB 0x04
@@ -608,6 +664,9 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS
608#define ATOM_ENCODER_MODE_TV 13 664#define ATOM_ENCODER_MODE_TV 13
609#define ATOM_ENCODER_MODE_CV 14 665#define ATOM_ENCODER_MODE_CV 14
610#define ATOM_ENCODER_MODE_CRT 15 666#define ATOM_ENCODER_MODE_CRT 15
667#define ATOM_ENCODER_MODE_DVO 16
668#define ATOM_ENCODER_MODE_DP_SST ATOM_ENCODER_MODE_DP // For DP1.2
669#define ATOM_ENCODER_MODE_DP_MST 5 // For DP1.2
611 670
612typedef struct _ATOM_DIG_ENCODER_CONFIG_V2 671typedef struct _ATOM_DIG_ENCODER_CONFIG_V2
613{ 672{
@@ -661,6 +720,7 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V2
661#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_START 0x08 720#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_START 0x08
662#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1 0x09 721#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN1 0x09
663#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2 0x0a 722#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN2 0x0a
723#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_PATTERN3 0x13
664#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE 0x0b 724#define ATOM_ENCODER_CMD_DP_LINK_TRAINING_COMPLETE 0x0b
665#define ATOM_ENCODER_CMD_DP_VIDEO_OFF 0x0c 725#define ATOM_ENCODER_CMD_DP_VIDEO_OFF 0x0c
666#define ATOM_ENCODER_CMD_DP_VIDEO_ON 0x0d 726#define ATOM_ENCODER_CMD_DP_VIDEO_ON 0x0d
@@ -671,24 +731,34 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V2
671#define ATOM_ENCODER_STATUS_LINK_TRAINING_COMPLETE 0x10 731#define ATOM_ENCODER_STATUS_LINK_TRAINING_COMPLETE 0x10
672#define ATOM_ENCODER_STATUS_LINK_TRAINING_INCOMPLETE 0x00 732#define ATOM_ENCODER_STATUS_LINK_TRAINING_INCOMPLETE 0x00
673 733
734//ucTableFormatRevision=1
735//ucTableContentRevision=3
674// Following function ENABLE sub-function will be used by driver when TMDS/HDMI/LVDS is used, disable function will be used by driver 736// Following function ENABLE sub-function will be used by driver when TMDS/HDMI/LVDS is used, disable function will be used by driver
675typedef struct _ATOM_DIG_ENCODER_CONFIG_V3 737typedef struct _ATOM_DIG_ENCODER_CONFIG_V3
676{ 738{
677#if ATOM_BIG_ENDIAN 739#if ATOM_BIG_ENDIAN
678 UCHAR ucReserved1:1; 740 UCHAR ucReserved1:1;
679 UCHAR ucDigSel:3; // =0: DIGA/B/C/D/E/F 741 UCHAR ucDigSel:3; // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also refered as DIGA/B/C/D/E/F)
680 UCHAR ucReserved:3; 742 UCHAR ucReserved:3;
681 UCHAR ucDPLinkRate:1; // =0: 1.62Ghz, =1: 2.7Ghz 743 UCHAR ucDPLinkRate:1; // =0: 1.62Ghz, =1: 2.7Ghz
682#else 744#else
683 UCHAR ucDPLinkRate:1; // =0: 1.62Ghz, =1: 2.7Ghz 745 UCHAR ucDPLinkRate:1; // =0: 1.62Ghz, =1: 2.7Ghz
684 UCHAR ucReserved:3; 746 UCHAR ucReserved:3;
685 UCHAR ucDigSel:3; // =0: DIGA/B/C/D/E/F 747 UCHAR ucDigSel:3; // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also refered as DIGA/B/C/D/E/F)
686 UCHAR ucReserved1:1; 748 UCHAR ucReserved1:1;
687#endif 749#endif
688}ATOM_DIG_ENCODER_CONFIG_V3; 750}ATOM_DIG_ENCODER_CONFIG_V3;
689 751
752#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_MASK 0x03
753#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_1_62GHZ 0x00
754#define ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ 0x01
690#define ATOM_ENCODER_CONFIG_V3_ENCODER_SEL 0x70 755#define ATOM_ENCODER_CONFIG_V3_ENCODER_SEL 0x70
691 756#define ATOM_ENCODER_CONFIG_V3_DIG0_ENCODER 0x00
757#define ATOM_ENCODER_CONFIG_V3_DIG1_ENCODER 0x10
758#define ATOM_ENCODER_CONFIG_V3_DIG2_ENCODER 0x20
759#define ATOM_ENCODER_CONFIG_V3_DIG3_ENCODER 0x30
760#define ATOM_ENCODER_CONFIG_V3_DIG4_ENCODER 0x40
761#define ATOM_ENCODER_CONFIG_V3_DIG5_ENCODER 0x50
692 762
693typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V3 763typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V3
694{ 764{
@@ -707,6 +777,56 @@ typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V3
707 UCHAR ucReserved; 777 UCHAR ucReserved;
708}DIG_ENCODER_CONTROL_PARAMETERS_V3; 778}DIG_ENCODER_CONTROL_PARAMETERS_V3;
709 779
780//ucTableFormatRevision=1
781//ucTableContentRevision=4
782// start from NI
783// Following function ENABLE sub-function will be used by driver when TMDS/HDMI/LVDS is used, disable function will be used by driver
784typedef struct _ATOM_DIG_ENCODER_CONFIG_V4
785{
786#if ATOM_BIG_ENDIAN
787 UCHAR ucReserved1:1;
788 UCHAR ucDigSel:3; // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also refered as DIGA/B/C/D/E/F)
789 UCHAR ucReserved:2;
790 UCHAR ucDPLinkRate:2; // =0: 1.62Ghz, =1: 2.7Ghz, 2=5.4Ghz <= Changed comparing to previous version
791#else
792 UCHAR ucDPLinkRate:2; // =0: 1.62Ghz, =1: 2.7Ghz, 2=5.4Ghz <= Changed comparing to previous version
793 UCHAR ucReserved:2;
794 UCHAR ucDigSel:3; // =0/1/2/3/4/5: DIG0/1/2/3/4/5 (In register spec also refered as DIGA/B/C/D/E/F)
795 UCHAR ucReserved1:1;
796#endif
797}ATOM_DIG_ENCODER_CONFIG_V4;
798
799#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_MASK 0x03
800#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_1_62GHZ 0x00
801#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ 0x01
802#define ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ 0x02
803#define ATOM_ENCODER_CONFIG_V4_ENCODER_SEL 0x70
804#define ATOM_ENCODER_CONFIG_V4_DIG0_ENCODER 0x00
805#define ATOM_ENCODER_CONFIG_V4_DIG1_ENCODER 0x10
806#define ATOM_ENCODER_CONFIG_V4_DIG2_ENCODER 0x20
807#define ATOM_ENCODER_CONFIG_V4_DIG3_ENCODER 0x30
808#define ATOM_ENCODER_CONFIG_V4_DIG4_ENCODER 0x40
809#define ATOM_ENCODER_CONFIG_V4_DIG5_ENCODER 0x50
810
811typedef struct _DIG_ENCODER_CONTROL_PARAMETERS_V4
812{
813 USHORT usPixelClock; // in 10KHz; for bios convenient
814 union{
815 ATOM_DIG_ENCODER_CONFIG_V4 acConfig;
816 UCHAR ucConfig;
817 };
818 UCHAR ucAction;
819 UCHAR ucEncoderMode;
820 // =0: DP encoder
821 // =1: LVDS encoder
822 // =2: DVI encoder
823 // =3: HDMI encoder
824 // =4: SDVO encoder
825 // =5: DP audio
826 UCHAR ucLaneNum; // how many lanes to enable
827 UCHAR ucBitPerColor; // only valid for DP mode when ucAction = ATOM_ENCODER_CMD_SETUP
828 UCHAR ucHPD_ID; // HPD ID (1-6). =0 means to skip HDP programming. New comparing to previous version
829}DIG_ENCODER_CONTROL_PARAMETERS_V4;
710 830
711// define ucBitPerColor: 831// define ucBitPerColor:
712#define PANEL_BPC_UNDEFINE 0x00 832#define PANEL_BPC_UNDEFINE 0x00
@@ -893,6 +1013,7 @@ typedef struct _ATOM_DIG_TRANSMITTER_CONFIG_V3
893#endif 1013#endif
894}ATOM_DIG_TRANSMITTER_CONFIG_V3; 1014}ATOM_DIG_TRANSMITTER_CONFIG_V3;
895 1015
1016
896typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 1017typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V3
897{ 1018{
898 union 1019 union
@@ -936,6 +1057,149 @@ typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V3
936#define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER2 0x40 //CD 1057#define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER2 0x40 //CD
937#define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER3 0x80 //EF 1058#define ATOM_TRANSMITTER_CONFIG_V3_TRANSMITTER3 0x80 //EF
938 1059
1060
1061/****************************************************************************/
1062// Structures used by UNIPHYTransmitterControlTable V1.4
1063// ASIC Families: NI
1064// ucTableFormatRevision=1
1065// ucTableContentRevision=4
1066/****************************************************************************/
1067typedef struct _ATOM_DP_VS_MODE_V4
1068{
1069 UCHAR ucLaneSel;
1070 union
1071 {
1072 UCHAR ucLaneSet;
1073 struct {
1074#if ATOM_BIG_ENDIAN
1075 UCHAR ucPOST_CURSOR2:2; //Bit[7:6] Post Cursor2 Level <= New in V4
1076 UCHAR ucPRE_EMPHASIS:3; //Bit[5:3] Pre-emphasis Level
1077 UCHAR ucVOLTAGE_SWING:3; //Bit[2:0] Voltage Swing Level
1078#else
1079 UCHAR ucVOLTAGE_SWING:3; //Bit[2:0] Voltage Swing Level
1080 UCHAR ucPRE_EMPHASIS:3; //Bit[5:3] Pre-emphasis Level
1081 UCHAR ucPOST_CURSOR2:2; //Bit[7:6] Post Cursor2 Level <= New in V4
1082#endif
1083 };
1084 };
1085}ATOM_DP_VS_MODE_V4;
1086
1087typedef struct _ATOM_DIG_TRANSMITTER_CONFIG_V4
1088{
1089#if ATOM_BIG_ENDIAN
1090 UCHAR ucTransmitterSel:2; //bit7:6: =0 Dig Transmitter 1 ( Uniphy AB )
1091 // =1 Dig Transmitter 2 ( Uniphy CD )
1092 // =2 Dig Transmitter 3 ( Uniphy EF )
1093 UCHAR ucRefClkSource:2; //bit5:4: PPLL1 =0, PPLL2=1, DCPLL=2, EXT_CLK=3 <= New
1094 UCHAR ucEncoderSel:1; //bit3=0: Data/Clk path source from DIGA/C/E. =1: Data/clk path source from DIGB/D/F
1095 UCHAR ucLinkSel:1; //bit2=0: Uniphy LINKA or C or E when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is A or C or E
1096 // =1: Uniphy LINKB or D or F when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is B or D or F
1097 UCHAR fCoherentMode:1; //bit1=1: Coherent Mode ( for DVI/HDMI mode )
1098 UCHAR fDualLinkConnector:1; //bit0=1: Dual Link DVI connector
1099#else
1100 UCHAR fDualLinkConnector:1; //bit0=1: Dual Link DVI connector
1101 UCHAR fCoherentMode:1; //bit1=1: Coherent Mode ( for DVI/HDMI mode )
1102 UCHAR ucLinkSel:1; //bit2=0: Uniphy LINKA or C or E when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is A or C or E
1103 // =1: Uniphy LINKB or D or F when fDualLinkConnector=0. when fDualLinkConnector=1, it means master link of dual link is B or D or F
1104 UCHAR ucEncoderSel:1; //bit3=0: Data/Clk path source from DIGA/C/E. =1: Data/clk path source from DIGB/D/F
1105 UCHAR ucRefClkSource:2; //bit5:4: PPLL1 =0, PPLL2=1, DCPLL=2, EXT_CLK=3 <= New
1106 UCHAR ucTransmitterSel:2; //bit7:6: =0 Dig Transmitter 1 ( Uniphy AB )
1107 // =1 Dig Transmitter 2 ( Uniphy CD )
1108 // =2 Dig Transmitter 3 ( Uniphy EF )
1109#endif
1110}ATOM_DIG_TRANSMITTER_CONFIG_V4;
1111
1112typedef struct _DIG_TRANSMITTER_CONTROL_PARAMETERS_V4
1113{
1114 union
1115 {
1116 USHORT usPixelClock; // in 10KHz; for bios convenient
1117 USHORT usInitInfo; // when init uniphy,lower 8bit is used for connector type defined in objectid.h
1118 ATOM_DP_VS_MODE_V4 asMode; // DP Voltage swing mode Redefined comparing to previous version
1119 };
1120 union
1121 {
1122 ATOM_DIG_TRANSMITTER_CONFIG_V4 acConfig;
1123 UCHAR ucConfig;
1124 };
1125 UCHAR ucAction; // define as ATOM_TRANSMITER_ACTION_XXX
1126 UCHAR ucLaneNum;
1127 UCHAR ucReserved[3];
1128}DIG_TRANSMITTER_CONTROL_PARAMETERS_V4;
1129
1130//ucConfig
1131//Bit0
1132#define ATOM_TRANSMITTER_CONFIG_V4_DUAL_LINK_CONNECTOR 0x01
1133//Bit1
1134#define ATOM_TRANSMITTER_CONFIG_V4_COHERENT 0x02
1135//Bit2
1136#define ATOM_TRANSMITTER_CONFIG_V4_LINK_SEL_MASK 0x04
1137#define ATOM_TRANSMITTER_CONFIG_V4_LINKA 0x00
1138#define ATOM_TRANSMITTER_CONFIG_V4_LINKB 0x04
1139// Bit3
1140#define ATOM_TRANSMITTER_CONFIG_V4_ENCODER_SEL_MASK 0x08
1141#define ATOM_TRANSMITTER_CONFIG_V4_DIG1_ENCODER 0x00
1142#define ATOM_TRANSMITTER_CONFIG_V4_DIG2_ENCODER 0x08
1143// Bit5:4
1144#define ATOM_TRANSMITTER_CONFIG_V4_REFCLK_SEL_MASK 0x30
1145#define ATOM_TRANSMITTER_CONFIG_V4_P1PLL 0x00
1146#define ATOM_TRANSMITTER_CONFIG_V4_P2PLL 0x10
1147#define ATOM_TRANSMITTER_CONFIG_V4_DCPLL 0x20 // New in _V4
1148#define ATOM_TRANSMITTER_CONFIG_V4_REFCLK_SRC_EXT 0x30 // Changed comparing to V3
1149// Bit7:6
1150#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER_SEL_MASK 0xC0
1151#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER1 0x00 //AB
1152#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER2 0x40 //CD
1153#define ATOM_TRANSMITTER_CONFIG_V4_TRANSMITTER3 0x80 //EF
1154
1155
1156/****************************************************************************/
1157// Structures used by ExternalEncoderControlTable V1.3
1158// ASIC Families: Evergreen, Llano, NI
1159// ucTableFormatRevision=1
1160// ucTableContentRevision=3
1161/****************************************************************************/
1162
1163typedef struct _EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3
1164{
1165 union{
1166 USHORT usPixelClock; // pixel clock in 10Khz, valid when ucAction=SETUP/ENABLE_OUTPUT
1167 USHORT usConnectorId; // connector id, valid when ucAction = INIT
1168 };
1169 UCHAR ucConfig; // indicate which encoder, and DP link rate when ucAction = SETUP/ENABLE_OUTPUT
1170 UCHAR ucAction; //
1171 UCHAR ucEncoderMode; // encoder mode, only used when ucAction = SETUP/ENABLE_OUTPUT
1172 UCHAR ucLaneNum; // lane number, only used when ucAction = SETUP/ENABLE_OUTPUT
1173 UCHAR ucBitPerColor; // output bit per color, only valid when ucAction = SETUP/ENABLE_OUTPUT and ucEncodeMode= DP
1174 UCHAR ucReserved;
1175}EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3;
1176
1177// ucAction
1178#define EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT 0x00
1179#define EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT 0x01
1180#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT 0x07
1181#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP 0x0f
1182#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING_OFF 0x10
1183#define EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING 0x11
1184#define EXTERNAL_ENCODER_ACTION_V3_DACLOAD_DETECTION 0x12
1185
1186// ucConfig
1187#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_MASK 0x03
1188#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_1_62GHZ 0x00
1189#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ 0x01
1190#define EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_5_40GHZ 0x02
1191#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER_SEL_MASK 0x70
1192#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER1 0x00
1193#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER2 0x10
1194#define EXTERNAL_ENCODER_CONFIG_V3_ENCODER3 0x20
1195
1196typedef struct _EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3
1197{
1198 EXTERNAL_ENCODER_CONTROL_PARAMETERS_V3 sExtEncoder;
1199 ULONG ulReserved[2];
1200}EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3;
1201
1202
939/****************************************************************************/ 1203/****************************************************************************/
940// Structures used by DAC1OuputControlTable 1204// Structures used by DAC1OuputControlTable
941// DAC2OuputControlTable 1205// DAC2OuputControlTable
@@ -1142,6 +1406,7 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V2
1142#define PIXEL_CLOCK_V4_MISC_SS_ENABLE 0x10 1406#define PIXEL_CLOCK_V4_MISC_SS_ENABLE 0x10
1143#define PIXEL_CLOCK_V4_MISC_COHERENT_MODE 0x20 1407#define PIXEL_CLOCK_V4_MISC_COHERENT_MODE 0x20
1144 1408
1409
1145typedef struct _PIXEL_CLOCK_PARAMETERS_V3 1410typedef struct _PIXEL_CLOCK_PARAMETERS_V3
1146{ 1411{
1147 USHORT usPixelClock; // in 10kHz unit; for bios convenient = (RefClk*FB_Div)/(Ref_Div*Post_Div) 1412 USHORT usPixelClock; // in 10kHz unit; for bios convenient = (RefClk*FB_Div)/(Ref_Div*Post_Div)
@@ -1202,6 +1467,55 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V5
1202#define PIXEL_CLOCK_V5_MISC_HDMI_32BPP 0x08 1467#define PIXEL_CLOCK_V5_MISC_HDMI_32BPP 0x08
1203#define PIXEL_CLOCK_V5_MISC_REF_DIV_SRC 0x10 1468#define PIXEL_CLOCK_V5_MISC_REF_DIV_SRC 0x10
1204 1469
1470typedef struct _CRTC_PIXEL_CLOCK_FREQ
1471{
1472#if ATOM_BIG_ENDIAN
1473 ULONG ucCRTC:8; // ATOM_CRTC1~6, indicate the CRTC controller to
1474 // drive the pixel clock. not used for DCPLL case.
1475 ULONG ulPixelClock:24; // target the pixel clock to drive the CRTC timing.
1476 // 0 means disable PPLL/DCPLL. Expanded to 24 bits comparing to previous version.
1477#else
1478 ULONG ulPixelClock:24; // target the pixel clock to drive the CRTC timing.
1479 // 0 means disable PPLL/DCPLL. Expanded to 24 bits comparing to previous version.
1480 ULONG ucCRTC:8; // ATOM_CRTC1~6, indicate the CRTC controller to
1481 // drive the pixel clock. not used for DCPLL case.
1482#endif
1483}CRTC_PIXEL_CLOCK_FREQ;
1484
1485typedef struct _PIXEL_CLOCK_PARAMETERS_V6
1486{
1487 union{
1488 CRTC_PIXEL_CLOCK_FREQ ulCrtcPclkFreq; // pixel clock and CRTC id frequency
1489 ULONG ulDispEngClkFreq; // dispclk frequency
1490 };
1491 USHORT usFbDiv; // feedback divider integer part.
1492 UCHAR ucPostDiv; // post divider.
1493 UCHAR ucRefDiv; // Reference divider
1494 UCHAR ucPpll; // ATOM_PPLL1/ATOM_PPLL2/ATOM_DCPLL
1495 UCHAR ucTransmitterID; // ASIC encoder id defined in objectId.h,
1496 // indicate which graphic encoder will be used.
1497 UCHAR ucEncoderMode; // Encoder mode:
1498 UCHAR ucMiscInfo; // bit[0]= Force program PPLL
1499 // bit[1]= when VGA timing is used.
1500 // bit[3:2]= HDMI panel bit depth: =0: 24bpp =1:30bpp, =2:32bpp
1501 // bit[4]= RefClock source for PPLL.
1502 // =0: XTLAIN( default mode )
1503 // =1: other external clock source, which is pre-defined
1504 // by VBIOS depend on the feature required.
1505 // bit[7:5]: reserved.
1506 ULONG ulFbDivDecFrac; // 20 bit feedback divider decimal fraction part, range from 1~999999 ( 0.000001 to 0.999999 )
1507
1508}PIXEL_CLOCK_PARAMETERS_V6;
1509
1510#define PIXEL_CLOCK_V6_MISC_FORCE_PROG_PPLL 0x01
1511#define PIXEL_CLOCK_V6_MISC_VGA_MODE 0x02
1512#define PIXEL_CLOCK_V6_MISC_HDMI_BPP_MASK 0x0c
1513#define PIXEL_CLOCK_V6_MISC_HDMI_24BPP 0x00
1514#define PIXEL_CLOCK_V6_MISC_HDMI_36BPP 0x04
1515#define PIXEL_CLOCK_V6_MISC_HDMI_30BPP 0x08
1516#define PIXEL_CLOCK_V6_MISC_HDMI_48BPP 0x0c
1517#define PIXEL_CLOCK_V6_MISC_REF_DIV_SRC 0x10
1518
1205typedef struct _GET_DISP_PLL_STATUS_INPUT_PARAMETERS_V2 1519typedef struct _GET_DISP_PLL_STATUS_INPUT_PARAMETERS_V2
1206{ 1520{
1207 PIXEL_CLOCK_PARAMETERS_V3 sDispClkInput; 1521 PIXEL_CLOCK_PARAMETERS_V3 sDispClkInput;
@@ -1241,10 +1555,11 @@ typedef struct _ADJUST_DISPLAY_PLL_PARAMETERS
1241typedef struct _ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3 1555typedef struct _ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3
1242{ 1556{
1243 USHORT usPixelClock; // target pixel clock 1557 USHORT usPixelClock; // target pixel clock
1244 UCHAR ucTransmitterID; // transmitter id defined in objectid.h 1558 UCHAR ucTransmitterID; // GPU transmitter id defined in objectid.h
1245 UCHAR ucEncodeMode; // encoder mode: CRT, LVDS, DP, TMDS or HDMI 1559 UCHAR ucEncodeMode; // encoder mode: CRT, LVDS, DP, TMDS or HDMI
1246 UCHAR ucDispPllConfig; // display pll configure parameter defined as following DISPPLL_CONFIG_XXXX 1560 UCHAR ucDispPllConfig; // display pll configure parameter defined as following DISPPLL_CONFIG_XXXX
1247 UCHAR ucReserved[3]; 1561 UCHAR ucExtTransmitterID; // external encoder id.
1562 UCHAR ucReserved[2];
1248}ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3; 1563}ADJUST_DISPLAY_PLL_INPUT_PARAMETERS_V3;
1249 1564
1250// usDispPllConfig v1.2 for RoadRunner 1565// usDispPllConfig v1.2 for RoadRunner
@@ -1358,6 +1673,7 @@ typedef struct _SET_UP_HW_I2C_DATA_PARAMETERS
1358/**************************************************************************/ 1673/**************************************************************************/
1359#define SPEED_FAN_CONTROL_PS_ALLOCATION WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS 1674#define SPEED_FAN_CONTROL_PS_ALLOCATION WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS
1360 1675
1676
1361/****************************************************************************/ 1677/****************************************************************************/
1362// Structures used by PowerConnectorDetectionTable 1678// Structures used by PowerConnectorDetectionTable
1363/****************************************************************************/ 1679/****************************************************************************/
@@ -1438,6 +1754,31 @@ typedef struct _ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2
1438#define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK 0x0F00 1754#define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK 0x0F00
1439#define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT 8 1755#define ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT 8
1440 1756
1757// Used by DCE5.0
1758 typedef struct _ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3
1759{
1760 USHORT usSpreadSpectrumAmountFrac; // SS_AMOUNT_DSFRAC New in DCE5.0
1761 UCHAR ucSpreadSpectrumType; // Bit[0]: 0-Down Spread,1-Center Spread.
1762 // Bit[1]: 1-Ext. 0-Int.
1763 // Bit[3:2]: =0 P1PLL =1 P2PLL =2 DCPLL
1764 // Bits[7:4] reserved
1765 UCHAR ucEnable; // ATOM_ENABLE or ATOM_DISABLE
1766 USHORT usSpreadSpectrumAmount; // Includes SS_AMOUNT_FBDIV[7:0] and SS_AMOUNT_NFRAC_SLIP[11:8]
1767 USHORT usSpreadSpectrumStep; // SS_STEP_SIZE_DSFRAC
1768}ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3;
1769
1770#define ATOM_PPLL_SS_TYPE_V3_DOWN_SPREAD 0x00
1771#define ATOM_PPLL_SS_TYPE_V3_CENTRE_SPREAD 0x01
1772#define ATOM_PPLL_SS_TYPE_V3_EXT_SPREAD 0x02
1773#define ATOM_PPLL_SS_TYPE_V3_PPLL_SEL_MASK 0x0c
1774#define ATOM_PPLL_SS_TYPE_V3_P1PLL 0x00
1775#define ATOM_PPLL_SS_TYPE_V3_P2PLL 0x04
1776#define ATOM_PPLL_SS_TYPE_V3_DCPLL 0x08
1777#define ATOM_PPLL_SS_AMOUNT_V3_FBDIV_MASK 0x00FF
1778#define ATOM_PPLL_SS_AMOUNT_V3_FBDIV_SHIFT 0
1779#define ATOM_PPLL_SS_AMOUNT_V3_NFRAC_MASK 0x0F00
1780#define ATOM_PPLL_SS_AMOUNT_V3_NFRAC_SHIFT 8
1781
1441#define ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION ENABLE_SPREAD_SPECTRUM_ON_PPLL 1782#define ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION ENABLE_SPREAD_SPECTRUM_ON_PPLL
1442 1783
1443/**************************************************************************/ 1784/**************************************************************************/
@@ -1706,7 +2047,7 @@ typedef struct _ATOM_MASTER_LIST_OF_DATA_TABLES
1706 USHORT StandardVESA_Timing; // Only used by Bios 2047 USHORT StandardVESA_Timing; // Only used by Bios
1707 USHORT FirmwareInfo; // Shared by various SW components,latest version 1.4 2048 USHORT FirmwareInfo; // Shared by various SW components,latest version 1.4
1708 USHORT DAC_Info; // Will be obsolete from R600 2049 USHORT DAC_Info; // Will be obsolete from R600
1709 USHORT LVDS_Info; // Shared by various SW components,latest version 1.1 2050 USHORT LCD_Info; // Shared by various SW components,latest version 1.3, was called LVDS_Info
1710 USHORT TMDS_Info; // Will be obsolete from R600 2051 USHORT TMDS_Info; // Will be obsolete from R600
1711 USHORT AnalogTV_Info; // Shared by various SW components,latest version 1.1 2052 USHORT AnalogTV_Info; // Shared by various SW components,latest version 1.1
1712 USHORT SupportedDevicesInfo; // Will be obsolete from R600 2053 USHORT SupportedDevicesInfo; // Will be obsolete from R600
@@ -1736,12 +2077,16 @@ typedef struct _ATOM_MASTER_LIST_OF_DATA_TABLES
1736 USHORT PowerSourceInfo; // Shared by various SW components, latest versoin 1.1 2077 USHORT PowerSourceInfo; // Shared by various SW components, latest versoin 1.1
1737}ATOM_MASTER_LIST_OF_DATA_TABLES; 2078}ATOM_MASTER_LIST_OF_DATA_TABLES;
1738 2079
2080// For backward compatible
2081#define LVDS_Info LCD_Info
2082
1739typedef struct _ATOM_MASTER_DATA_TABLE 2083typedef struct _ATOM_MASTER_DATA_TABLE
1740{ 2084{
1741 ATOM_COMMON_TABLE_HEADER sHeader; 2085 ATOM_COMMON_TABLE_HEADER sHeader;
1742 ATOM_MASTER_LIST_OF_DATA_TABLES ListOfDataTables; 2086 ATOM_MASTER_LIST_OF_DATA_TABLES ListOfDataTables;
1743}ATOM_MASTER_DATA_TABLE; 2087}ATOM_MASTER_DATA_TABLE;
1744 2088
2089
1745/****************************************************************************/ 2090/****************************************************************************/
1746// Structure used in MultimediaCapabilityInfoTable 2091// Structure used in MultimediaCapabilityInfoTable
1747/****************************************************************************/ 2092/****************************************************************************/
@@ -1776,6 +2121,7 @@ typedef struct _ATOM_MULTIMEDIA_CONFIG_INFO
1776 UCHAR ucVideoInput4Info;// Video Input 4 Type (1:0) F/B setting (2) physical connector ID (5:3) reserved (7:6) 2121 UCHAR ucVideoInput4Info;// Video Input 4 Type (1:0) F/B setting (2) physical connector ID (5:3) reserved (7:6)
1777}ATOM_MULTIMEDIA_CONFIG_INFO; 2122}ATOM_MULTIMEDIA_CONFIG_INFO;
1778 2123
2124
1779/****************************************************************************/ 2125/****************************************************************************/
1780// Structures used in FirmwareInfoTable 2126// Structures used in FirmwareInfoTable
1781/****************************************************************************/ 2127/****************************************************************************/
@@ -2031,8 +2377,47 @@ typedef struct _ATOM_FIRMWARE_INFO_V2_1
2031 UCHAR ucReserved4[3]; 2377 UCHAR ucReserved4[3];
2032}ATOM_FIRMWARE_INFO_V2_1; 2378}ATOM_FIRMWARE_INFO_V2_1;
2033 2379
2380//the structure below to be used from NI
2381//ucTableFormatRevision=2
2382//ucTableContentRevision=2
2383typedef struct _ATOM_FIRMWARE_INFO_V2_2
2384{
2385 ATOM_COMMON_TABLE_HEADER sHeader;
2386 ULONG ulFirmwareRevision;
2387 ULONG ulDefaultEngineClock; //In 10Khz unit
2388 ULONG ulDefaultMemoryClock; //In 10Khz unit
2389 ULONG ulReserved[2];
2390 ULONG ulReserved1; //Was ulMaxEngineClockPLL_Output; //In 10Khz unit*
2391 ULONG ulReserved2; //Was ulMaxMemoryClockPLL_Output; //In 10Khz unit*
2392 ULONG ulMaxPixelClockPLL_Output; //In 10Khz unit
2393 ULONG ulBinaryAlteredInfo; //Was ulASICMaxEngineClock ?
2394 ULONG ulDefaultDispEngineClkFreq; //In 10Khz unit. This is the frequency before DCDTO, corresponding to usBootUpVDDCVoltage.
2395 UCHAR ucReserved3; //Was ucASICMaxTemperature;
2396 UCHAR ucMinAllowedBL_Level;
2397 USHORT usBootUpVDDCVoltage; //In MV unit
2398 USHORT usLcdMinPixelClockPLL_Output; // In MHz unit
2399 USHORT usLcdMaxPixelClockPLL_Output; // In MHz unit
2400 ULONG ulReserved4; //Was ulAsicMaximumVoltage
2401 ULONG ulMinPixelClockPLL_Output; //In 10Khz unit
2402 ULONG ulReserved5; //Was usMinEngineClockPLL_Input and usMaxEngineClockPLL_Input
2403 ULONG ulReserved6; //Was usMinEngineClockPLL_Output and usMinMemoryClockPLL_Input
2404 ULONG ulReserved7; //Was usMaxMemoryClockPLL_Input and usMinMemoryClockPLL_Output
2405 USHORT usReserved11; //Was usMaxPixelClock; //In 10Khz unit, Max. Pclk used only for DAC
2406 USHORT usMinPixelClockPLL_Input; //In 10Khz unit
2407 USHORT usMaxPixelClockPLL_Input; //In 10Khz unit
2408 USHORT usBootUpVDDCIVoltage; //In unit of mv; Was usMinPixelClockPLL_Output;
2409 ATOM_FIRMWARE_CAPABILITY_ACCESS usFirmwareCapability;
2410 USHORT usCoreReferenceClock; //In 10Khz unit
2411 USHORT usMemoryReferenceClock; //In 10Khz unit
2412 USHORT usUniphyDPModeExtClkFreq; //In 10Khz unit, if it is 0, In DP Mode Uniphy Input clock from internal PPLL, otherwise Input clock from external Spread clock
2413 UCHAR ucMemoryModule_ID; //Indicate what is the board design
2414 UCHAR ucReserved9[3];
2415 USHORT usBootUpMVDDCVoltage; //In unit of mv; Was usMinPixelClockPLL_Output;
2416 USHORT usReserved12;
2417 ULONG ulReserved10[3]; // New added comparing to previous version
2418}ATOM_FIRMWARE_INFO_V2_2;
2034 2419
2035#define ATOM_FIRMWARE_INFO_LAST ATOM_FIRMWARE_INFO_V2_1 2420#define ATOM_FIRMWARE_INFO_LAST ATOM_FIRMWARE_INFO_V2_2
2036 2421
2037/****************************************************************************/ 2422/****************************************************************************/
2038// Structures used in IntegratedSystemInfoTable 2423// Structures used in IntegratedSystemInfoTable
@@ -2212,7 +2597,7 @@ ulDockingPinCFGInfo: [15:0]-Bus/Device/Function # to CFG to read this Docking Pi
2212ucDockingPinBit: which bit in this register to read the pin status; 2597ucDockingPinBit: which bit in this register to read the pin status;
2213ucDockingPinPolarity:Polarity of the pin when docked; 2598ucDockingPinPolarity:Polarity of the pin when docked;
2214 2599
2215ulCPUCapInfo: [7:0]=1:Griffin;[7:0]=2:Greyhound;[7:0]=3:K8, other bits reserved for now and must be 0x0 2600ulCPUCapInfo: [7:0]=1:Griffin;[7:0]=2:Greyhound;[7:0]=3:K8, [7:0]=4:Pharaoh, other bits reserved for now and must be 0x0
2216 2601
2217usNumberOfCyclesInPeriod:Indicate how many cycles when PWM duty is 100%. 2602usNumberOfCyclesInPeriod:Indicate how many cycles when PWM duty is 100%.
2218 2603
@@ -2250,6 +2635,14 @@ usMinUpStreamHTLinkWidth: Asymmetric link width support in the future, to rep
2250usMinDownStreamHTLinkWidth: same as above. 2635usMinDownStreamHTLinkWidth: same as above.
2251*/ 2636*/
2252 2637
2638// ATOM_INTEGRATED_SYSTEM_INFO::ulCPUCapInfo - CPU type definition
2639#define INTEGRATED_SYSTEM_INFO__UNKNOWN_CPU 0
2640#define INTEGRATED_SYSTEM_INFO__AMD_CPU__GRIFFIN 1
2641#define INTEGRATED_SYSTEM_INFO__AMD_CPU__GREYHOUND 2
2642#define INTEGRATED_SYSTEM_INFO__AMD_CPU__K8 3
2643#define INTEGRATED_SYSTEM_INFO__AMD_CPU__PHARAOH 4
2644
2645#define INTEGRATED_SYSTEM_INFO__AMD_CPU__MAX_CODE INTEGRATED_SYSTEM_INFO__AMD_CPU__PHARAOH // this deff reflects max defined CPU code
2253 2646
2254#define SYSTEM_CONFIG_POWEREXPRESS_ENABLE 0x00000001 2647#define SYSTEM_CONFIG_POWEREXPRESS_ENABLE 0x00000001
2255#define SYSTEM_CONFIG_RUN_AT_OVERDRIVE_ENGINE 0x00000002 2648#define SYSTEM_CONFIG_RUN_AT_OVERDRIVE_ENGINE 0x00000002
@@ -2778,8 +3171,88 @@ typedef struct _ATOM_LVDS_INFO_V12
2778#define PANEL_RANDOM_DITHER 0x80 3171#define PANEL_RANDOM_DITHER 0x80
2779#define PANEL_RANDOM_DITHER_MASK 0x80 3172#define PANEL_RANDOM_DITHER_MASK 0x80
2780 3173
3174#define ATOM_LVDS_INFO_LAST ATOM_LVDS_INFO_V12 // no need to change this
3175
3176/****************************************************************************/
3177// Structures used by LCD_InfoTable V1.3 Note: previous version was called ATOM_LVDS_INFO_V12
3178// ASIC Families: NI
3179// ucTableFormatRevision=1
3180// ucTableContentRevision=3
3181/****************************************************************************/
3182typedef struct _ATOM_LCD_INFO_V13
3183{
3184 ATOM_COMMON_TABLE_HEADER sHeader;
3185 ATOM_DTD_FORMAT sLCDTiming;
3186 USHORT usExtInfoTableOffset;
3187 USHORT usSupportedRefreshRate; //Refer to panel info table in ATOMBIOS extension Spec.
3188 ULONG ulReserved0;
3189 UCHAR ucLCD_Misc; // Reorganized in V13
3190 // Bit0: {=0:single, =1:dual},
3191 // Bit1: {=0:LDI format for RGB888, =1 FPDI format for RGB888} // was {=0:666RGB, =1:888RGB},
3192 // Bit3:2: {Grey level}
3193 // Bit6:4 Color Bit Depth definition (see below definition in EDID V1.4 @BYTE 14h)
3194 // Bit7 Reserved. was for ATOM_PANEL_MISC_API_ENABLED, still need it?
3195 UCHAR ucPanelDefaultRefreshRate;
3196 UCHAR ucPanelIdentification;
3197 UCHAR ucSS_Id;
3198 USHORT usLCDVenderID;
3199 USHORT usLCDProductID;
3200 UCHAR ucLCDPanel_SpecialHandlingCap; // Reorganized in V13
3201 // Bit0: Once DAL sees this CAP is set, it will read EDID from LCD on its own
3202 // Bit1: See LCDPANEL_CAP_DRR_SUPPORTED
3203 // Bit2: a quick reference whether an embadded panel (LCD1 ) is LVDS (0) or eDP (1)
3204 // Bit7-3: Reserved
3205 UCHAR ucPanelInfoSize; // start from ATOM_DTD_FORMAT to end of panel info, include ExtInfoTable
3206 USHORT usBacklightPWM; // Backlight PWM in Hz. New in _V13
3207
3208 UCHAR ucPowerSequenceDIGONtoDE_in4Ms;
3209 UCHAR ucPowerSequenceDEtoVARY_BL_in4Ms;
3210 UCHAR ucPowerSequenceDEtoDIGON_in4Ms;
3211 UCHAR ucPowerSequenceVARY_BLtoDE_in4Ms;
3212
3213 UCHAR ucOffDelay_in4Ms;
3214 UCHAR ucPowerSequenceVARY_BLtoBLON_in4Ms;
3215 UCHAR ucPowerSequenceBLONtoVARY_BL_in4Ms;
3216 UCHAR ucReserved1;
3217
3218 ULONG ulReserved[4];
3219}ATOM_LCD_INFO_V13;
3220
3221#define ATOM_LCD_INFO_LAST ATOM_LCD_INFO_V13
3222
3223//Definitions for ucLCD_Misc
3224#define ATOM_PANEL_MISC_V13_DUAL 0x00000001
3225#define ATOM_PANEL_MISC_V13_FPDI 0x00000002
3226#define ATOM_PANEL_MISC_V13_GREY_LEVEL 0x0000000C
3227#define ATOM_PANEL_MISC_V13_GREY_LEVEL_SHIFT 2
3228#define ATOM_PANEL_MISC_V13_COLOR_BIT_DEPTH_MASK 0x70
3229#define ATOM_PANEL_MISC_V13_6BIT_PER_COLOR 0x10
3230#define ATOM_PANEL_MISC_V13_8BIT_PER_COLOR 0x20
3231
3232//Color Bit Depth definition in EDID V1.4 @BYTE 14h
3233//Bit 6 5 4
3234 // 0 0 0 - Color bit depth is undefined
3235 // 0 0 1 - 6 Bits per Primary Color
3236 // 0 1 0 - 8 Bits per Primary Color
3237 // 0 1 1 - 10 Bits per Primary Color
3238 // 1 0 0 - 12 Bits per Primary Color
3239 // 1 0 1 - 14 Bits per Primary Color
3240 // 1 1 0 - 16 Bits per Primary Color
3241 // 1 1 1 - Reserved
3242
3243//Definitions for ucLCDPanel_SpecialHandlingCap:
3244
3245//Once DAL sees this CAP is set, it will read EDID from LCD on its own instead of using sLCDTiming in ATOM_LVDS_INFO_V12.
3246//Other entries in ATOM_LVDS_INFO_V12 are still valid/useful to DAL
3247#define LCDPANEL_CAP_V13_READ_EDID 0x1 // = LCDPANEL_CAP_READ_EDID no change comparing to previous version
3248
3249//If a design supports DRR (dynamic refresh rate) on internal panels (LVDS or EDP), this cap is set in ucLCDPanel_SpecialHandlingCap together
3250//with multiple supported refresh rates@usSupportedRefreshRate. This cap should not be set when only slow refresh rate is supported (static
3251//refresh rate switch by SW. This is only valid from ATOM_LVDS_INFO_V12
3252#define LCDPANEL_CAP_V13_DRR_SUPPORTED 0x2 // = LCDPANEL_CAP_DRR_SUPPORTED no change comparing to previous version
2781 3253
2782#define ATOM_LVDS_INFO_LAST ATOM_LVDS_INFO_V12 3254//Use this cap bit for a quick reference whether an embadded panel (LCD1 ) is LVDS or eDP.
3255#define LCDPANEL_CAP_V13_eDP 0x4 // = LCDPANEL_CAP_eDP no change comparing to previous version
2783 3256
2784typedef struct _ATOM_PATCH_RECORD_MODE 3257typedef struct _ATOM_PATCH_RECORD_MODE
2785{ 3258{
@@ -2944,9 +3417,9 @@ typedef struct _ATOM_DPCD_INFO
2944#define MAX_DTD_MODE_IN_VRAM 6 3417#define MAX_DTD_MODE_IN_VRAM 6
2945#define ATOM_DTD_MODE_SUPPORT_TBL_SIZE (MAX_DTD_MODE_IN_VRAM*28) //28= (SIZEOF ATOM_DTD_FORMAT) 3418#define ATOM_DTD_MODE_SUPPORT_TBL_SIZE (MAX_DTD_MODE_IN_VRAM*28) //28= (SIZEOF ATOM_DTD_FORMAT)
2946#define ATOM_STD_MODE_SUPPORT_TBL_SIZE 32*8 //32 is a predefined number,8= (SIZEOF ATOM_STD_FORMAT) 3419#define ATOM_STD_MODE_SUPPORT_TBL_SIZE 32*8 //32 is a predefined number,8= (SIZEOF ATOM_STD_FORMAT)
2947#define DFP_ENCODER_TYPE_OFFSET 0x80 3420//20 bytes for Encoder Type and DPCD in STD EDID area
2948#define DP_ENCODER_LANE_NUM_OFFSET 0x84 3421#define DFP_ENCODER_TYPE_OFFSET (ATOM_EDID_RAW_DATASIZE + ATOM_DTD_MODE_SUPPORT_TBL_SIZE + ATOM_STD_MODE_SUPPORT_TBL_SIZE - 20)
2949#define DP_ENCODER_LINK_RATE_OFFSET 0x88 3422#define ATOM_DP_DPCD_OFFSET (DFP_ENCODER_TYPE_OFFSET + 4 )
2950 3423
2951#define ATOM_HWICON1_SURFACE_ADDR 0 3424#define ATOM_HWICON1_SURFACE_ADDR 0
2952#define ATOM_HWICON2_SURFACE_ADDR (ATOM_HWICON1_SURFACE_ADDR + ATOM_HWICON_SURFACE_SIZE) 3425#define ATOM_HWICON2_SURFACE_ADDR (ATOM_HWICON1_SURFACE_ADDR + ATOM_HWICON_SURFACE_SIZE)
@@ -2997,14 +3470,16 @@ typedef struct _ATOM_DPCD_INFO
2997#define ATOM_DFP5_DTD_MODE_TBL_ADDR (ATOM_DFP5_EDID_ADDR + ATOM_EDID_RAW_DATASIZE) 3470#define ATOM_DFP5_DTD_MODE_TBL_ADDR (ATOM_DFP5_EDID_ADDR + ATOM_EDID_RAW_DATASIZE)
2998#define ATOM_DFP5_STD_MODE_TBL_ADDR (ATOM_DFP5_DTD_MODE_TBL_ADDR + ATOM_DTD_MODE_SUPPORT_TBL_SIZE) 3471#define ATOM_DFP5_STD_MODE_TBL_ADDR (ATOM_DFP5_DTD_MODE_TBL_ADDR + ATOM_DTD_MODE_SUPPORT_TBL_SIZE)
2999 3472
3000#define ATOM_DP_TRAINING_TBL_ADDR (ATOM_DFP5_STD_MODE_TBL_ADDR+ATOM_STD_MODE_SUPPORT_TBL_SIZE) 3473#define ATOM_DP_TRAINING_TBL_ADDR (ATOM_DFP5_STD_MODE_TBL_ADDR + ATOM_STD_MODE_SUPPORT_TBL_SIZE)
3001 3474
3002#define ATOM_STACK_STORAGE_START (ATOM_DP_TRAINING_TBL_ADDR+256) 3475#define ATOM_STACK_STORAGE_START (ATOM_DP_TRAINING_TBL_ADDR + 1024)
3003#define ATOM_STACK_STORAGE_END ATOM_STACK_STORAGE_START+512 3476#define ATOM_STACK_STORAGE_END ATOM_STACK_STORAGE_START + 512
3004 3477
3005//The size below is in Kb! 3478//The size below is in Kb!
3006#define ATOM_VRAM_RESERVE_SIZE ((((ATOM_STACK_STORAGE_END - ATOM_HWICON1_SURFACE_ADDR)>>10)+4)&0xFFFC) 3479#define ATOM_VRAM_RESERVE_SIZE ((((ATOM_STACK_STORAGE_END - ATOM_HWICON1_SURFACE_ADDR)>>10)+4)&0xFFFC)
3007 3480
3481#define ATOM_VRAM_RESERVE_V2_SIZE 32
3482
3008#define ATOM_VRAM_OPERATION_FLAGS_MASK 0xC0000000L 3483#define ATOM_VRAM_OPERATION_FLAGS_MASK 0xC0000000L
3009#define ATOM_VRAM_OPERATION_FLAGS_SHIFT 30 3484#define ATOM_VRAM_OPERATION_FLAGS_SHIFT 30
3010#define ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION 0x1 3485#define ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION 0x1
@@ -3206,6 +3681,15 @@ typedef struct _ATOM_DISPLAY_OBJECT_PATH
3206 USHORT usGraphicObjIds[1]; //1st Encoder Obj source from GPU to last Graphic Obj destinate to connector. 3681 USHORT usGraphicObjIds[1]; //1st Encoder Obj source from GPU to last Graphic Obj destinate to connector.
3207}ATOM_DISPLAY_OBJECT_PATH; 3682}ATOM_DISPLAY_OBJECT_PATH;
3208 3683
3684typedef struct _ATOM_DISPLAY_EXTERNAL_OBJECT_PATH
3685{
3686 USHORT usDeviceTag; //supported device
3687 USHORT usSize; //the size of ATOM_DISPLAY_OBJECT_PATH
3688 USHORT usConnObjectId; //Connector Object ID
3689 USHORT usGPUObjectId; //GPU ID
3690 USHORT usGraphicObjIds[2]; //usGraphicObjIds[0]= GPU internal encoder, usGraphicObjIds[1]= external encoder
3691}ATOM_DISPLAY_EXTERNAL_OBJECT_PATH;
3692
3209typedef struct _ATOM_DISPLAY_OBJECT_PATH_TABLE 3693typedef struct _ATOM_DISPLAY_OBJECT_PATH_TABLE
3210{ 3694{
3211 UCHAR ucNumOfDispPath; 3695 UCHAR ucNumOfDispPath;
@@ -3261,6 +3745,47 @@ typedef struct _ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT //usSrcDstTableOffset
3261#define EXT_AUXDDC_LUTINDEX_7 7 3745#define EXT_AUXDDC_LUTINDEX_7 7
3262#define MAX_NUMBER_OF_EXT_AUXDDC_LUT_ENTRIES (EXT_AUXDDC_LUTINDEX_7+1) 3746#define MAX_NUMBER_OF_EXT_AUXDDC_LUT_ENTRIES (EXT_AUXDDC_LUTINDEX_7+1)
3263 3747
3748//ucChannelMapping are defined as following
3749//for DP connector, eDP, DP to VGA/LVDS
3750//Bit[1:0]: Define which pin connect to DP connector DP_Lane0, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3751//Bit[3:2]: Define which pin connect to DP connector DP_Lane1, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3752//Bit[5:4]: Define which pin connect to DP connector DP_Lane2, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3753//Bit[7:6]: Define which pin connect to DP connector DP_Lane3, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3754typedef struct _ATOM_DP_CONN_CHANNEL_MAPPING
3755{
3756#if ATOM_BIG_ENDIAN
3757 UCHAR ucDP_Lane3_Source:2;
3758 UCHAR ucDP_Lane2_Source:2;
3759 UCHAR ucDP_Lane1_Source:2;
3760 UCHAR ucDP_Lane0_Source:2;
3761#else
3762 UCHAR ucDP_Lane0_Source:2;
3763 UCHAR ucDP_Lane1_Source:2;
3764 UCHAR ucDP_Lane2_Source:2;
3765 UCHAR ucDP_Lane3_Source:2;
3766#endif
3767}ATOM_DP_CONN_CHANNEL_MAPPING;
3768
3769//for DVI/HDMI, in dual link case, both links have to have same mapping.
3770//Bit[1:0]: Define which pin connect to DVI connector data Lane2, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3771//Bit[3:2]: Define which pin connect to DVI connector data Lane1, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3772//Bit[5:4]: Define which pin connect to DVI connector data Lane0, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3773//Bit[7:6]: Define which pin connect to DVI connector clock lane, =0: source from GPU pin TX0, =1: from GPU pin TX1, =2: from GPU pin TX2, =3 from GPU pin TX3
3774typedef struct _ATOM_DVI_CONN_CHANNEL_MAPPING
3775{
3776#if ATOM_BIG_ENDIAN
3777 UCHAR ucDVI_CLK_Source:2;
3778 UCHAR ucDVI_DATA0_Source:2;
3779 UCHAR ucDVI_DATA1_Source:2;
3780 UCHAR ucDVI_DATA2_Source:2;
3781#else
3782 UCHAR ucDVI_DATA2_Source:2;
3783 UCHAR ucDVI_DATA1_Source:2;
3784 UCHAR ucDVI_DATA0_Source:2;
3785 UCHAR ucDVI_CLK_Source:2;
3786#endif
3787}ATOM_DVI_CONN_CHANNEL_MAPPING;
3788
3264typedef struct _EXT_DISPLAY_PATH 3789typedef struct _EXT_DISPLAY_PATH
3265{ 3790{
3266 USHORT usDeviceTag; //A bit vector to show what devices are supported 3791 USHORT usDeviceTag; //A bit vector to show what devices are supported
@@ -3269,7 +3794,13 @@ typedef struct _EXT_DISPLAY_PATH
3269 UCHAR ucExtAUXDDCLutIndex; //An index into external AUX/DDC channel LUT 3794 UCHAR ucExtAUXDDCLutIndex; //An index into external AUX/DDC channel LUT
3270 UCHAR ucExtHPDPINLutIndex; //An index into external HPD pin LUT 3795 UCHAR ucExtHPDPINLutIndex; //An index into external HPD pin LUT
3271 USHORT usExtEncoderObjId; //external encoder object id 3796 USHORT usExtEncoderObjId; //external encoder object id
3272 USHORT usReserved[3]; 3797 union{
3798 UCHAR ucChannelMapping; // if ucChannelMapping=0, using default one to one mapping
3799 ATOM_DP_CONN_CHANNEL_MAPPING asDPMapping;
3800 ATOM_DVI_CONN_CHANNEL_MAPPING asDVIMapping;
3801 };
3802 UCHAR ucReserved;
3803 USHORT usReserved[2];
3273}EXT_DISPLAY_PATH; 3804}EXT_DISPLAY_PATH;
3274 3805
3275#define NUMBER_OF_UCHAR_FOR_GUID 16 3806#define NUMBER_OF_UCHAR_FOR_GUID 16
@@ -3281,7 +3812,8 @@ typedef struct _ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO
3281 UCHAR ucGuid [NUMBER_OF_UCHAR_FOR_GUID]; // a GUID is a 16 byte long string 3812 UCHAR ucGuid [NUMBER_OF_UCHAR_FOR_GUID]; // a GUID is a 16 byte long string
3282 EXT_DISPLAY_PATH sPath[MAX_NUMBER_OF_EXT_DISPLAY_PATH]; // total of fixed 7 entries. 3813 EXT_DISPLAY_PATH sPath[MAX_NUMBER_OF_EXT_DISPLAY_PATH]; // total of fixed 7 entries.
3283 UCHAR ucChecksum; // a simple Checksum of the sum of whole structure equal to 0x0. 3814 UCHAR ucChecksum; // a simple Checksum of the sum of whole structure equal to 0x0.
3284 UCHAR Reserved [7]; // for potential expansion 3815 UCHAR uc3DStereoPinId; // use for eDP panel
3816 UCHAR Reserved [6]; // for potential expansion
3285}ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO; 3817}ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO;
3286 3818
3287//Related definitions, all records are differnt but they have a commond header 3819//Related definitions, all records are differnt but they have a commond header
@@ -3311,10 +3843,11 @@ typedef struct _ATOM_COMMON_RECORD_HEADER
3311#define ATOM_CONNECTOR_AUXDDC_LUT_RECORD_TYPE 17 //This is for the case when connectors are not known to object table 3843#define ATOM_CONNECTOR_AUXDDC_LUT_RECORD_TYPE 17 //This is for the case when connectors are not known to object table
3312#define ATOM_OBJECT_LINK_RECORD_TYPE 18 //Once this record is present under one object, it indicats the oobject is linked to another obj described by the record 3844#define ATOM_OBJECT_LINK_RECORD_TYPE 18 //Once this record is present under one object, it indicats the oobject is linked to another obj described by the record
3313#define ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE 19 3845#define ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE 19
3846#define ATOM_ENCODER_CAP_RECORD_TYPE 20
3314 3847
3315 3848
3316//Must be updated when new record type is added,equal to that record definition! 3849//Must be updated when new record type is added,equal to that record definition!
3317#define ATOM_MAX_OBJECT_RECORD_NUMBER ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE 3850#define ATOM_MAX_OBJECT_RECORD_NUMBER ATOM_ENCODER_CAP_RECORD_TYPE
3318 3851
3319typedef struct _ATOM_I2C_RECORD 3852typedef struct _ATOM_I2C_RECORD
3320{ 3853{
@@ -3441,6 +3974,26 @@ typedef struct _ATOM_ENCODER_DVO_CF_RECORD
3441 UCHAR ucPadding[2]; 3974 UCHAR ucPadding[2];
3442}ATOM_ENCODER_DVO_CF_RECORD; 3975}ATOM_ENCODER_DVO_CF_RECORD;
3443 3976
3977// Bit maps for ATOM_ENCODER_CAP_RECORD.ucEncoderCap
3978#define ATOM_ENCODER_CAP_RECORD_HBR2 0x01 // DP1.2 HBR2 is supported by this path
3979
3980typedef struct _ATOM_ENCODER_CAP_RECORD
3981{
3982 ATOM_COMMON_RECORD_HEADER sheader;
3983 union {
3984 USHORT usEncoderCap;
3985 struct {
3986#if ATOM_BIG_ENDIAN
3987 USHORT usReserved:15; // Bit1-15 may be defined for other capability in future
3988 USHORT usHBR2Cap:1; // Bit0 is for DP1.2 HBR2 capability.
3989#else
3990 USHORT usHBR2Cap:1; // Bit0 is for DP1.2 HBR2 capability.
3991 USHORT usReserved:15; // Bit1-15 may be defined for other capability in future
3992#endif
3993 };
3994 };
3995}ATOM_ENCODER_CAP_RECORD;
3996
3444// value for ATOM_CONNECTOR_CF_RECORD.ucConnectedDvoBundle 3997// value for ATOM_CONNECTOR_CF_RECORD.ucConnectedDvoBundle
3445#define ATOM_CONNECTOR_CF_RECORD_CONNECTED_UPPER12BITBUNDLEA 1 3998#define ATOM_CONNECTOR_CF_RECORD_CONNECTED_UPPER12BITBUNDLEA 1
3446#define ATOM_CONNECTOR_CF_RECORD_CONNECTED_LOWER12BITBUNDLEB 2 3999#define ATOM_CONNECTOR_CF_RECORD_CONNECTED_LOWER12BITBUNDLEB 2
@@ -3580,6 +4133,11 @@ typedef struct _ATOM_VOLTAGE_CONTROL
3580#define VOLTAGE_CONTROL_ID_DAC 0x02 //I2C control, used for R5xx/R6xx MVDDC,MVDDQ or VDDCI 4133#define VOLTAGE_CONTROL_ID_DAC 0x02 //I2C control, used for R5xx/R6xx MVDDC,MVDDQ or VDDCI
3581#define VOLTAGE_CONTROL_ID_VT116xM 0x03 //I2C control, used for R6xx Core Voltage 4134#define VOLTAGE_CONTROL_ID_VT116xM 0x03 //I2C control, used for R6xx Core Voltage
3582#define VOLTAGE_CONTROL_ID_DS4402 0x04 4135#define VOLTAGE_CONTROL_ID_DS4402 0x04
4136#define VOLTAGE_CONTROL_ID_UP6266 0x05
4137#define VOLTAGE_CONTROL_ID_SCORPIO 0x06
4138#define VOLTAGE_CONTROL_ID_VT1556M 0x07
4139#define VOLTAGE_CONTROL_ID_CHL822x 0x08
4140#define VOLTAGE_CONTROL_ID_VT1586M 0x09
3583 4141
3584typedef struct _ATOM_VOLTAGE_OBJECT 4142typedef struct _ATOM_VOLTAGE_OBJECT
3585{ 4143{
@@ -3670,66 +4228,157 @@ typedef struct _ATOM_POWER_SOURCE_INFO
3670#define POWER_SENSOR_GPIO 0x01 4228#define POWER_SENSOR_GPIO 0x01
3671#define POWER_SENSOR_I2C 0x02 4229#define POWER_SENSOR_I2C 0x02
3672 4230
4231typedef struct _ATOM_CLK_VOLT_CAPABILITY
4232{
4233 ULONG ulVoltageIndex; // The Voltage Index indicated by FUSE, same voltage index shared with SCLK DPM fuse table
4234 ULONG ulMaximumSupportedCLK; // Maximum clock supported with specified voltage index, unit in 10kHz
4235}ATOM_CLK_VOLT_CAPABILITY;
4236
4237typedef struct _ATOM_AVAILABLE_SCLK_LIST
4238{
4239 ULONG ulSupportedSCLK; // Maximum clock supported with specified voltage index, unit in 10kHz
4240 USHORT usVoltageIndex; // The Voltage Index indicated by FUSE for specified SCLK
4241 USHORT usVoltageID; // The Voltage ID indicated by FUSE for specified SCLK
4242}ATOM_AVAILABLE_SCLK_LIST;
4243
4244// ATOM_INTEGRATED_SYSTEM_INFO_V6 ulSystemConfig cap definition
4245#define ATOM_IGP_INFO_V6_SYSTEM_CONFIG__PCIE_POWER_GATING_ENABLE 1 // refer to ulSystemConfig bit[0]
4246
4247// this IntegrateSystemInfoTable is used for Liano/Ontario APU
3673typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 4248typedef struct _ATOM_INTEGRATED_SYSTEM_INFO_V6
3674{ 4249{
3675 ATOM_COMMON_TABLE_HEADER sHeader; 4250 ATOM_COMMON_TABLE_HEADER sHeader;
3676 ULONG ulBootUpEngineClock; 4251 ULONG ulBootUpEngineClock;
3677 ULONG ulDentistVCOFreq; 4252 ULONG ulDentistVCOFreq;
3678 ULONG ulBootUpUMAClock; 4253 ULONG ulBootUpUMAClock;
3679 ULONG ulReserved1[8]; 4254 ATOM_CLK_VOLT_CAPABILITY sDISPCLK_Voltage[4];
3680 ULONG ulBootUpReqDisplayVector; 4255 ULONG ulBootUpReqDisplayVector;
3681 ULONG ulOtherDisplayMisc; 4256 ULONG ulOtherDisplayMisc;
3682 ULONG ulGPUCapInfo; 4257 ULONG ulGPUCapInfo;
3683 ULONG ulReserved2[3]; 4258 ULONG ulSB_MMIO_Base_Addr;
4259 USHORT usRequestedPWMFreqInHz;
4260 UCHAR ucHtcTmpLmt;
4261 UCHAR ucHtcHystLmt;
4262 ULONG ulMinEngineClock;
3684 ULONG ulSystemConfig; 4263 ULONG ulSystemConfig;
3685 ULONG ulCPUCapInfo; 4264 ULONG ulCPUCapInfo;
3686 USHORT usMaxNBVoltage; 4265 USHORT usNBP0Voltage;
3687 USHORT usMinNBVoltage; 4266 USHORT usNBP1Voltage;
3688 USHORT usBootUpNBVoltage; 4267 USHORT usBootUpNBVoltage;
3689 USHORT usExtDispConnInfoOffset; 4268 USHORT usExtDispConnInfoOffset;
3690 UCHAR ucHtcTmpLmt; 4269 USHORT usPanelRefreshRateRange;
3691 UCHAR ucTjOffset;
3692 UCHAR ucMemoryType; 4270 UCHAR ucMemoryType;
3693 UCHAR ucUMAChannelNumber; 4271 UCHAR ucUMAChannelNumber;
3694 ULONG ulCSR_M3_ARB_CNTL_DEFAULT[10]; 4272 ULONG ulCSR_M3_ARB_CNTL_DEFAULT[10];
3695 ULONG ulCSR_M3_ARB_CNTL_UVD[10]; 4273 ULONG ulCSR_M3_ARB_CNTL_UVD[10];
3696 ULONG ulCSR_M3_ARB_CNTL_FS3D[10]; 4274 ULONG ulCSR_M3_ARB_CNTL_FS3D[10];
3697 ULONG ulReserved3[42]; 4275 ATOM_AVAILABLE_SCLK_LIST sAvail_SCLK[5];
4276 ULONG ulGMCRestoreResetTime;
4277 ULONG ulMinimumNClk;
4278 ULONG ulIdleNClk;
4279 ULONG ulDDR_DLL_PowerUpTime;
4280 ULONG ulDDR_PLL_PowerUpTime;
4281 USHORT usPCIEClkSSPercentage;
4282 USHORT usPCIEClkSSType;
4283 USHORT usLvdsSSPercentage;
4284 USHORT usLvdsSSpreadRateIn10Hz;
4285 USHORT usHDMISSPercentage;
4286 USHORT usHDMISSpreadRateIn10Hz;
4287 USHORT usDVISSPercentage;
4288 USHORT usDVISSpreadRateIn10Hz;
4289 ULONG ulReserved3[21];
3698 ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO sExtDispConnInfo; 4290 ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO sExtDispConnInfo;
3699}ATOM_INTEGRATED_SYSTEM_INFO_V6; 4291}ATOM_INTEGRATED_SYSTEM_INFO_V6;
3700 4292
4293// ulGPUCapInfo
4294#define INTEGRATED_SYSTEM_INFO_V6_GPUCAPINFO__TMDSHDMI_COHERENT_SINGLEPLL_MODE 0x01
4295#define INTEGRATED_SYSTEM_INFO_V6_GPUCAPINFO__DISABLE_AUX_HW_MODE_DETECTION 0x08
4296
4297// ulOtherDisplayMisc
4298#define INTEGRATED_SYSTEM_INFO__GET_EDID_CALLBACK_FUNC_SUPPORT 0x01
4299
4300
3701/********************************************************************************************************************** 4301/**********************************************************************************************************************
3702// ATOM_INTEGRATED_SYSTEM_INFO_V6 Description 4302 ATOM_INTEGRATED_SYSTEM_INFO_V6 Description
3703//ulBootUpEngineClock: VBIOS bootup Engine clock frequency, in 10kHz unit. 4303ulBootUpEngineClock: VBIOS bootup Engine clock frequency, in 10kHz unit. if it is equal 0, then VBIOS use pre-defined bootup engine clock
3704//ulDentistVCOFreq: Dentist VCO clock in 10kHz unit. 4304ulDentistVCOFreq: Dentist VCO clock in 10kHz unit.
3705//ulBootUpUMAClock: System memory boot up clock frequency in 10Khz unit. 4305ulBootUpUMAClock: System memory boot up clock frequency in 10Khz unit.
3706//ulReserved1[8] Reserved by now, must be 0x0. 4306sDISPCLK_Voltage: Report Display clock voltage requirement.
3707//ulBootUpReqDisplayVector VBIOS boot up display IDs 4307
3708// ATOM_DEVICE_CRT1_SUPPORT 0x0001 4308ulBootUpReqDisplayVector: VBIOS boot up display IDs, following are supported devices in Liano/Ontaio projects:
3709// ATOM_DEVICE_CRT2_SUPPORT 0x0010 4309 ATOM_DEVICE_CRT1_SUPPORT 0x0001
3710// ATOM_DEVICE_DFP1_SUPPORT 0x0008 4310 ATOM_DEVICE_CRT2_SUPPORT 0x0010
3711// ATOM_DEVICE_DFP6_SUPPORT 0x0040 4311 ATOM_DEVICE_DFP1_SUPPORT 0x0008
3712// ATOM_DEVICE_DFP2_SUPPORT 0x0080 4312 ATOM_DEVICE_DFP6_SUPPORT 0x0040
3713// ATOM_DEVICE_DFP3_SUPPORT 0x0200 4313 ATOM_DEVICE_DFP2_SUPPORT 0x0080
3714// ATOM_DEVICE_DFP4_SUPPORT 0x0400 4314 ATOM_DEVICE_DFP3_SUPPORT 0x0200
3715// ATOM_DEVICE_DFP5_SUPPORT 0x0800 4315 ATOM_DEVICE_DFP4_SUPPORT 0x0400
3716// ATOM_DEVICE_LCD1_SUPPORT 0x0002 4316 ATOM_DEVICE_DFP5_SUPPORT 0x0800
3717//ulOtherDisplayMisc Other display related flags, not defined yet. 4317 ATOM_DEVICE_LCD1_SUPPORT 0x0002
3718//ulGPUCapInfo TBD 4318ulOtherDisplayMisc: Other display related flags, not defined yet.
3719//ulReserved2[3] must be 0x0 for the reserved. 4319ulGPUCapInfo: bit[0]=0: TMDS/HDMI Coherent Mode use cascade PLL mode.
3720//ulSystemConfig TBD 4320 =1: TMDS/HDMI Coherent Mode use signel PLL mode.
3721//ulCPUCapInfo TBD 4321 bit[3]=0: Enable HW AUX mode detection logic
3722//usMaxNBVoltage High NB voltage in unit of mv, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse. 4322 =1: Disable HW AUX mode dettion logic
3723//usMinNBVoltage Low NB voltage in unit of mv, calculated using current VDDNB (D24F2xDC) and VDDNB offset fuse. 4323ulSB_MMIO_Base_Addr: Physical Base address to SB MMIO space. Driver needs to initialize it for SMU usage.
3724//usBootUpNBVoltage Boot up NB voltage in unit of mv. 4324
3725//ucHtcTmpLmt Bit [22:16] of D24F3x64 Thermal Control (HTC) Register. 4325usRequestedPWMFreqInHz: When it's set to 0x0 by SBIOS: the LCD BackLight is not controlled by GPU(SW).
3726//ucTjOffset Bit [28:22] of D24F3xE4 Thermtrip Status Register,may not be needed. 4326 Any attempt to change BL using VBIOS function or enable VariBri from PP table is not effective since ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==0;
3727//ucMemoryType [3:0]=1:DDR1;=2:DDR2;=3:DDR3.[7:4] is reserved. 4327
3728//ucUMAChannelNumber System memory channel numbers. 4328 When it's set to a non-zero frequency, the BackLight is controlled by GPU (SW) in one of two ways below:
3729//usExtDispConnectionInfoOffset ATOM_EXTERNAL_DISPLAY_CONNECTION_INFO offset relative to beginning of this table. 4329 1. SW uses the GPU BL PWM output to control the BL, in chis case, this non-zero frequency determines what freq GPU should use;
3730//ulCSR_M3_ARB_CNTL_DEFAULT[10] Arrays with values for CSR M3 arbiter for default 4330 VBIOS will set up proper PWM frequency and ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1,as the result,
3731//ulCSR_M3_ARB_CNTL_UVD[10] Arrays with values for CSR M3 arbiter for UVD playback. 4331 Changing BL using VBIOS function is functional in both driver and non-driver present environment;
3732//ulCSR_M3_ARB_CNTL_FS3D[10] Arrays with values for CSR M3 arbiter for Full Screen 3D applications. 4332 and enabling VariBri under the driver environment from PP table is optional.
4333
4334 2. SW uses other means to control BL (like DPCD),this non-zero frequency serves as a flag only indicating
4335 that BL control from GPU is expected.
4336 VBIOS will NOT set up PWM frequency but make ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU==1
4337 Changing BL using VBIOS function could be functional in both driver and non-driver present environment,but
4338 it's per platform
4339 and enabling VariBri under the driver environment from PP table is optional.
4340
4341ucHtcTmpLmt: Refer to D18F3x64 bit[22:16], HtcTmpLmt.
4342 Threshold on value to enter HTC_active state.
4343ucHtcHystLmt: Refer to D18F3x64 bit[27:24], HtcHystLmt.
4344 To calculate threshold off value to exit HTC_active state, which is Threshold on vlaue minus ucHtcHystLmt.
4345ulMinEngineClock: Minimum SCLK allowed in 10kHz unit. This is calculated based on WRCK Fuse settings.
4346ulSystemConfig: Bit[0]=0: PCIE Power Gating Disabled
4347 =1: PCIE Power Gating Enabled
4348 Bit[1]=0: DDR-DLL shut-down feature disabled.
4349 1: DDR-DLL shut-down feature enabled.
4350 Bit[2]=0: DDR-PLL Power down feature disabled.
4351 1: DDR-PLL Power down feature enabled.
4352ulCPUCapInfo: TBD
4353usNBP0Voltage: VID for voltage on NB P0 State
4354usNBP1Voltage: VID for voltage on NB P1 State
4355usBootUpNBVoltage: Voltage Index of GNB voltage configured by SBIOS, which is suffcient to support VBIOS DISPCLK requirement.
4356usExtDispConnInfoOffset: Offset to sExtDispConnInfo inside the structure
4357usPanelRefreshRateRange: Bit vector for LCD supported refresh rate range. If DRR is requestd by the platform, at least two bits need to be set
4358 to indicate a range.
4359 SUPPORTED_LCD_REFRESHRATE_30Hz 0x0004
4360 SUPPORTED_LCD_REFRESHRATE_40Hz 0x0008
4361 SUPPORTED_LCD_REFRESHRATE_50Hz 0x0010
4362 SUPPORTED_LCD_REFRESHRATE_60Hz 0x0020
4363ucMemoryType: [3:0]=1:DDR1;=2:DDR2;=3:DDR3.[7:4] is reserved.
4364ucUMAChannelNumber: System memory channel numbers.
4365ulCSR_M3_ARB_CNTL_DEFAULT[10]: Arrays with values for CSR M3 arbiter for default
4366ulCSR_M3_ARB_CNTL_UVD[10]: Arrays with values for CSR M3 arbiter for UVD playback.
4367ulCSR_M3_ARB_CNTL_FS3D[10]: Arrays with values for CSR M3 arbiter for Full Screen 3D applications.
4368sAvail_SCLK[5]: Arrays to provide availabe list of SLCK and corresponding voltage, order from low to high
4369ulGMCRestoreResetTime: GMC power restore and GMC reset time to calculate data reconnection latency. Unit in ns.
4370ulMinimumNClk: Minimum NCLK speed among all NB-Pstates to calcualte data reconnection latency. Unit in 10kHz.
4371ulIdleNClk: NCLK speed while memory runs in self-refresh state. Unit in 10kHz.
4372ulDDR_DLL_PowerUpTime: DDR PHY DLL power up time. Unit in ns.
4373ulDDR_PLL_PowerUpTime: DDR PHY PLL power up time. Unit in ns.
4374usPCIEClkSSPercentage: PCIE Clock Spred Spectrum Percentage in unit 0.01%; 100 mean 1%.
4375usPCIEClkSSType: PCIE Clock Spred Spectrum Type. 0 for Down spread(default); 1 for Center spread.
4376usLvdsSSPercentage: LVDS panel ( not include eDP ) Spread Spectrum Percentage in unit of 0.01%, =0, use VBIOS default setting.
4377usLvdsSSpreadRateIn10Hz: LVDS panel ( not include eDP ) Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting.
4378usHDMISSPercentage: HDMI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%, =0, use VBIOS default setting.
4379usHDMISSpreadRateIn10Hz: HDMI Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting.
4380usDVISSPercentage: DVI Spread Spectrum Percentage in unit 0.01%; 100 mean 1%, =0, use VBIOS default setting.
4381usDVISSpreadRateIn10Hz: DVI Spread Spectrum frequency in unit of 10Hz, =0, use VBIOS default setting.
3733**********************************************************************************************************************/ 4382**********************************************************************************************************************/
3734 4383
3735/**************************************************************************/ 4384/**************************************************************************/
@@ -3790,6 +4439,7 @@ typedef struct _ATOM_ASIC_SS_ASSIGNMENT
3790#define ASIC_INTERNAL_SS_ON_LVDS 6 4439#define ASIC_INTERNAL_SS_ON_LVDS 6
3791#define ASIC_INTERNAL_SS_ON_DP 7 4440#define ASIC_INTERNAL_SS_ON_DP 7
3792#define ASIC_INTERNAL_SS_ON_DCPLL 8 4441#define ASIC_INTERNAL_SS_ON_DCPLL 8
4442#define ASIC_EXTERNAL_SS_ON_DP_CLOCK 9
3793 4443
3794typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V2 4444typedef struct _ATOM_ASIC_SS_ASSIGNMENT_V2
3795{ 4445{
@@ -3903,6 +4553,7 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
3903#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_AC 1 4553#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_AC 1
3904#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_DC 2 4554#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_DC 2
3905#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_LITEAC 3 4555#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_LITEAC 3
4556#define ATOM_S0_SYSTEM_POWER_STATE_VALUE_LIT2AC 4
3906 4557
3907//Byte aligned defintion for BIOS usage 4558//Byte aligned defintion for BIOS usage
3908#define ATOM_S0_CRT1_MONOb0 0x01 4559#define ATOM_S0_CRT1_MONOb0 0x01
@@ -4529,7 +5180,8 @@ typedef struct _ATOM_INIT_REG_BLOCK{
4529#define INDEX_ACCESS_RANGE_BEGIN (VALUE_DWORD + 1) 5180#define INDEX_ACCESS_RANGE_BEGIN (VALUE_DWORD + 1)
4530#define INDEX_ACCESS_RANGE_END (INDEX_ACCESS_RANGE_BEGIN + 1) 5181#define INDEX_ACCESS_RANGE_END (INDEX_ACCESS_RANGE_BEGIN + 1)
4531#define VALUE_INDEX_ACCESS_SINGLE (INDEX_ACCESS_RANGE_END + 1) 5182#define VALUE_INDEX_ACCESS_SINGLE (INDEX_ACCESS_RANGE_END + 1)
4532 5183//#define ACCESS_MCIODEBUGIND 0x40 //defined in BIOS code
5184#define ACCESS_PLACEHOLDER 0x80
4533 5185
4534typedef struct _ATOM_MC_INIT_PARAM_TABLE 5186typedef struct _ATOM_MC_INIT_PARAM_TABLE
4535{ 5187{
@@ -4554,6 +5206,10 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
4554#define _32Mx32 0x33 5206#define _32Mx32 0x33
4555#define _64Mx8 0x41 5207#define _64Mx8 0x41
4556#define _64Mx16 0x42 5208#define _64Mx16 0x42
5209#define _64Mx32 0x43
5210#define _128Mx8 0x51
5211#define _128Mx16 0x52
5212#define _256Mx8 0x61
4557 5213
4558#define SAMSUNG 0x1 5214#define SAMSUNG 0x1
4559#define INFINEON 0x2 5215#define INFINEON 0x2
@@ -4569,10 +5225,11 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
4569#define QIMONDA INFINEON 5225#define QIMONDA INFINEON
4570#define PROMOS MOSEL 5226#define PROMOS MOSEL
4571#define KRETON INFINEON 5227#define KRETON INFINEON
5228#define ELIXIR NANYA
4572 5229
4573/////////////Support for GDDR5 MC uCode to reside in upper 64K of ROM///////////// 5230/////////////Support for GDDR5 MC uCode to reside in upper 64K of ROM/////////////
4574 5231
4575#define UCODE_ROM_START_ADDRESS 0x1c000 5232#define UCODE_ROM_START_ADDRESS 0x1b800
4576#define UCODE_SIGNATURE 0x4375434d // 'MCuC' - MC uCode 5233#define UCODE_SIGNATURE 0x4375434d // 'MCuC' - MC uCode
4577 5234
4578//uCode block header for reference 5235//uCode block header for reference
@@ -4903,7 +5560,34 @@ typedef struct _ATOM_VRAM_MODULE_V6
4903 ATOM_MEMORY_TIMING_FORMAT_V2 asMemTiming[5];//Memory Timing block sort from lower clock to higher clock 5560 ATOM_MEMORY_TIMING_FORMAT_V2 asMemTiming[5];//Memory Timing block sort from lower clock to higher clock
4904}ATOM_VRAM_MODULE_V6; 5561}ATOM_VRAM_MODULE_V6;
4905 5562
4906 5563typedef struct _ATOM_VRAM_MODULE_V7
5564{
5565// Design Specific Values
5566 ULONG ulChannelMapCfg; // mmMC_SHARED_CHREMAP
5567 USHORT usModuleSize; // Size of ATOM_VRAM_MODULE_V7
5568 USHORT usPrivateReserved; // MC_ARB_RAMCFG (includes NOOFBANK,NOOFRANKS,NOOFROWS,NOOFCOLS)
5569 USHORT usReserved;
5570 UCHAR ucExtMemoryID; // Current memory module ID
5571 UCHAR ucMemoryType; // MEM_TYPE_DDR2/DDR3/GDDR3/GDDR5
5572 UCHAR ucChannelNum; // Number of mem. channels supported in this module
5573 UCHAR ucChannelWidth; // CHANNEL_16BIT/CHANNEL_32BIT/CHANNEL_64BIT
5574 UCHAR ucDensity; // _8Mx32, _16Mx32, _16Mx16, _32Mx16
5575 UCHAR ucReserve; // Former container for Mx_FLAGS like DBI_AC_MODE_ENABLE_ASIC for GDDR4. Not used now.
5576 UCHAR ucMisc; // RANK_OF_THISMEMORY etc.
5577 UCHAR ucVREFI; // Not used.
5578 UCHAR ucNPL_RT; // Round trip delay (MC_SEQ_CAS_TIMING [28:24]:TCL=CL+NPL_RT-2). Always 2.
5579 UCHAR ucPreamble; // [7:4] Write Preamble, [3:0] Read Preamble
5580 UCHAR ucMemorySize; // Total memory size in unit of 16MB for CONFIG_MEMSIZE - bit[23:0] zeros
5581 UCHAR ucReserved[3];
5582// Memory Module specific values
5583 USHORT usEMRS2Value; // EMRS2/MR2 Value.
5584 USHORT usEMRS3Value; // EMRS3/MR3 Value.
5585 UCHAR ucMemoryVenderID; // [7:4] Revision, [3:0] Vendor code
5586 UCHAR ucRefreshRateFactor; // [1:0]=RefreshFactor (00=8ms, 01=16ms, 10=32ms,11=64ms)
5587 UCHAR ucFIFODepth; // FIFO depth can be detected during vendor detection, here is hardcoded per memory
5588 UCHAR ucCDR_Bandwidth; // [0:3]=Read CDR bandwidth, [4:7] - Write CDR Bandwidth
5589 char strMemPNString[20]; // part number end with '0'.
5590}ATOM_VRAM_MODULE_V7;
4907 5591
4908typedef struct _ATOM_VRAM_INFO_V2 5592typedef struct _ATOM_VRAM_INFO_V2
4909{ 5593{
@@ -4942,6 +5626,20 @@ typedef struct _ATOM_VRAM_INFO_V4
4942 // ATOM_INIT_REG_BLOCK aMemAdjust; 5626 // ATOM_INIT_REG_BLOCK aMemAdjust;
4943}ATOM_VRAM_INFO_V4; 5627}ATOM_VRAM_INFO_V4;
4944 5628
5629typedef struct _ATOM_VRAM_INFO_HEADER_V2_1
5630{
5631 ATOM_COMMON_TABLE_HEADER sHeader;
5632 USHORT usMemAdjustTblOffset; // offset of ATOM_INIT_REG_BLOCK structure for memory vendor specific MC adjust setting
5633 USHORT usMemClkPatchTblOffset; // offset of ATOM_INIT_REG_BLOCK structure for memory clock specific MC setting
5634 USHORT usReserved[4];
5635 UCHAR ucNumOfVRAMModule; // indicate number of VRAM module
5636 UCHAR ucMemoryClkPatchTblVer; // version of memory AC timing register list
5637 UCHAR ucVramModuleVer; // indicate ATOM_VRAM_MODUE version
5638 UCHAR ucReserved;
5639 ATOM_VRAM_MODULE_V7 aVramInfo[ATOM_MAX_NUMBER_OF_VRAM_MODULE]; // just for allocation, real number of blocks is in ucNumOfVRAMModule;
5640}ATOM_VRAM_INFO_HEADER_V2_1;
5641
5642
4945typedef struct _ATOM_VRAM_GPIO_DETECTION_INFO 5643typedef struct _ATOM_VRAM_GPIO_DETECTION_INFO
4946{ 5644{
4947 ATOM_COMMON_TABLE_HEADER sHeader; 5645 ATOM_COMMON_TABLE_HEADER sHeader;
@@ -5182,6 +5880,16 @@ typedef struct _ASIC_TRANSMITTER_INFO
5182 UCHAR ucReserved; 5880 UCHAR ucReserved;
5183}ASIC_TRANSMITTER_INFO; 5881}ASIC_TRANSMITTER_INFO;
5184 5882
5883#define ASIC_TRANSMITTER_INFO_CONFIG__DVO_SDR_MODE 0x01
5884#define ASIC_TRANSMITTER_INFO_CONFIG__COHERENT_MODE 0x02
5885#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODEROBJ_ID_MASK 0xc4
5886#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_A 0x00
5887#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_B 0x04
5888#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_C 0x40
5889#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_D 0x44
5890#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_E 0x80
5891#define ASIC_TRANSMITTER_INFO_CONFIG__ENCODER_F 0x84
5892
5185typedef struct _ASIC_ENCODER_INFO 5893typedef struct _ASIC_ENCODER_INFO
5186{ 5894{
5187 UCHAR ucEncoderID; 5895 UCHAR ucEncoderID;
@@ -5284,6 +5992,28 @@ typedef struct _DP_ENCODER_SERVICE_PARAMETERS
5284/* /obselete */ 5992/* /obselete */
5285#define DP_ENCODER_SERVICE_PS_ALLOCATION WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS 5993#define DP_ENCODER_SERVICE_PS_ALLOCATION WRITE_ONE_BYTE_HW_I2C_DATA_PARAMETERS
5286 5994
5995
5996typedef struct _DP_ENCODER_SERVICE_PARAMETERS_V2
5997{
5998 USHORT usExtEncoderObjId; // External Encoder Object Id, output parameter only, use when ucAction = DP_SERVICE_V2_ACTION_DET_EXT_CONNECTION
5999 UCHAR ucAuxId;
6000 UCHAR ucAction;
6001 UCHAR ucSinkType; // Iput and Output parameters.
6002 UCHAR ucHPDId; // Input parameter, used when ucAction = DP_SERVICE_V2_ACTION_DET_EXT_CONNECTION
6003 UCHAR ucReserved[2];
6004}DP_ENCODER_SERVICE_PARAMETERS_V2;
6005
6006typedef struct _DP_ENCODER_SERVICE_PS_ALLOCATION_V2
6007{
6008 DP_ENCODER_SERVICE_PARAMETERS_V2 asDPServiceParam;
6009 PROCESS_AUX_CHANNEL_TRANSACTION_PARAMETERS_V2 asAuxParam;
6010}DP_ENCODER_SERVICE_PS_ALLOCATION_V2;
6011
6012// ucAction
6013#define DP_SERVICE_V2_ACTION_GET_SINK_TYPE 0x01
6014#define DP_SERVICE_V2_ACTION_DET_LCD_CONNECTION 0x02
6015
6016
5287// DP_TRAINING_TABLE 6017// DP_TRAINING_TABLE
5288#define DPCD_SET_LINKRATE_LANENUM_PATTERN1_TBL_ADDR ATOM_DP_TRAINING_TBL_ADDR 6018#define DPCD_SET_LINKRATE_LANENUM_PATTERN1_TBL_ADDR ATOM_DP_TRAINING_TBL_ADDR
5289#define DPCD_SET_SS_CNTL_TBL_ADDR (ATOM_DP_TRAINING_TBL_ADDR + 8 ) 6019#define DPCD_SET_SS_CNTL_TBL_ADDR (ATOM_DP_TRAINING_TBL_ADDR + 8 )
@@ -5339,6 +6069,7 @@ typedef struct _SET_HWBLOCK_INSTANCE_PARAMETER_V2
5339#define SELECT_DCIO_IMPCAL 4 6069#define SELECT_DCIO_IMPCAL 4
5340#define SELECT_DCIO_DIG 6 6070#define SELECT_DCIO_DIG 6
5341#define SELECT_CRTC_PIXEL_RATE 7 6071#define SELECT_CRTC_PIXEL_RATE 7
6072#define SELECT_VGA_BLK 8
5342 6073
5343/****************************************************************************/ 6074/****************************************************************************/
5344//Portion VI: Definitinos for vbios MC scratch registers that driver used 6075//Portion VI: Definitinos for vbios MC scratch registers that driver used
@@ -5744,7 +6475,17 @@ typedef struct _ATOM_PPLIB_THERMALCONTROLLER
5744#define ATOM_PP_THERMALCONTROLLER_ADT7473 9 6475#define ATOM_PP_THERMALCONTROLLER_ADT7473 9
5745#define ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO 11 6476#define ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO 11
5746#define ATOM_PP_THERMALCONTROLLER_EVERGREEN 12 6477#define ATOM_PP_THERMALCONTROLLER_EVERGREEN 12
6478#define ATOM_PP_THERMALCONTROLLER_EMC2103 13 /* 0x0D */ // Only fan control will be implemented, do NOT show this in PPGen.
6479#define ATOM_PP_THERMALCONTROLLER_SUMO 14 /* 0x0E */ // Sumo type, used internally
6480#define ATOM_PP_THERMALCONTROLLER_NISLANDS 15
6481
6482// Thermal controller 'combo type' to use an external controller for Fan control and an internal controller for thermal.
6483// We probably should reserve the bit 0x80 for this use.
6484// To keep the number of these types low we should also use the same code for all ASICs (i.e. do not distinguish RV6xx and RV7xx Internal here).
6485// The driver can pick the correct internal controller based on the ASIC.
6486
5747#define ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL 0x89 // ADT7473 Fan Control + Internal Thermal Controller 6487#define ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL 0x89 // ADT7473 Fan Control + Internal Thermal Controller
6488#define ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL 0x8D // EMC2103 Fan Control + Internal Thermal Controller
5748 6489
5749typedef struct _ATOM_PPLIB_STATE 6490typedef struct _ATOM_PPLIB_STATE
5750{ 6491{
@@ -5841,6 +6582,29 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
5841 USHORT usExtendendedHeaderOffset; 6582 USHORT usExtendendedHeaderOffset;
5842} ATOM_PPLIB_POWERPLAYTABLE3, *LPATOM_PPLIB_POWERPLAYTABLE3; 6583} ATOM_PPLIB_POWERPLAYTABLE3, *LPATOM_PPLIB_POWERPLAYTABLE3;
5843 6584
6585typedef struct _ATOM_PPLIB_POWERPLAYTABLE4
6586{
6587 ATOM_PPLIB_POWERPLAYTABLE3 basicTable3;
6588 ULONG ulGoldenPPID; // PPGen use only
6589 ULONG ulGoldenRevision; // PPGen use only
6590 USHORT usVddcDependencyOnSCLKOffset;
6591 USHORT usVddciDependencyOnMCLKOffset;
6592 USHORT usVddcDependencyOnMCLKOffset;
6593 USHORT usMaxClockVoltageOnDCOffset;
6594 USHORT usReserved[2];
6595} ATOM_PPLIB_POWERPLAYTABLE4, *LPATOM_PPLIB_POWERPLAYTABLE4;
6596
6597typedef struct _ATOM_PPLIB_POWERPLAYTABLE5
6598{
6599 ATOM_PPLIB_POWERPLAYTABLE4 basicTable4;
6600 ULONG ulTDPLimit;
6601 ULONG ulNearTDPLimit;
6602 ULONG ulSQRampingThreshold;
6603 USHORT usCACLeakageTableOffset; // Points to ATOM_PPLIB_CAC_Leakage_Table
6604 ULONG ulCACLeakage; // TBD, this parameter is still under discussion. Change to ulReserved if not needed.
6605 ULONG ulReserved;
6606} ATOM_PPLIB_POWERPLAYTABLE5, *LPATOM_PPLIB_POWERPLAYTABLE5;
6607
5844//// ATOM_PPLIB_NONCLOCK_INFO::usClassification 6608//// ATOM_PPLIB_NONCLOCK_INFO::usClassification
5845#define ATOM_PPLIB_CLASSIFICATION_UI_MASK 0x0007 6609#define ATOM_PPLIB_CLASSIFICATION_UI_MASK 0x0007
5846#define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT 0 6610#define ATOM_PPLIB_CLASSIFICATION_UI_SHIFT 0
@@ -5864,6 +6628,10 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
5864#define ATOM_PPLIB_CLASSIFICATION_HDSTATE 0x4000 6628#define ATOM_PPLIB_CLASSIFICATION_HDSTATE 0x4000
5865#define ATOM_PPLIB_CLASSIFICATION_SDSTATE 0x8000 6629#define ATOM_PPLIB_CLASSIFICATION_SDSTATE 0x8000
5866 6630
6631//// ATOM_PPLIB_NONCLOCK_INFO::usClassification2
6632#define ATOM_PPLIB_CLASSIFICATION2_LIMITEDPOWERSOURCE_2 0x0001
6633#define ATOM_PPLIB_CLASSIFICATION2_ULV 0x0002
6634
5867//// ATOM_PPLIB_NONCLOCK_INFO::ulCapsAndSettings 6635//// ATOM_PPLIB_NONCLOCK_INFO::ulCapsAndSettings
5868#define ATOM_PPLIB_SINGLE_DISPLAY_ONLY 0x00000001 6636#define ATOM_PPLIB_SINGLE_DISPLAY_ONLY 0x00000001
5869#define ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK 0x00000002 6637#define ATOM_PPLIB_SUPPORTS_VIDEO_PLAYBACK 0x00000002
@@ -5896,9 +6664,21 @@ typedef struct _ATOM_PPLIB_POWERPLAYTABLE3
5896#define ATOM_PPLIB_M3ARB_MASK 0x00060000 6664#define ATOM_PPLIB_M3ARB_MASK 0x00060000
5897#define ATOM_PPLIB_M3ARB_SHIFT 17 6665#define ATOM_PPLIB_M3ARB_SHIFT 17
5898 6666
6667#define ATOM_PPLIB_ENABLE_DRR 0x00080000
6668
6669// remaining 16 bits are reserved
6670typedef struct _ATOM_PPLIB_THERMAL_STATE
6671{
6672 UCHAR ucMinTemperature;
6673 UCHAR ucMaxTemperature;
6674 UCHAR ucThermalAction;
6675}ATOM_PPLIB_THERMAL_STATE, *LPATOM_PPLIB_THERMAL_STATE;
6676
5899// Contained in an array starting at the offset 6677// Contained in an array starting at the offset
5900// in ATOM_PPLIB_POWERPLAYTABLE::usNonClockInfoArrayOffset. 6678// in ATOM_PPLIB_POWERPLAYTABLE::usNonClockInfoArrayOffset.
5901// referenced from ATOM_PPLIB_STATE_INFO::ucNonClockStateIndex 6679// referenced from ATOM_PPLIB_STATE_INFO::ucNonClockStateIndex
6680#define ATOM_PPLIB_NONCLOCKINFO_VER1 12
6681#define ATOM_PPLIB_NONCLOCKINFO_VER2 24
5902typedef struct _ATOM_PPLIB_NONCLOCK_INFO 6682typedef struct _ATOM_PPLIB_NONCLOCK_INFO
5903{ 6683{
5904 USHORT usClassification; 6684 USHORT usClassification;
@@ -5906,15 +6686,15 @@ typedef struct _ATOM_PPLIB_NONCLOCK_INFO
5906 UCHAR ucMaxTemperature; 6686 UCHAR ucMaxTemperature;
5907 ULONG ulCapsAndSettings; 6687 ULONG ulCapsAndSettings;
5908 UCHAR ucRequiredPower; 6688 UCHAR ucRequiredPower;
5909 UCHAR ucUnused1[3]; 6689 USHORT usClassification2;
6690 ULONG ulVCLK;
6691 ULONG ulDCLK;
6692 UCHAR ucUnused[5];
5910} ATOM_PPLIB_NONCLOCK_INFO; 6693} ATOM_PPLIB_NONCLOCK_INFO;
5911 6694
5912// Contained in an array starting at the offset 6695// Contained in an array starting at the offset
5913// in ATOM_PPLIB_POWERPLAYTABLE::usClockInfoArrayOffset. 6696// in ATOM_PPLIB_POWERPLAYTABLE::usClockInfoArrayOffset.
5914// referenced from ATOM_PPLIB_STATE::ucClockStateIndices 6697// referenced from ATOM_PPLIB_STATE::ucClockStateIndices
5915#define ATOM_PPLIB_NONCLOCKINFO_VER1 12
5916#define ATOM_PPLIB_NONCLOCKINFO_VER2 24
5917
5918typedef struct _ATOM_PPLIB_R600_CLOCK_INFO 6698typedef struct _ATOM_PPLIB_R600_CLOCK_INFO
5919{ 6699{
5920 USHORT usEngineClockLow; 6700 USHORT usEngineClockLow;
@@ -5985,6 +6765,93 @@ typedef struct _ATOM_PPLIB_RS780_CLOCK_INFO
5985#define ATOM_PPLIB_RS780_HTLINKFREQ_LOW 1 6765#define ATOM_PPLIB_RS780_HTLINKFREQ_LOW 1
5986#define ATOM_PPLIB_RS780_HTLINKFREQ_HIGH 2 6766#define ATOM_PPLIB_RS780_HTLINKFREQ_HIGH 2
5987 6767
6768typedef struct _ATOM_PPLIB_SUMO_CLOCK_INFO{
6769 USHORT usEngineClockLow; //clockfrequency & 0xFFFF. The unit is in 10khz
6770 UCHAR ucEngineClockHigh; //clockfrequency >> 16.
6771 UCHAR vddcIndex; //2-bit vddc index;
6772 UCHAR leakage; //please use 8-bit absolute value, not the 6-bit % value
6773 //please initalize to 0
6774 UCHAR rsv;
6775 //please initalize to 0
6776 USHORT rsv1;
6777 //please initialize to 0s
6778 ULONG rsv2[2];
6779}ATOM_PPLIB_SUMO_CLOCK_INFO;
6780
6781
6782
6783typedef struct _ATOM_PPLIB_STATE_V2
6784{
6785 //number of valid dpm levels in this state; Driver uses it to calculate the whole
6786 //size of the state: sizeof(ATOM_PPLIB_STATE_V2) + (ucNumDPMLevels - 1) * sizeof(UCHAR)
6787 UCHAR ucNumDPMLevels;
6788
6789 //a index to the array of nonClockInfos
6790 UCHAR nonClockInfoIndex;
6791 /**
6792 * Driver will read the first ucNumDPMLevels in this array
6793 */
6794 UCHAR clockInfoIndex[1];
6795} ATOM_PPLIB_STATE_V2;
6796
6797typedef struct StateArray{
6798 //how many states we have
6799 UCHAR ucNumEntries;
6800
6801 ATOM_PPLIB_STATE_V2 states[1];
6802}StateArray;
6803
6804
6805typedef struct ClockInfoArray{
6806 //how many clock levels we have
6807 UCHAR ucNumEntries;
6808
6809 //sizeof(ATOM_PPLIB_SUMO_CLOCK_INFO)
6810 UCHAR ucEntrySize;
6811
6812 //this is for Sumo
6813 ATOM_PPLIB_SUMO_CLOCK_INFO clockInfo[1];
6814}ClockInfoArray;
6815
6816typedef struct NonClockInfoArray{
6817
6818 //how many non-clock levels we have. normally should be same as number of states
6819 UCHAR ucNumEntries;
6820 //sizeof(ATOM_PPLIB_NONCLOCK_INFO)
6821 UCHAR ucEntrySize;
6822
6823 ATOM_PPLIB_NONCLOCK_INFO nonClockInfo[1];
6824}NonClockInfoArray;
6825
6826typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Record
6827{
6828 USHORT usClockLow;
6829 UCHAR ucClockHigh;
6830 USHORT usVoltage;
6831}ATOM_PPLIB_Clock_Voltage_Dependency_Record;
6832
6833typedef struct _ATOM_PPLIB_Clock_Voltage_Dependency_Table
6834{
6835 UCHAR ucNumEntries; // Number of entries.
6836 ATOM_PPLIB_Clock_Voltage_Dependency_Record entries[1]; // Dynamically allocate entries.
6837}ATOM_PPLIB_Clock_Voltage_Dependency_Table;
6838
6839typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Record
6840{
6841 USHORT usSclkLow;
6842 UCHAR ucSclkHigh;
6843 USHORT usMclkLow;
6844 UCHAR ucMclkHigh;
6845 USHORT usVddc;
6846 USHORT usVddci;
6847}ATOM_PPLIB_Clock_Voltage_Limit_Record;
6848
6849typedef struct _ATOM_PPLIB_Clock_Voltage_Limit_Table
6850{
6851 UCHAR ucNumEntries; // Number of entries.
6852 ATOM_PPLIB_Clock_Voltage_Limit_Record entries[1]; // Dynamically allocate entries.
6853}ATOM_PPLIB_Clock_Voltage_Limit_Table;
6854
5988/**************************************************************************/ 6855/**************************************************************************/
5989 6856
5990 6857
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 9fbabaa6ee44..b0ab185b86f6 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -403,6 +403,7 @@ union atom_enable_ss {
403 ENABLE_LVDS_SS_PARAMETERS_V2 lvds_ss_2; 403 ENABLE_LVDS_SS_PARAMETERS_V2 lvds_ss_2;
404 ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION v1; 404 ENABLE_SPREAD_SPECTRUM_ON_PPLL_PS_ALLOCATION v1;
405 ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2 v2; 405 ENABLE_SPREAD_SPECTRUM_ON_PPLL_V2 v2;
406 ENABLE_SPREAD_SPECTRUM_ON_PPLL_V3 v3;
406}; 407};
407 408
408static void atombios_crtc_program_ss(struct drm_crtc *crtc, 409static void atombios_crtc_program_ss(struct drm_crtc *crtc,
@@ -417,7 +418,30 @@ static void atombios_crtc_program_ss(struct drm_crtc *crtc,
417 418
418 memset(&args, 0, sizeof(args)); 419 memset(&args, 0, sizeof(args));
419 420
420 if (ASIC_IS_DCE4(rdev)) { 421 if (ASIC_IS_DCE5(rdev)) {
422 args.v3.usSpreadSpectrumAmountFrac = 0;
423 args.v3.ucSpreadSpectrumType = ss->type;
424 switch (pll_id) {
425 case ATOM_PPLL1:
426 args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P1PLL;
427 args.v3.usSpreadSpectrumAmount = ss->amount;
428 args.v3.usSpreadSpectrumStep = ss->step;
429 break;
430 case ATOM_PPLL2:
431 args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_P2PLL;
432 args.v3.usSpreadSpectrumAmount = ss->amount;
433 args.v3.usSpreadSpectrumStep = ss->step;
434 break;
435 case ATOM_DCPLL:
436 args.v3.ucSpreadSpectrumType |= ATOM_PPLL_SS_TYPE_V3_DCPLL;
437 args.v3.usSpreadSpectrumAmount = 0;
438 args.v3.usSpreadSpectrumStep = 0;
439 break;
440 case ATOM_PPLL_INVALID:
441 return;
442 }
443 args.v2.ucEnable = enable;
444 } else if (ASIC_IS_DCE4(rdev)) {
421 args.v2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage); 445 args.v2.usSpreadSpectrumPercentage = cpu_to_le16(ss->percentage);
422 args.v2.ucSpreadSpectrumType = ss->type; 446 args.v2.ucSpreadSpectrumType = ss->type;
423 switch (pll_id) { 447 switch (pll_id) {
@@ -673,9 +697,14 @@ union set_pixel_clock {
673 PIXEL_CLOCK_PARAMETERS_V2 v2; 697 PIXEL_CLOCK_PARAMETERS_V2 v2;
674 PIXEL_CLOCK_PARAMETERS_V3 v3; 698 PIXEL_CLOCK_PARAMETERS_V3 v3;
675 PIXEL_CLOCK_PARAMETERS_V5 v5; 699 PIXEL_CLOCK_PARAMETERS_V5 v5;
700 PIXEL_CLOCK_PARAMETERS_V6 v6;
676}; 701};
677 702
678static void atombios_crtc_set_dcpll(struct drm_crtc *crtc) 703/* on DCE5, make sure the voltage is high enough to support the
704 * required disp clk.
705 */
706static void atombios_crtc_set_dcpll(struct drm_crtc *crtc,
707 u32 dispclk)
679{ 708{
680 struct drm_device *dev = crtc->dev; 709 struct drm_device *dev = crtc->dev;
681 struct radeon_device *rdev = dev->dev_private; 710 struct radeon_device *rdev = dev->dev_private;
@@ -698,9 +727,16 @@ static void atombios_crtc_set_dcpll(struct drm_crtc *crtc)
698 * SetPixelClock provides the dividers 727 * SetPixelClock provides the dividers
699 */ 728 */
700 args.v5.ucCRTC = ATOM_CRTC_INVALID; 729 args.v5.ucCRTC = ATOM_CRTC_INVALID;
701 args.v5.usPixelClock = rdev->clock.default_dispclk; 730 args.v5.usPixelClock = dispclk;
702 args.v5.ucPpll = ATOM_DCPLL; 731 args.v5.ucPpll = ATOM_DCPLL;
703 break; 732 break;
733 case 6:
734 /* if the default dcpll clock is specified,
735 * SetPixelClock provides the dividers
736 */
737 args.v6.ulDispEngClkFreq = dispclk;
738 args.v6.ucPpll = ATOM_DCPLL;
739 break;
704 default: 740 default:
705 DRM_ERROR("Unknown table version %d %d\n", frev, crev); 741 DRM_ERROR("Unknown table version %d %d\n", frev, crev);
706 return; 742 return;
@@ -784,6 +820,18 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
784 args.v5.ucEncoderMode = encoder_mode; 820 args.v5.ucEncoderMode = encoder_mode;
785 args.v5.ucPpll = pll_id; 821 args.v5.ucPpll = pll_id;
786 break; 822 break;
823 case 6:
824 args.v6.ulCrtcPclkFreq.ucCRTC = crtc_id;
825 args.v6.ulCrtcPclkFreq.ulPixelClock = cpu_to_le32(clock / 10);
826 args.v6.ucRefDiv = ref_div;
827 args.v6.usFbDiv = cpu_to_le16(fb_div);
828 args.v6.ulFbDivDecFrac = cpu_to_le32(frac_fb_div * 100000);
829 args.v6.ucPostDiv = post_div;
830 args.v6.ucMiscInfo = 0; /* HDMI depth, etc. */
831 args.v6.ucTransmitterID = encoder_id;
832 args.v6.ucEncoderMode = encoder_mode;
833 args.v6.ucPpll = pll_id;
834 break;
787 default: 835 default:
788 DRM_ERROR("Unknown table version %d %d\n", frev, crev); 836 DRM_ERROR("Unknown table version %d %d\n", frev, crev);
789 return; 837 return;
@@ -1377,7 +1425,8 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc,
1377 rdev->clock.default_dispclk); 1425 rdev->clock.default_dispclk);
1378 if (ss_enabled) 1426 if (ss_enabled)
1379 atombios_crtc_program_ss(crtc, ATOM_DISABLE, ATOM_DCPLL, &ss); 1427 atombios_crtc_program_ss(crtc, ATOM_DISABLE, ATOM_DCPLL, &ss);
1380 atombios_crtc_set_dcpll(crtc); 1428 /* XXX: DCE5, make sure voltage, dispclk is high enough */
1429 atombios_crtc_set_dcpll(crtc, rdev->clock.default_dispclk);
1381 if (ss_enabled) 1430 if (ss_enabled)
1382 atombios_crtc_program_ss(crtc, ATOM_ENABLE, ATOM_DCPLL, &ss); 1431 atombios_crtc_program_ss(crtc, ATOM_ENABLE, ATOM_DCPLL, &ss);
1383 } 1432 }
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 7b337c361a12..7fe8ebdcdc0e 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -39,6 +39,62 @@
39 39
40static void evergreen_gpu_init(struct radeon_device *rdev); 40static void evergreen_gpu_init(struct radeon_device *rdev);
41void evergreen_fini(struct radeon_device *rdev); 41void evergreen_fini(struct radeon_device *rdev);
42static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
43
44void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
45{
46 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
47 u32 tmp;
48
49 /* make sure flip is at vb rather than hb */
50 tmp = RREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset);
51 tmp &= ~EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN;
52 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp);
53
54 /* set pageflip to happen anywhere in vblank interval */
55 WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
56
57 /* enable the pflip int */
58 radeon_irq_kms_pflip_irq_get(rdev, crtc);
59}
60
61void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
62{
63 /* disable the pflip int */
64 radeon_irq_kms_pflip_irq_put(rdev, crtc);
65}
66
67u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
68{
69 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
70 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
71
72 /* Lock the graphics update lock */
73 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
74 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
75
76 /* update the scanout addresses */
77 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
78 upper_32_bits(crtc_base));
79 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
80 (u32)crtc_base);
81
82 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
83 upper_32_bits(crtc_base));
84 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
85 (u32)crtc_base);
86
87 /* Wait for update_pending to go high. */
88 while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
89 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
90
91 /* Unlock the lock, so double-buffering can take place inside vblank */
92 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
93 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
94
95 /* Return current update_pending status: */
96 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
97}
42 98
43/* get temperature in millidegrees */ 99/* get temperature in millidegrees */
44u32 evergreen_get_temp(struct radeon_device *rdev) 100u32 evergreen_get_temp(struct radeon_device *rdev)
@@ -57,6 +113,14 @@ u32 evergreen_get_temp(struct radeon_device *rdev)
57 return actual_temp * 1000; 113 return actual_temp * 1000;
58} 114}
59 115
116u32 sumo_get_temp(struct radeon_device *rdev)
117{
118 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
119 u32 actual_temp = (temp >> 1) & 0xff;
120
121 return actual_temp * 1000;
122}
123
60void evergreen_pm_misc(struct radeon_device *rdev) 124void evergreen_pm_misc(struct radeon_device *rdev)
61{ 125{
62 int req_ps_idx = rdev->pm.requested_power_state_index; 126 int req_ps_idx = rdev->pm.requested_power_state_index;
@@ -337,16 +401,28 @@ static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
337 case 0: 401 case 0:
338 case 4: 402 case 4:
339 default: 403 default:
340 return 3840 * 2; 404 if (ASIC_IS_DCE5(rdev))
405 return 4096 * 2;
406 else
407 return 3840 * 2;
341 case 1: 408 case 1:
342 case 5: 409 case 5:
343 return 5760 * 2; 410 if (ASIC_IS_DCE5(rdev))
411 return 6144 * 2;
412 else
413 return 5760 * 2;
344 case 2: 414 case 2:
345 case 6: 415 case 6:
346 return 7680 * 2; 416 if (ASIC_IS_DCE5(rdev))
417 return 8192 * 2;
418 else
419 return 7680 * 2;
347 case 3: 420 case 3:
348 case 7: 421 case 7:
349 return 1920 * 2; 422 if (ASIC_IS_DCE5(rdev))
423 return 2048 * 2;
424 else
425 return 1920 * 2;
350 } 426 }
351} 427}
352 428
@@ -890,31 +966,39 @@ static void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_sa
890 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 966 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
891 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 967 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
892 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 968 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
893 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 969 if (!(rdev->flags & RADEON_IS_IGP)) {
894 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 970 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
895 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 971 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
896 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 972 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
973 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
974 }
897 975
898 /* Stop all video */ 976 /* Stop all video */
899 WREG32(VGA_RENDER_CONTROL, 0); 977 WREG32(VGA_RENDER_CONTROL, 0);
900 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1); 978 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
901 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1); 979 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
902 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1); 980 if (!(rdev->flags & RADEON_IS_IGP)) {
903 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1); 981 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
904 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1); 982 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
905 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1); 983 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
984 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
985 }
906 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 986 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
907 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 987 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
908 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 988 if (!(rdev->flags & RADEON_IS_IGP)) {
909 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 989 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
910 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 990 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
911 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 991 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
992 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
993 }
912 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 994 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
913 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 995 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
914 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 996 if (!(rdev->flags & RADEON_IS_IGP)) {
915 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 997 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
916 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 998 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
917 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 999 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1000 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1001 }
918 1002
919 WREG32(D1VGA_CONTROL, 0); 1003 WREG32(D1VGA_CONTROL, 0);
920 WREG32(D2VGA_CONTROL, 0); 1004 WREG32(D2VGA_CONTROL, 0);
@@ -944,41 +1028,43 @@ static void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_
944 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET, 1028 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
945 (u32)rdev->mc.vram_start); 1029 (u32)rdev->mc.vram_start);
946 1030
947 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET, 1031 if (!(rdev->flags & RADEON_IS_IGP)) {
948 upper_32_bits(rdev->mc.vram_start)); 1032 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
949 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET, 1033 upper_32_bits(rdev->mc.vram_start));
950 upper_32_bits(rdev->mc.vram_start)); 1034 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
951 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET, 1035 upper_32_bits(rdev->mc.vram_start));
952 (u32)rdev->mc.vram_start); 1036 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
953 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET, 1037 (u32)rdev->mc.vram_start);
954 (u32)rdev->mc.vram_start); 1038 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
955 1039 (u32)rdev->mc.vram_start);
956 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET, 1040
957 upper_32_bits(rdev->mc.vram_start)); 1041 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
958 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET, 1042 upper_32_bits(rdev->mc.vram_start));
959 upper_32_bits(rdev->mc.vram_start)); 1043 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
960 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET, 1044 upper_32_bits(rdev->mc.vram_start));
961 (u32)rdev->mc.vram_start); 1045 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
962 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET, 1046 (u32)rdev->mc.vram_start);
963 (u32)rdev->mc.vram_start); 1047 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
964 1048 (u32)rdev->mc.vram_start);
965 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET, 1049
966 upper_32_bits(rdev->mc.vram_start)); 1050 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
967 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET, 1051 upper_32_bits(rdev->mc.vram_start));
968 upper_32_bits(rdev->mc.vram_start)); 1052 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
969 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET, 1053 upper_32_bits(rdev->mc.vram_start));
970 (u32)rdev->mc.vram_start); 1054 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
971 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET, 1055 (u32)rdev->mc.vram_start);
972 (u32)rdev->mc.vram_start); 1056 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
973 1057 (u32)rdev->mc.vram_start);
974 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET, 1058
975 upper_32_bits(rdev->mc.vram_start)); 1059 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
976 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET, 1060 upper_32_bits(rdev->mc.vram_start));
977 upper_32_bits(rdev->mc.vram_start)); 1061 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
978 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET, 1062 upper_32_bits(rdev->mc.vram_start));
979 (u32)rdev->mc.vram_start); 1063 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
980 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET, 1064 (u32)rdev->mc.vram_start);
981 (u32)rdev->mc.vram_start); 1065 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1066 (u32)rdev->mc.vram_start);
1067 }
982 1068
983 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 1069 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
984 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 1070 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
@@ -994,22 +1080,28 @@ static void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_
994 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]); 1080 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
995 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1); 1081 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
996 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1); 1082 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
997 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1); 1083 if (!(rdev->flags & RADEON_IS_IGP)) {
998 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1); 1084 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
999 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1); 1085 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
1000 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1); 1086 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1087 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1088 }
1001 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]); 1089 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1002 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]); 1090 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1003 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]); 1091 if (!(rdev->flags & RADEON_IS_IGP)) {
1004 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]); 1092 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1005 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]); 1093 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
1006 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]); 1094 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1095 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1096 }
1007 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 1097 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1008 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 1098 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1009 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 1099 if (!(rdev->flags & RADEON_IS_IGP)) {
1010 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 1100 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1011 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 1101 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
1012 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 1102 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1103 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1104 }
1013 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 1105 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1014} 1106}
1015 1107
@@ -1057,11 +1149,17 @@ static void evergreen_mc_program(struct radeon_device *rdev)
1057 rdev->mc.vram_end >> 12); 1149 rdev->mc.vram_end >> 12);
1058 } 1150 }
1059 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0); 1151 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1152 if (rdev->flags & RADEON_IS_IGP) {
1153 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1154 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1155 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1156 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1157 }
1060 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 1158 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1061 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 1159 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1062 WREG32(MC_VM_FB_LOCATION, tmp); 1160 WREG32(MC_VM_FB_LOCATION, tmp);
1063 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 1161 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1064 WREG32(HDP_NONSURFACE_INFO, (2 << 7)); 1162 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1065 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 1163 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1066 if (rdev->flags & RADEON_IS_AGP) { 1164 if (rdev->flags & RADEON_IS_AGP) {
1067 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 1165 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
@@ -1285,11 +1383,15 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1285 switch (rdev->family) { 1383 switch (rdev->family) {
1286 case CHIP_CEDAR: 1384 case CHIP_CEDAR:
1287 case CHIP_REDWOOD: 1385 case CHIP_REDWOOD:
1386 case CHIP_PALM:
1387 case CHIP_TURKS:
1388 case CHIP_CAICOS:
1288 force_no_swizzle = false; 1389 force_no_swizzle = false;
1289 break; 1390 break;
1290 case CHIP_CYPRESS: 1391 case CHIP_CYPRESS:
1291 case CHIP_HEMLOCK: 1392 case CHIP_HEMLOCK:
1292 case CHIP_JUNIPER: 1393 case CHIP_JUNIPER:
1394 case CHIP_BARTS:
1293 default: 1395 default:
1294 force_no_swizzle = true; 1396 force_no_swizzle = true;
1295 break; 1397 break;
@@ -1384,6 +1486,46 @@ static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1384 return backend_map; 1486 return backend_map;
1385} 1487}
1386 1488
1489static void evergreen_program_channel_remap(struct radeon_device *rdev)
1490{
1491 u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
1492
1493 tmp = RREG32(MC_SHARED_CHMAP);
1494 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1495 case 0:
1496 case 1:
1497 case 2:
1498 case 3:
1499 default:
1500 /* default mapping */
1501 mc_shared_chremap = 0x00fac688;
1502 break;
1503 }
1504
1505 switch (rdev->family) {
1506 case CHIP_HEMLOCK:
1507 case CHIP_CYPRESS:
1508 case CHIP_BARTS:
1509 tcp_chan_steer_lo = 0x54763210;
1510 tcp_chan_steer_hi = 0x0000ba98;
1511 break;
1512 case CHIP_JUNIPER:
1513 case CHIP_REDWOOD:
1514 case CHIP_CEDAR:
1515 case CHIP_PALM:
1516 case CHIP_TURKS:
1517 case CHIP_CAICOS:
1518 default:
1519 tcp_chan_steer_lo = 0x76543210;
1520 tcp_chan_steer_hi = 0x0000ba98;
1521 break;
1522 }
1523
1524 WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
1525 WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
1526 WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
1527}
1528
1387static void evergreen_gpu_init(struct radeon_device *rdev) 1529static void evergreen_gpu_init(struct radeon_device *rdev)
1388{ 1530{
1389 u32 cc_rb_backend_disable = 0; 1531 u32 cc_rb_backend_disable = 0;
@@ -1495,6 +1637,90 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1495 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1637 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1496 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1638 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1497 break; 1639 break;
1640 case CHIP_PALM:
1641 rdev->config.evergreen.num_ses = 1;
1642 rdev->config.evergreen.max_pipes = 2;
1643 rdev->config.evergreen.max_tile_pipes = 2;
1644 rdev->config.evergreen.max_simds = 2;
1645 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1646 rdev->config.evergreen.max_gprs = 256;
1647 rdev->config.evergreen.max_threads = 192;
1648 rdev->config.evergreen.max_gs_threads = 16;
1649 rdev->config.evergreen.max_stack_entries = 256;
1650 rdev->config.evergreen.sx_num_of_sets = 4;
1651 rdev->config.evergreen.sx_max_export_size = 128;
1652 rdev->config.evergreen.sx_max_export_pos_size = 32;
1653 rdev->config.evergreen.sx_max_export_smx_size = 96;
1654 rdev->config.evergreen.max_hw_contexts = 4;
1655 rdev->config.evergreen.sq_num_cf_insts = 1;
1656
1657 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1658 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1659 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1660 break;
1661 case CHIP_BARTS:
1662 rdev->config.evergreen.num_ses = 2;
1663 rdev->config.evergreen.max_pipes = 4;
1664 rdev->config.evergreen.max_tile_pipes = 8;
1665 rdev->config.evergreen.max_simds = 7;
1666 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1667 rdev->config.evergreen.max_gprs = 256;
1668 rdev->config.evergreen.max_threads = 248;
1669 rdev->config.evergreen.max_gs_threads = 32;
1670 rdev->config.evergreen.max_stack_entries = 512;
1671 rdev->config.evergreen.sx_num_of_sets = 4;
1672 rdev->config.evergreen.sx_max_export_size = 256;
1673 rdev->config.evergreen.sx_max_export_pos_size = 64;
1674 rdev->config.evergreen.sx_max_export_smx_size = 192;
1675 rdev->config.evergreen.max_hw_contexts = 8;
1676 rdev->config.evergreen.sq_num_cf_insts = 2;
1677
1678 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1679 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1680 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1681 break;
1682 case CHIP_TURKS:
1683 rdev->config.evergreen.num_ses = 1;
1684 rdev->config.evergreen.max_pipes = 4;
1685 rdev->config.evergreen.max_tile_pipes = 4;
1686 rdev->config.evergreen.max_simds = 6;
1687 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1688 rdev->config.evergreen.max_gprs = 256;
1689 rdev->config.evergreen.max_threads = 248;
1690 rdev->config.evergreen.max_gs_threads = 32;
1691 rdev->config.evergreen.max_stack_entries = 256;
1692 rdev->config.evergreen.sx_num_of_sets = 4;
1693 rdev->config.evergreen.sx_max_export_size = 256;
1694 rdev->config.evergreen.sx_max_export_pos_size = 64;
1695 rdev->config.evergreen.sx_max_export_smx_size = 192;
1696 rdev->config.evergreen.max_hw_contexts = 8;
1697 rdev->config.evergreen.sq_num_cf_insts = 2;
1698
1699 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1700 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1701 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1702 break;
1703 case CHIP_CAICOS:
1704 rdev->config.evergreen.num_ses = 1;
1705 rdev->config.evergreen.max_pipes = 4;
1706 rdev->config.evergreen.max_tile_pipes = 2;
1707 rdev->config.evergreen.max_simds = 2;
1708 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1709 rdev->config.evergreen.max_gprs = 256;
1710 rdev->config.evergreen.max_threads = 192;
1711 rdev->config.evergreen.max_gs_threads = 16;
1712 rdev->config.evergreen.max_stack_entries = 256;
1713 rdev->config.evergreen.sx_num_of_sets = 4;
1714 rdev->config.evergreen.sx_max_export_size = 128;
1715 rdev->config.evergreen.sx_max_export_pos_size = 32;
1716 rdev->config.evergreen.sx_max_export_smx_size = 96;
1717 rdev->config.evergreen.max_hw_contexts = 4;
1718 rdev->config.evergreen.sq_num_cf_insts = 1;
1719
1720 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1721 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1722 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1723 break;
1498 } 1724 }
1499 1725
1500 /* Initialize HDP */ 1726 /* Initialize HDP */
@@ -1636,6 +1862,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1636 switch (rdev->family) { 1862 switch (rdev->family) {
1637 case CHIP_CYPRESS: 1863 case CHIP_CYPRESS:
1638 case CHIP_HEMLOCK: 1864 case CHIP_HEMLOCK:
1865 case CHIP_BARTS:
1639 gb_backend_map = 0x66442200; 1866 gb_backend_map = 0x66442200;
1640 break; 1867 break;
1641 case CHIP_JUNIPER: 1868 case CHIP_JUNIPER:
@@ -1687,6 +1914,8 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1687 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 1914 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1688 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 1915 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1689 1916
1917 evergreen_program_channel_remap(rdev);
1918
1690 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1; 1919 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
1691 grbm_gfx_index = INSTANCE_BROADCAST_WRITES; 1920 grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
1692 1921
@@ -1769,9 +1998,16 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1769 GS_PRIO(2) | 1998 GS_PRIO(2) |
1770 ES_PRIO(3)); 1999 ES_PRIO(3));
1771 2000
1772 if (rdev->family == CHIP_CEDAR) 2001 switch (rdev->family) {
2002 case CHIP_CEDAR:
2003 case CHIP_PALM:
2004 case CHIP_CAICOS:
1773 /* no vertex cache */ 2005 /* no vertex cache */
1774 sq_config &= ~VC_ENABLE; 2006 sq_config &= ~VC_ENABLE;
2007 break;
2008 default:
2009 break;
2010 }
1775 2011
1776 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 2012 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
1777 2013
@@ -1783,10 +2019,15 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1783 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2019 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
1784 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2020 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
1785 2021
1786 if (rdev->family == CHIP_CEDAR) 2022 switch (rdev->family) {
2023 case CHIP_CEDAR:
2024 case CHIP_PALM:
1787 ps_thread_count = 96; 2025 ps_thread_count = 96;
1788 else 2026 break;
2027 default:
1789 ps_thread_count = 128; 2028 ps_thread_count = 128;
2029 break;
2030 }
1790 2031
1791 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 2032 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
1792 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2033 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
@@ -1817,10 +2058,16 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
1817 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 2058 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1818 FORCE_EOV_MAX_REZ_CNT(255))); 2059 FORCE_EOV_MAX_REZ_CNT(255)));
1819 2060
1820 if (rdev->family == CHIP_CEDAR) 2061 switch (rdev->family) {
2062 case CHIP_CEDAR:
2063 case CHIP_PALM:
2064 case CHIP_CAICOS:
1821 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 2065 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
1822 else 2066 break;
2067 default:
1823 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 2068 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2069 break;
2070 }
1824 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 2071 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
1825 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 2072 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
1826 2073
@@ -1904,12 +2151,18 @@ int evergreen_mc_init(struct radeon_device *rdev)
1904 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 2151 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1905 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 2152 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1906 /* Setup GPU memory space */ 2153 /* Setup GPU memory space */
1907 /* size in MB on evergreen */ 2154 if (rdev->flags & RADEON_IS_IGP) {
1908 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2155 /* size in bytes on fusion */
1909 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2156 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2157 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2158 } else {
2159 /* size in MB on evergreen */
2160 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2161 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2162 }
1910 rdev->mc.visible_vram_size = rdev->mc.aper_size; 2163 rdev->mc.visible_vram_size = rdev->mc.aper_size;
1911 rdev->mc.active_vram_size = rdev->mc.visible_vram_size; 2164 rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
1912 r600_vram_gtt_location(rdev, &rdev->mc); 2165 r700_vram_gtt_location(rdev, &rdev->mc);
1913 radeon_update_bandwidth_info(rdev); 2166 radeon_update_bandwidth_info(rdev);
1914 2167
1915 return 0; 2168 return 0;
@@ -1917,8 +2170,30 @@ int evergreen_mc_init(struct radeon_device *rdev)
1917 2170
1918bool evergreen_gpu_is_lockup(struct radeon_device *rdev) 2171bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1919{ 2172{
1920 /* FIXME: implement for evergreen */ 2173 u32 srbm_status;
1921 return false; 2174 u32 grbm_status;
2175 u32 grbm_status_se0, grbm_status_se1;
2176 struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2177 int r;
2178
2179 srbm_status = RREG32(SRBM_STATUS);
2180 grbm_status = RREG32(GRBM_STATUS);
2181 grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2182 grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2183 if (!(grbm_status & GUI_ACTIVE)) {
2184 r100_gpu_lockup_update(lockup, &rdev->cp);
2185 return false;
2186 }
2187 /* force CP activities */
2188 r = radeon_ring_lock(rdev, 2);
2189 if (!r) {
2190 /* PACKET2 NOP */
2191 radeon_ring_write(rdev, 0x80000000);
2192 radeon_ring_write(rdev, 0x80000000);
2193 radeon_ring_unlock_commit(rdev);
2194 }
2195 rdev->cp.rptr = RREG32(CP_RB_RPTR);
2196 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1922} 2197}
1923 2198
1924static int evergreen_gpu_soft_reset(struct radeon_device *rdev) 2199static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
@@ -2011,17 +2286,21 @@ void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2011 WREG32(GRBM_INT_CNTL, 0); 2286 WREG32(GRBM_INT_CNTL, 0);
2012 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2287 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2013 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2288 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2014 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2289 if (!(rdev->flags & RADEON_IS_IGP)) {
2015 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2290 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2016 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2291 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2017 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2292 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2293 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2294 }
2018 2295
2019 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2296 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2020 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2297 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2021 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2298 if (!(rdev->flags & RADEON_IS_IGP)) {
2022 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2299 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2023 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2300 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2024 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2301 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2302 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2303 }
2025 2304
2026 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 2305 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2027 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 2306 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
@@ -2047,6 +2326,7 @@ int evergreen_irq_set(struct radeon_device *rdev)
2047 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 2326 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2048 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 2327 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2049 u32 grbm_int_cntl = 0; 2328 u32 grbm_int_cntl = 0;
2329 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2050 2330
2051 if (!rdev->irq.installed) { 2331 if (!rdev->irq.installed) {
2052 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 2332 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -2072,27 +2352,33 @@ int evergreen_irq_set(struct radeon_device *rdev)
2072 cp_int_cntl |= RB_INT_ENABLE; 2352 cp_int_cntl |= RB_INT_ENABLE;
2073 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 2353 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2074 } 2354 }
2075 if (rdev->irq.crtc_vblank_int[0]) { 2355 if (rdev->irq.crtc_vblank_int[0] ||
2356 rdev->irq.pflip[0]) {
2076 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 2357 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2077 crtc1 |= VBLANK_INT_MASK; 2358 crtc1 |= VBLANK_INT_MASK;
2078 } 2359 }
2079 if (rdev->irq.crtc_vblank_int[1]) { 2360 if (rdev->irq.crtc_vblank_int[1] ||
2361 rdev->irq.pflip[1]) {
2080 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 2362 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2081 crtc2 |= VBLANK_INT_MASK; 2363 crtc2 |= VBLANK_INT_MASK;
2082 } 2364 }
2083 if (rdev->irq.crtc_vblank_int[2]) { 2365 if (rdev->irq.crtc_vblank_int[2] ||
2366 rdev->irq.pflip[2]) {
2084 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 2367 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2085 crtc3 |= VBLANK_INT_MASK; 2368 crtc3 |= VBLANK_INT_MASK;
2086 } 2369 }
2087 if (rdev->irq.crtc_vblank_int[3]) { 2370 if (rdev->irq.crtc_vblank_int[3] ||
2371 rdev->irq.pflip[3]) {
2088 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 2372 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2089 crtc4 |= VBLANK_INT_MASK; 2373 crtc4 |= VBLANK_INT_MASK;
2090 } 2374 }
2091 if (rdev->irq.crtc_vblank_int[4]) { 2375 if (rdev->irq.crtc_vblank_int[4] ||
2376 rdev->irq.pflip[4]) {
2092 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 2377 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2093 crtc5 |= VBLANK_INT_MASK; 2378 crtc5 |= VBLANK_INT_MASK;
2094 } 2379 }
2095 if (rdev->irq.crtc_vblank_int[5]) { 2380 if (rdev->irq.crtc_vblank_int[5] ||
2381 rdev->irq.pflip[5]) {
2096 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 2382 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2097 crtc6 |= VBLANK_INT_MASK; 2383 crtc6 |= VBLANK_INT_MASK;
2098 } 2384 }
@@ -2130,10 +2416,19 @@ int evergreen_irq_set(struct radeon_device *rdev)
2130 2416
2131 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 2417 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2132 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 2418 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2133 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 2419 if (!(rdev->flags & RADEON_IS_IGP)) {
2134 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 2420 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2135 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 2421 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2136 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 2422 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2423 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2424 }
2425
2426 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2427 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2428 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2429 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2430 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2431 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2137 2432
2138 WREG32(DC_HPD1_INT_CONTROL, hpd1); 2433 WREG32(DC_HPD1_INT_CONTROL, hpd1);
2139 WREG32(DC_HPD2_INT_CONTROL, hpd2); 2434 WREG32(DC_HPD2_INT_CONTROL, hpd2);
@@ -2145,79 +2440,92 @@ int evergreen_irq_set(struct radeon_device *rdev)
2145 return 0; 2440 return 0;
2146} 2441}
2147 2442
2148static inline void evergreen_irq_ack(struct radeon_device *rdev, 2443static inline void evergreen_irq_ack(struct radeon_device *rdev)
2149 u32 *disp_int,
2150 u32 *disp_int_cont,
2151 u32 *disp_int_cont2,
2152 u32 *disp_int_cont3,
2153 u32 *disp_int_cont4,
2154 u32 *disp_int_cont5)
2155{ 2444{
2156 u32 tmp; 2445 u32 tmp;
2157 2446
2158 *disp_int = RREG32(DISP_INTERRUPT_STATUS); 2447 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2159 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 2448 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2160 *disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 2449 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2161 *disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 2450 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2162 *disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 2451 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2163 *disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 2452 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2164 2453 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2165 if (*disp_int & LB_D1_VBLANK_INTERRUPT) 2454 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2455 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2456 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2457 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2458 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2459
2460 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2461 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2462 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2463 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2464 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2465 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2466 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2467 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2468 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2469 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2470 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2471 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2472
2473 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2166 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 2474 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2167 if (*disp_int & LB_D1_VLINE_INTERRUPT) 2475 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2168 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 2476 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2169 2477
2170 if (*disp_int_cont & LB_D2_VBLANK_INTERRUPT) 2478 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2171 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 2479 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2172 if (*disp_int_cont & LB_D2_VLINE_INTERRUPT) 2480 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2173 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 2481 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2174 2482
2175 if (*disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 2483 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2176 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 2484 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2177 if (*disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 2485 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2178 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 2486 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2179 2487
2180 if (*disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 2488 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2181 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 2489 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2182 if (*disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 2490 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2183 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 2491 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2184 2492
2185 if (*disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 2493 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2186 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 2494 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2187 if (*disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 2495 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2188 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 2496 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2189 2497
2190 if (*disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 2498 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2191 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 2499 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2192 if (*disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 2500 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2193 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 2501 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2194 2502
2195 if (*disp_int & DC_HPD1_INTERRUPT) { 2503 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2196 tmp = RREG32(DC_HPD1_INT_CONTROL); 2504 tmp = RREG32(DC_HPD1_INT_CONTROL);
2197 tmp |= DC_HPDx_INT_ACK; 2505 tmp |= DC_HPDx_INT_ACK;
2198 WREG32(DC_HPD1_INT_CONTROL, tmp); 2506 WREG32(DC_HPD1_INT_CONTROL, tmp);
2199 } 2507 }
2200 if (*disp_int_cont & DC_HPD2_INTERRUPT) { 2508 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2201 tmp = RREG32(DC_HPD2_INT_CONTROL); 2509 tmp = RREG32(DC_HPD2_INT_CONTROL);
2202 tmp |= DC_HPDx_INT_ACK; 2510 tmp |= DC_HPDx_INT_ACK;
2203 WREG32(DC_HPD2_INT_CONTROL, tmp); 2511 WREG32(DC_HPD2_INT_CONTROL, tmp);
2204 } 2512 }
2205 if (*disp_int_cont2 & DC_HPD3_INTERRUPT) { 2513 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2206 tmp = RREG32(DC_HPD3_INT_CONTROL); 2514 tmp = RREG32(DC_HPD3_INT_CONTROL);
2207 tmp |= DC_HPDx_INT_ACK; 2515 tmp |= DC_HPDx_INT_ACK;
2208 WREG32(DC_HPD3_INT_CONTROL, tmp); 2516 WREG32(DC_HPD3_INT_CONTROL, tmp);
2209 } 2517 }
2210 if (*disp_int_cont3 & DC_HPD4_INTERRUPT) { 2518 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2211 tmp = RREG32(DC_HPD4_INT_CONTROL); 2519 tmp = RREG32(DC_HPD4_INT_CONTROL);
2212 tmp |= DC_HPDx_INT_ACK; 2520 tmp |= DC_HPDx_INT_ACK;
2213 WREG32(DC_HPD4_INT_CONTROL, tmp); 2521 WREG32(DC_HPD4_INT_CONTROL, tmp);
2214 } 2522 }
2215 if (*disp_int_cont4 & DC_HPD5_INTERRUPT) { 2523 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2216 tmp = RREG32(DC_HPD5_INT_CONTROL); 2524 tmp = RREG32(DC_HPD5_INT_CONTROL);
2217 tmp |= DC_HPDx_INT_ACK; 2525 tmp |= DC_HPDx_INT_ACK;
2218 WREG32(DC_HPD5_INT_CONTROL, tmp); 2526 WREG32(DC_HPD5_INT_CONTROL, tmp);
2219 } 2527 }
2220 if (*disp_int_cont5 & DC_HPD6_INTERRUPT) { 2528 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2221 tmp = RREG32(DC_HPD5_INT_CONTROL); 2529 tmp = RREG32(DC_HPD5_INT_CONTROL);
2222 tmp |= DC_HPDx_INT_ACK; 2530 tmp |= DC_HPDx_INT_ACK;
2223 WREG32(DC_HPD6_INT_CONTROL, tmp); 2531 WREG32(DC_HPD6_INT_CONTROL, tmp);
@@ -2226,14 +2534,10 @@ static inline void evergreen_irq_ack(struct radeon_device *rdev,
2226 2534
2227void evergreen_irq_disable(struct radeon_device *rdev) 2535void evergreen_irq_disable(struct radeon_device *rdev)
2228{ 2536{
2229 u32 disp_int, disp_int_cont, disp_int_cont2;
2230 u32 disp_int_cont3, disp_int_cont4, disp_int_cont5;
2231
2232 r600_disable_interrupts(rdev); 2537 r600_disable_interrupts(rdev);
2233 /* Wait and acknowledge irq */ 2538 /* Wait and acknowledge irq */
2234 mdelay(1); 2539 mdelay(1);
2235 evergreen_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2, 2540 evergreen_irq_ack(rdev);
2236 &disp_int_cont3, &disp_int_cont4, &disp_int_cont5);
2237 evergreen_disable_interrupt_state(rdev); 2541 evergreen_disable_interrupt_state(rdev);
2238} 2542}
2239 2543
@@ -2273,8 +2577,6 @@ int evergreen_irq_process(struct radeon_device *rdev)
2273 u32 rptr = rdev->ih.rptr; 2577 u32 rptr = rdev->ih.rptr;
2274 u32 src_id, src_data; 2578 u32 src_id, src_data;
2275 u32 ring_index; 2579 u32 ring_index;
2276 u32 disp_int, disp_int_cont, disp_int_cont2;
2277 u32 disp_int_cont3, disp_int_cont4, disp_int_cont5;
2278 unsigned long flags; 2580 unsigned long flags;
2279 bool queue_hotplug = false; 2581 bool queue_hotplug = false;
2280 2582
@@ -2295,8 +2597,7 @@ int evergreen_irq_process(struct radeon_device *rdev)
2295 2597
2296restart_ih: 2598restart_ih:
2297 /* display interrupts */ 2599 /* display interrupts */
2298 evergreen_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2, 2600 evergreen_irq_ack(rdev);
2299 &disp_int_cont3, &disp_int_cont4, &disp_int_cont5);
2300 2601
2301 rdev->ih.wptr = wptr; 2602 rdev->ih.wptr = wptr;
2302 while (rptr != wptr) { 2603 while (rptr != wptr) {
@@ -2309,17 +2610,21 @@ restart_ih:
2309 case 1: /* D1 vblank/vline */ 2610 case 1: /* D1 vblank/vline */
2310 switch (src_data) { 2611 switch (src_data) {
2311 case 0: /* D1 vblank */ 2612 case 0: /* D1 vblank */
2312 if (disp_int & LB_D1_VBLANK_INTERRUPT) { 2613 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2313 drm_handle_vblank(rdev->ddev, 0); 2614 if (rdev->irq.crtc_vblank_int[0]) {
2314 rdev->pm.vblank_sync = true; 2615 drm_handle_vblank(rdev->ddev, 0);
2315 wake_up(&rdev->irq.vblank_queue); 2616 rdev->pm.vblank_sync = true;
2316 disp_int &= ~LB_D1_VBLANK_INTERRUPT; 2617 wake_up(&rdev->irq.vblank_queue);
2618 }
2619 if (rdev->irq.pflip[0])
2620 radeon_crtc_handle_flip(rdev, 0);
2621 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2317 DRM_DEBUG("IH: D1 vblank\n"); 2622 DRM_DEBUG("IH: D1 vblank\n");
2318 } 2623 }
2319 break; 2624 break;
2320 case 1: /* D1 vline */ 2625 case 1: /* D1 vline */
2321 if (disp_int & LB_D1_VLINE_INTERRUPT) { 2626 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2322 disp_int &= ~LB_D1_VLINE_INTERRUPT; 2627 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2323 DRM_DEBUG("IH: D1 vline\n"); 2628 DRM_DEBUG("IH: D1 vline\n");
2324 } 2629 }
2325 break; 2630 break;
@@ -2331,17 +2636,21 @@ restart_ih:
2331 case 2: /* D2 vblank/vline */ 2636 case 2: /* D2 vblank/vline */
2332 switch (src_data) { 2637 switch (src_data) {
2333 case 0: /* D2 vblank */ 2638 case 0: /* D2 vblank */
2334 if (disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 2639 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2335 drm_handle_vblank(rdev->ddev, 1); 2640 if (rdev->irq.crtc_vblank_int[1]) {
2336 rdev->pm.vblank_sync = true; 2641 drm_handle_vblank(rdev->ddev, 1);
2337 wake_up(&rdev->irq.vblank_queue); 2642 rdev->pm.vblank_sync = true;
2338 disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 2643 wake_up(&rdev->irq.vblank_queue);
2644 }
2645 if (rdev->irq.pflip[1])
2646 radeon_crtc_handle_flip(rdev, 1);
2647 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2339 DRM_DEBUG("IH: D2 vblank\n"); 2648 DRM_DEBUG("IH: D2 vblank\n");
2340 } 2649 }
2341 break; 2650 break;
2342 case 1: /* D2 vline */ 2651 case 1: /* D2 vline */
2343 if (disp_int_cont & LB_D2_VLINE_INTERRUPT) { 2652 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2344 disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 2653 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2345 DRM_DEBUG("IH: D2 vline\n"); 2654 DRM_DEBUG("IH: D2 vline\n");
2346 } 2655 }
2347 break; 2656 break;
@@ -2353,17 +2662,21 @@ restart_ih:
2353 case 3: /* D3 vblank/vline */ 2662 case 3: /* D3 vblank/vline */
2354 switch (src_data) { 2663 switch (src_data) {
2355 case 0: /* D3 vblank */ 2664 case 0: /* D3 vblank */
2356 if (disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 2665 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2357 drm_handle_vblank(rdev->ddev, 2); 2666 if (rdev->irq.crtc_vblank_int[2]) {
2358 rdev->pm.vblank_sync = true; 2667 drm_handle_vblank(rdev->ddev, 2);
2359 wake_up(&rdev->irq.vblank_queue); 2668 rdev->pm.vblank_sync = true;
2360 disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 2669 wake_up(&rdev->irq.vblank_queue);
2670 }
2671 if (rdev->irq.pflip[2])
2672 radeon_crtc_handle_flip(rdev, 2);
2673 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2361 DRM_DEBUG("IH: D3 vblank\n"); 2674 DRM_DEBUG("IH: D3 vblank\n");
2362 } 2675 }
2363 break; 2676 break;
2364 case 1: /* D3 vline */ 2677 case 1: /* D3 vline */
2365 if (disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 2678 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2366 disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 2679 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2367 DRM_DEBUG("IH: D3 vline\n"); 2680 DRM_DEBUG("IH: D3 vline\n");
2368 } 2681 }
2369 break; 2682 break;
@@ -2375,17 +2688,21 @@ restart_ih:
2375 case 4: /* D4 vblank/vline */ 2688 case 4: /* D4 vblank/vline */
2376 switch (src_data) { 2689 switch (src_data) {
2377 case 0: /* D4 vblank */ 2690 case 0: /* D4 vblank */
2378 if (disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 2691 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2379 drm_handle_vblank(rdev->ddev, 3); 2692 if (rdev->irq.crtc_vblank_int[3]) {
2380 rdev->pm.vblank_sync = true; 2693 drm_handle_vblank(rdev->ddev, 3);
2381 wake_up(&rdev->irq.vblank_queue); 2694 rdev->pm.vblank_sync = true;
2382 disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 2695 wake_up(&rdev->irq.vblank_queue);
2696 }
2697 if (rdev->irq.pflip[3])
2698 radeon_crtc_handle_flip(rdev, 3);
2699 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2383 DRM_DEBUG("IH: D4 vblank\n"); 2700 DRM_DEBUG("IH: D4 vblank\n");
2384 } 2701 }
2385 break; 2702 break;
2386 case 1: /* D4 vline */ 2703 case 1: /* D4 vline */
2387 if (disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 2704 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2388 disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 2705 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2389 DRM_DEBUG("IH: D4 vline\n"); 2706 DRM_DEBUG("IH: D4 vline\n");
2390 } 2707 }
2391 break; 2708 break;
@@ -2397,17 +2714,21 @@ restart_ih:
2397 case 5: /* D5 vblank/vline */ 2714 case 5: /* D5 vblank/vline */
2398 switch (src_data) { 2715 switch (src_data) {
2399 case 0: /* D5 vblank */ 2716 case 0: /* D5 vblank */
2400 if (disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 2717 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2401 drm_handle_vblank(rdev->ddev, 4); 2718 if (rdev->irq.crtc_vblank_int[4]) {
2402 rdev->pm.vblank_sync = true; 2719 drm_handle_vblank(rdev->ddev, 4);
2403 wake_up(&rdev->irq.vblank_queue); 2720 rdev->pm.vblank_sync = true;
2404 disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 2721 wake_up(&rdev->irq.vblank_queue);
2722 }
2723 if (rdev->irq.pflip[4])
2724 radeon_crtc_handle_flip(rdev, 4);
2725 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2405 DRM_DEBUG("IH: D5 vblank\n"); 2726 DRM_DEBUG("IH: D5 vblank\n");
2406 } 2727 }
2407 break; 2728 break;
2408 case 1: /* D5 vline */ 2729 case 1: /* D5 vline */
2409 if (disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 2730 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2410 disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 2731 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2411 DRM_DEBUG("IH: D5 vline\n"); 2732 DRM_DEBUG("IH: D5 vline\n");
2412 } 2733 }
2413 break; 2734 break;
@@ -2419,17 +2740,21 @@ restart_ih:
2419 case 6: /* D6 vblank/vline */ 2740 case 6: /* D6 vblank/vline */
2420 switch (src_data) { 2741 switch (src_data) {
2421 case 0: /* D6 vblank */ 2742 case 0: /* D6 vblank */
2422 if (disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 2743 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2423 drm_handle_vblank(rdev->ddev, 5); 2744 if (rdev->irq.crtc_vblank_int[5]) {
2424 rdev->pm.vblank_sync = true; 2745 drm_handle_vblank(rdev->ddev, 5);
2425 wake_up(&rdev->irq.vblank_queue); 2746 rdev->pm.vblank_sync = true;
2426 disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 2747 wake_up(&rdev->irq.vblank_queue);
2748 }
2749 if (rdev->irq.pflip[5])
2750 radeon_crtc_handle_flip(rdev, 5);
2751 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2427 DRM_DEBUG("IH: D6 vblank\n"); 2752 DRM_DEBUG("IH: D6 vblank\n");
2428 } 2753 }
2429 break; 2754 break;
2430 case 1: /* D6 vline */ 2755 case 1: /* D6 vline */
2431 if (disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 2756 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2432 disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 2757 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2433 DRM_DEBUG("IH: D6 vline\n"); 2758 DRM_DEBUG("IH: D6 vline\n");
2434 } 2759 }
2435 break; 2760 break;
@@ -2441,43 +2766,43 @@ restart_ih:
2441 case 42: /* HPD hotplug */ 2766 case 42: /* HPD hotplug */
2442 switch (src_data) { 2767 switch (src_data) {
2443 case 0: 2768 case 0:
2444 if (disp_int & DC_HPD1_INTERRUPT) { 2769 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2445 disp_int &= ~DC_HPD1_INTERRUPT; 2770 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2446 queue_hotplug = true; 2771 queue_hotplug = true;
2447 DRM_DEBUG("IH: HPD1\n"); 2772 DRM_DEBUG("IH: HPD1\n");
2448 } 2773 }
2449 break; 2774 break;
2450 case 1: 2775 case 1:
2451 if (disp_int_cont & DC_HPD2_INTERRUPT) { 2776 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2452 disp_int_cont &= ~DC_HPD2_INTERRUPT; 2777 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2453 queue_hotplug = true; 2778 queue_hotplug = true;
2454 DRM_DEBUG("IH: HPD2\n"); 2779 DRM_DEBUG("IH: HPD2\n");
2455 } 2780 }
2456 break; 2781 break;
2457 case 2: 2782 case 2:
2458 if (disp_int_cont2 & DC_HPD3_INTERRUPT) { 2783 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2459 disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 2784 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2460 queue_hotplug = true; 2785 queue_hotplug = true;
2461 DRM_DEBUG("IH: HPD3\n"); 2786 DRM_DEBUG("IH: HPD3\n");
2462 } 2787 }
2463 break; 2788 break;
2464 case 3: 2789 case 3:
2465 if (disp_int_cont3 & DC_HPD4_INTERRUPT) { 2790 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2466 disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 2791 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2467 queue_hotplug = true; 2792 queue_hotplug = true;
2468 DRM_DEBUG("IH: HPD4\n"); 2793 DRM_DEBUG("IH: HPD4\n");
2469 } 2794 }
2470 break; 2795 break;
2471 case 4: 2796 case 4:
2472 if (disp_int_cont4 & DC_HPD5_INTERRUPT) { 2797 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2473 disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 2798 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2474 queue_hotplug = true; 2799 queue_hotplug = true;
2475 DRM_DEBUG("IH: HPD5\n"); 2800 DRM_DEBUG("IH: HPD5\n");
2476 } 2801 }
2477 break; 2802 break;
2478 case 5: 2803 case 5:
2479 if (disp_int_cont5 & DC_HPD6_INTERRUPT) { 2804 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2480 disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 2805 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2481 queue_hotplug = true; 2806 queue_hotplug = true;
2482 DRM_DEBUG("IH: HPD6\n"); 2807 DRM_DEBUG("IH: HPD6\n");
2483 } 2808 }
@@ -2516,7 +2841,7 @@ restart_ih:
2516 if (wptr != rdev->ih.wptr) 2841 if (wptr != rdev->ih.wptr)
2517 goto restart_ih; 2842 goto restart_ih;
2518 if (queue_hotplug) 2843 if (queue_hotplug)
2519 queue_work(rdev->wq, &rdev->hotplug_work); 2844 schedule_work(&rdev->hotplug_work);
2520 rdev->ih.rptr = rptr; 2845 rdev->ih.rptr = rptr;
2521 WREG32(IH_RB_RPTR, rdev->ih.rptr); 2846 WREG32(IH_RB_RPTR, rdev->ih.rptr);
2522 spin_unlock_irqrestore(&rdev->ih.lock, flags); 2847 spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -2527,12 +2852,31 @@ static int evergreen_startup(struct radeon_device *rdev)
2527{ 2852{
2528 int r; 2853 int r;
2529 2854
2530 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 2855 /* enable pcie gen2 link */
2531 r = r600_init_microcode(rdev); 2856 if (!ASIC_IS_DCE5(rdev))
2857 evergreen_pcie_gen2_enable(rdev);
2858
2859 if (ASIC_IS_DCE5(rdev)) {
2860 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2861 r = ni_init_microcode(rdev);
2862 if (r) {
2863 DRM_ERROR("Failed to load firmware!\n");
2864 return r;
2865 }
2866 }
2867 r = btc_mc_load_microcode(rdev);
2532 if (r) { 2868 if (r) {
2533 DRM_ERROR("Failed to load firmware!\n"); 2869 DRM_ERROR("Failed to load MC firmware!\n");
2534 return r; 2870 return r;
2535 } 2871 }
2872 } else {
2873 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2874 r = r600_init_microcode(rdev);
2875 if (r) {
2876 DRM_ERROR("Failed to load firmware!\n");
2877 return r;
2878 }
2879 }
2536 } 2880 }
2537 2881
2538 evergreen_mc_program(rdev); 2882 evergreen_mc_program(rdev);
@@ -2551,6 +2895,11 @@ static int evergreen_startup(struct radeon_device *rdev)
2551 rdev->asic->copy = NULL; 2895 rdev->asic->copy = NULL;
2552 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 2896 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2553 } 2897 }
2898 /* XXX: ontario has problems blitting to gart at the moment */
2899 if (rdev->family == CHIP_PALM) {
2900 rdev->asic->copy = NULL;
2901 rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
2902 }
2554 2903
2555 /* allocate wb buffer */ 2904 /* allocate wb buffer */
2556 r = radeon_wb_init(rdev); 2905 r = radeon_wb_init(rdev);
@@ -2658,12 +3007,16 @@ static bool evergreen_card_posted(struct radeon_device *rdev)
2658 u32 reg; 3007 u32 reg;
2659 3008
2660 /* first check CRTCs */ 3009 /* first check CRTCs */
2661 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) | 3010 if (rdev->flags & RADEON_IS_IGP)
2662 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) | 3011 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
2663 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) | 3012 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
2664 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) | 3013 else
2665 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) | 3014 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
2666 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 3015 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) |
3016 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
3017 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) |
3018 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) |
3019 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
2667 if (reg & EVERGREEN_CRTC_MASTER_EN) 3020 if (reg & EVERGREEN_CRTC_MASTER_EN)
2668 return true; 3021 return true;
2669 3022
@@ -2800,3 +3153,52 @@ void evergreen_fini(struct radeon_device *rdev)
2800 rdev->bios = NULL; 3153 rdev->bios = NULL;
2801 radeon_dummy_page_fini(rdev); 3154 radeon_dummy_page_fini(rdev);
2802} 3155}
3156
3157static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
3158{
3159 u32 link_width_cntl, speed_cntl;
3160
3161 if (rdev->flags & RADEON_IS_IGP)
3162 return;
3163
3164 if (!(rdev->flags & RADEON_IS_PCIE))
3165 return;
3166
3167 /* x2 cards have a special sequence */
3168 if (ASIC_IS_X2(rdev))
3169 return;
3170
3171 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3172 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3173 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3174
3175 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3176 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3177 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3178
3179 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3180 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3181 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3182
3183 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3184 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3185 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3186
3187 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3188 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3189 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3190
3191 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3192 speed_cntl |= LC_GEN2_EN_STRAP;
3193 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3194
3195 } else {
3196 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3197 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3198 if (1)
3199 link_width_cntl |= LC_UPCONFIGURE_DIS;
3200 else
3201 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3202 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3203 }
3204}
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_kms.c b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
index e0e590110dd4..b758dc7f2f2c 100644
--- a/drivers/gpu/drm/radeon/evergreen_blit_kms.c
+++ b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
@@ -147,7 +147,9 @@ set_vtx_resource(struct radeon_device *rdev, u64 gpu_addr)
147 radeon_ring_write(rdev, 0); 147 radeon_ring_write(rdev, 0);
148 radeon_ring_write(rdev, SQ_TEX_VTX_VALID_BUFFER << 30); 148 radeon_ring_write(rdev, SQ_TEX_VTX_VALID_BUFFER << 30);
149 149
150 if (rdev->family == CHIP_CEDAR) 150 if ((rdev->family == CHIP_CEDAR) ||
151 (rdev->family == CHIP_PALM) ||
152 (rdev->family == CHIP_CAICOS))
151 cp_set_surface_sync(rdev, 153 cp_set_surface_sync(rdev,
152 PACKET3_TC_ACTION_ENA, 48, gpu_addr); 154 PACKET3_TC_ACTION_ENA, 48, gpu_addr);
153 else 155 else
@@ -331,9 +333,95 @@ set_default_state(struct radeon_device *rdev)
331 num_hs_stack_entries = 85; 333 num_hs_stack_entries = 85;
332 num_ls_stack_entries = 85; 334 num_ls_stack_entries = 85;
333 break; 335 break;
336 case CHIP_PALM:
337 num_ps_gprs = 93;
338 num_vs_gprs = 46;
339 num_temp_gprs = 4;
340 num_gs_gprs = 31;
341 num_es_gprs = 31;
342 num_hs_gprs = 23;
343 num_ls_gprs = 23;
344 num_ps_threads = 96;
345 num_vs_threads = 16;
346 num_gs_threads = 16;
347 num_es_threads = 16;
348 num_hs_threads = 16;
349 num_ls_threads = 16;
350 num_ps_stack_entries = 42;
351 num_vs_stack_entries = 42;
352 num_gs_stack_entries = 42;
353 num_es_stack_entries = 42;
354 num_hs_stack_entries = 42;
355 num_ls_stack_entries = 42;
356 break;
357 case CHIP_BARTS:
358 num_ps_gprs = 93;
359 num_vs_gprs = 46;
360 num_temp_gprs = 4;
361 num_gs_gprs = 31;
362 num_es_gprs = 31;
363 num_hs_gprs = 23;
364 num_ls_gprs = 23;
365 num_ps_threads = 128;
366 num_vs_threads = 20;
367 num_gs_threads = 20;
368 num_es_threads = 20;
369 num_hs_threads = 20;
370 num_ls_threads = 20;
371 num_ps_stack_entries = 85;
372 num_vs_stack_entries = 85;
373 num_gs_stack_entries = 85;
374 num_es_stack_entries = 85;
375 num_hs_stack_entries = 85;
376 num_ls_stack_entries = 85;
377 break;
378 case CHIP_TURKS:
379 num_ps_gprs = 93;
380 num_vs_gprs = 46;
381 num_temp_gprs = 4;
382 num_gs_gprs = 31;
383 num_es_gprs = 31;
384 num_hs_gprs = 23;
385 num_ls_gprs = 23;
386 num_ps_threads = 128;
387 num_vs_threads = 20;
388 num_gs_threads = 20;
389 num_es_threads = 20;
390 num_hs_threads = 20;
391 num_ls_threads = 20;
392 num_ps_stack_entries = 42;
393 num_vs_stack_entries = 42;
394 num_gs_stack_entries = 42;
395 num_es_stack_entries = 42;
396 num_hs_stack_entries = 42;
397 num_ls_stack_entries = 42;
398 break;
399 case CHIP_CAICOS:
400 num_ps_gprs = 93;
401 num_vs_gprs = 46;
402 num_temp_gprs = 4;
403 num_gs_gprs = 31;
404 num_es_gprs = 31;
405 num_hs_gprs = 23;
406 num_ls_gprs = 23;
407 num_ps_threads = 128;
408 num_vs_threads = 10;
409 num_gs_threads = 10;
410 num_es_threads = 10;
411 num_hs_threads = 10;
412 num_ls_threads = 10;
413 num_ps_stack_entries = 42;
414 num_vs_stack_entries = 42;
415 num_gs_stack_entries = 42;
416 num_es_stack_entries = 42;
417 num_hs_stack_entries = 42;
418 num_ls_stack_entries = 42;
419 break;
334 } 420 }
335 421
336 if (rdev->family == CHIP_CEDAR) 422 if ((rdev->family == CHIP_CEDAR) ||
423 (rdev->family == CHIP_PALM) ||
424 (rdev->family == CHIP_CAICOS))
337 sq_config = 0; 425 sq_config = 0;
338 else 426 else
339 sq_config = VC_ENABLE; 427 sq_config = VC_ENABLE;
diff --git a/drivers/gpu/drm/radeon/evergreen_reg.h b/drivers/gpu/drm/radeon/evergreen_reg.h
index 2330f3a36fd5..c781c92c3451 100644
--- a/drivers/gpu/drm/radeon/evergreen_reg.h
+++ b/drivers/gpu/drm/radeon/evergreen_reg.h
@@ -105,6 +105,11 @@
105#define EVERGREEN_GRPH_Y_START 0x6830 105#define EVERGREEN_GRPH_Y_START 0x6830
106#define EVERGREEN_GRPH_X_END 0x6834 106#define EVERGREEN_GRPH_X_END 0x6834
107#define EVERGREEN_GRPH_Y_END 0x6838 107#define EVERGREEN_GRPH_Y_END 0x6838
108#define EVERGREEN_GRPH_UPDATE 0x6844
109# define EVERGREEN_GRPH_SURFACE_UPDATE_PENDING (1 << 2)
110# define EVERGREEN_GRPH_UPDATE_LOCK (1 << 16)
111#define EVERGREEN_GRPH_FLIP_CONTROL 0x6848
112# define EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN (1 << 0)
108 113
109/* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */ 114/* CUR blocks at 0x6998, 0x7598, 0x10198, 0x10d98, 0x11998, 0x12598 */
110#define EVERGREEN_CUR_CONTROL 0x6998 115#define EVERGREEN_CUR_CONTROL 0x6998
@@ -178,6 +183,7 @@
178# define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24) 183# define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24)
179#define EVERGREEN_CRTC_STATUS 0x6e8c 184#define EVERGREEN_CRTC_STATUS 0x6e8c
180#define EVERGREEN_CRTC_STATUS_POSITION 0x6e90 185#define EVERGREEN_CRTC_STATUS_POSITION 0x6e90
186#define EVERGREEN_MASTER_UPDATE_MODE 0x6ef8
181#define EVERGREEN_CRTC_UPDATE_LOCK 0x6ed4 187#define EVERGREEN_CRTC_UPDATE_LOCK 0x6ed4
182 188
183#define EVERGREEN_DC_GPIO_HPD_MASK 0x64b0 189#define EVERGREEN_DC_GPIO_HPD_MASK 0x64b0
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index a73b53c44359..36d32d83d866 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -164,11 +164,13 @@
164#define SE_SC_BUSY (1 << 29) 164#define SE_SC_BUSY (1 << 29)
165#define SE_DB_BUSY (1 << 30) 165#define SE_DB_BUSY (1 << 30)
166#define SE_CB_BUSY (1 << 31) 166#define SE_CB_BUSY (1 << 31)
167 167/* evergreen */
168#define CG_MULT_THERMAL_STATUS 0x740 168#define CG_MULT_THERMAL_STATUS 0x740
169#define ASIC_T(x) ((x) << 16) 169#define ASIC_T(x) ((x) << 16)
170#define ASIC_T_MASK 0x7FF0000 170#define ASIC_T_MASK 0x7FF0000
171#define ASIC_T_SHIFT 16 171#define ASIC_T_SHIFT 16
172/* APU */
173#define CG_THERMAL_STATUS 0x678
172 174
173#define HDP_HOST_PATH_CNTL 0x2C00 175#define HDP_HOST_PATH_CNTL 0x2C00
174#define HDP_NONSURFACE_BASE 0x2C04 176#define HDP_NONSURFACE_BASE 0x2C04
@@ -181,6 +183,7 @@
181#define MC_SHARED_CHMAP 0x2004 183#define MC_SHARED_CHMAP 0x2004
182#define NOOFCHAN_SHIFT 12 184#define NOOFCHAN_SHIFT 12
183#define NOOFCHAN_MASK 0x00003000 185#define NOOFCHAN_MASK 0x00003000
186#define MC_SHARED_CHREMAP 0x2008
184 187
185#define MC_ARB_RAMCFG 0x2760 188#define MC_ARB_RAMCFG 0x2760
186#define NOOFBANK_SHIFT 0 189#define NOOFBANK_SHIFT 0
@@ -200,6 +203,7 @@
200#define MC_VM_AGP_BOT 0x202C 203#define MC_VM_AGP_BOT 0x202C
201#define MC_VM_AGP_BASE 0x2030 204#define MC_VM_AGP_BASE 0x2030
202#define MC_VM_FB_LOCATION 0x2024 205#define MC_VM_FB_LOCATION 0x2024
206#define MC_FUS_VM_FB_OFFSET 0x2898
203#define MC_VM_MB_L1_TLB0_CNTL 0x2234 207#define MC_VM_MB_L1_TLB0_CNTL 0x2234
204#define MC_VM_MB_L1_TLB1_CNTL 0x2238 208#define MC_VM_MB_L1_TLB1_CNTL 0x2238
205#define MC_VM_MB_L1_TLB2_CNTL 0x223C 209#define MC_VM_MB_L1_TLB2_CNTL 0x223C
@@ -349,6 +353,9 @@
349#define SYNC_WALKER (1 << 25) 353#define SYNC_WALKER (1 << 25)
350#define SYNC_ALIGNER (1 << 26) 354#define SYNC_ALIGNER (1 << 26)
351 355
356#define TCP_CHAN_STEER_LO 0x960c
357#define TCP_CHAN_STEER_HI 0x9610
358
352#define VGT_CACHE_INVALIDATION 0x88C4 359#define VGT_CACHE_INVALIDATION 0x88C4
353#define CACHE_INVALIDATION(x) ((x) << 0) 360#define CACHE_INVALIDATION(x) ((x) << 0)
354#define VC_ONLY 0 361#define VC_ONLY 0
@@ -574,6 +581,44 @@
574# define DC_HPDx_RX_INT_TIMER(x) ((x) << 16) 581# define DC_HPDx_RX_INT_TIMER(x) ((x) << 16)
575# define DC_HPDx_EN (1 << 28) 582# define DC_HPDx_EN (1 << 28)
576 583
584/* PCIE link stuff */
585#define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */
586#define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */
587# define LC_LINK_WIDTH_SHIFT 0
588# define LC_LINK_WIDTH_MASK 0x7
589# define LC_LINK_WIDTH_X0 0
590# define LC_LINK_WIDTH_X1 1
591# define LC_LINK_WIDTH_X2 2
592# define LC_LINK_WIDTH_X4 3
593# define LC_LINK_WIDTH_X8 4
594# define LC_LINK_WIDTH_X16 6
595# define LC_LINK_WIDTH_RD_SHIFT 4
596# define LC_LINK_WIDTH_RD_MASK 0x70
597# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7)
598# define LC_RECONFIG_NOW (1 << 8)
599# define LC_RENEGOTIATION_SUPPORT (1 << 9)
600# define LC_RENEGOTIATE_EN (1 << 10)
601# define LC_SHORT_RECONFIG_EN (1 << 11)
602# define LC_UPCONFIGURE_SUPPORT (1 << 12)
603# define LC_UPCONFIGURE_DIS (1 << 13)
604#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */
605# define LC_GEN2_EN_STRAP (1 << 0)
606# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1)
607# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 5)
608# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 6)
609# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8)
610# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3
611# define LC_CURRENT_DATA_RATE (1 << 11)
612# define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14)
613# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21)
614# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23)
615# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24)
616#define MM_CFGREGS_CNTL 0x544c
617# define MM_WR_TO_CFG_EN (1 << 3)
618#define LINK_CNTL2 0x88 /* F0 */
619# define TARGET_LINK_SPEED_MASK (0xf << 0)
620# define SELECTABLE_DEEMPHASIS (1 << 6)
621
577/* 622/*
578 * PM4 623 * PM4
579 */ 624 */
@@ -603,7 +648,7 @@
603#define PACKET3_NOP 0x10 648#define PACKET3_NOP 0x10
604#define PACKET3_SET_BASE 0x11 649#define PACKET3_SET_BASE 0x11
605#define PACKET3_CLEAR_STATE 0x12 650#define PACKET3_CLEAR_STATE 0x12
606#define PACKET3_INDIRECT_BUFFER_SIZE 0x13 651#define PACKET3_INDEX_BUFFER_SIZE 0x13
607#define PACKET3_DISPATCH_DIRECT 0x15 652#define PACKET3_DISPATCH_DIRECT 0x15
608#define PACKET3_DISPATCH_INDIRECT 0x16 653#define PACKET3_DISPATCH_INDIRECT 0x16
609#define PACKET3_INDIRECT_BUFFER_END 0x17 654#define PACKET3_INDIRECT_BUFFER_END 0x17
@@ -644,14 +689,14 @@
644# define PACKET3_CB8_DEST_BASE_ENA (1 << 15) 689# define PACKET3_CB8_DEST_BASE_ENA (1 << 15)
645# define PACKET3_CB9_DEST_BASE_ENA (1 << 16) 690# define PACKET3_CB9_DEST_BASE_ENA (1 << 16)
646# define PACKET3_CB10_DEST_BASE_ENA (1 << 17) 691# define PACKET3_CB10_DEST_BASE_ENA (1 << 17)
647# define PACKET3_CB11_DEST_BASE_ENA (1 << 17) 692# define PACKET3_CB11_DEST_BASE_ENA (1 << 18)
648# define PACKET3_FULL_CACHE_ENA (1 << 20) 693# define PACKET3_FULL_CACHE_ENA (1 << 20)
649# define PACKET3_TC_ACTION_ENA (1 << 23) 694# define PACKET3_TC_ACTION_ENA (1 << 23)
650# define PACKET3_VC_ACTION_ENA (1 << 24) 695# define PACKET3_VC_ACTION_ENA (1 << 24)
651# define PACKET3_CB_ACTION_ENA (1 << 25) 696# define PACKET3_CB_ACTION_ENA (1 << 25)
652# define PACKET3_DB_ACTION_ENA (1 << 26) 697# define PACKET3_DB_ACTION_ENA (1 << 26)
653# define PACKET3_SH_ACTION_ENA (1 << 27) 698# define PACKET3_SH_ACTION_ENA (1 << 27)
654# define PACKET3_SMX_ACTION_ENA (1 << 28) 699# define PACKET3_SX_ACTION_ENA (1 << 28)
655#define PACKET3_ME_INITIALIZE 0x44 700#define PACKET3_ME_INITIALIZE 0x44
656#define PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16) 701#define PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
657#define PACKET3_COND_WRITE 0x45 702#define PACKET3_COND_WRITE 0x45
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
new file mode 100644
index 000000000000..5e0bef80ad7f
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -0,0 +1,316 @@
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include "drmP.h"
28#include "radeon.h"
29#include "radeon_asic.h"
30#include "radeon_drm.h"
31#include "nid.h"
32#include "atom.h"
33#include "ni_reg.h"
34
35#define EVERGREEN_PFP_UCODE_SIZE 1120
36#define EVERGREEN_PM4_UCODE_SIZE 1376
37#define EVERGREEN_RLC_UCODE_SIZE 768
38#define BTC_MC_UCODE_SIZE 6024
39
40/* Firmware Names */
41MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
42MODULE_FIRMWARE("radeon/BARTS_me.bin");
43MODULE_FIRMWARE("radeon/BARTS_mc.bin");
44MODULE_FIRMWARE("radeon/BTC_rlc.bin");
45MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
46MODULE_FIRMWARE("radeon/TURKS_me.bin");
47MODULE_FIRMWARE("radeon/TURKS_mc.bin");
48MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
49MODULE_FIRMWARE("radeon/CAICOS_me.bin");
50MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
51
52#define BTC_IO_MC_REGS_SIZE 29
53
54static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
55 {0x00000077, 0xff010100},
56 {0x00000078, 0x00000000},
57 {0x00000079, 0x00001434},
58 {0x0000007a, 0xcc08ec08},
59 {0x0000007b, 0x00040000},
60 {0x0000007c, 0x000080c0},
61 {0x0000007d, 0x09000000},
62 {0x0000007e, 0x00210404},
63 {0x00000081, 0x08a8e800},
64 {0x00000082, 0x00030444},
65 {0x00000083, 0x00000000},
66 {0x00000085, 0x00000001},
67 {0x00000086, 0x00000002},
68 {0x00000087, 0x48490000},
69 {0x00000088, 0x20244647},
70 {0x00000089, 0x00000005},
71 {0x0000008b, 0x66030000},
72 {0x0000008c, 0x00006603},
73 {0x0000008d, 0x00000100},
74 {0x0000008f, 0x00001c0a},
75 {0x00000090, 0xff000001},
76 {0x00000094, 0x00101101},
77 {0x00000095, 0x00000fff},
78 {0x00000096, 0x00116fff},
79 {0x00000097, 0x60010000},
80 {0x00000098, 0x10010000},
81 {0x00000099, 0x00006000},
82 {0x0000009a, 0x00001000},
83 {0x0000009f, 0x00946a00}
84};
85
86static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
87 {0x00000077, 0xff010100},
88 {0x00000078, 0x00000000},
89 {0x00000079, 0x00001434},
90 {0x0000007a, 0xcc08ec08},
91 {0x0000007b, 0x00040000},
92 {0x0000007c, 0x000080c0},
93 {0x0000007d, 0x09000000},
94 {0x0000007e, 0x00210404},
95 {0x00000081, 0x08a8e800},
96 {0x00000082, 0x00030444},
97 {0x00000083, 0x00000000},
98 {0x00000085, 0x00000001},
99 {0x00000086, 0x00000002},
100 {0x00000087, 0x48490000},
101 {0x00000088, 0x20244647},
102 {0x00000089, 0x00000005},
103 {0x0000008b, 0x66030000},
104 {0x0000008c, 0x00006603},
105 {0x0000008d, 0x00000100},
106 {0x0000008f, 0x00001c0a},
107 {0x00000090, 0xff000001},
108 {0x00000094, 0x00101101},
109 {0x00000095, 0x00000fff},
110 {0x00000096, 0x00116fff},
111 {0x00000097, 0x60010000},
112 {0x00000098, 0x10010000},
113 {0x00000099, 0x00006000},
114 {0x0000009a, 0x00001000},
115 {0x0000009f, 0x00936a00}
116};
117
118static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
119 {0x00000077, 0xff010100},
120 {0x00000078, 0x00000000},
121 {0x00000079, 0x00001434},
122 {0x0000007a, 0xcc08ec08},
123 {0x0000007b, 0x00040000},
124 {0x0000007c, 0x000080c0},
125 {0x0000007d, 0x09000000},
126 {0x0000007e, 0x00210404},
127 {0x00000081, 0x08a8e800},
128 {0x00000082, 0x00030444},
129 {0x00000083, 0x00000000},
130 {0x00000085, 0x00000001},
131 {0x00000086, 0x00000002},
132 {0x00000087, 0x48490000},
133 {0x00000088, 0x20244647},
134 {0x00000089, 0x00000005},
135 {0x0000008b, 0x66030000},
136 {0x0000008c, 0x00006603},
137 {0x0000008d, 0x00000100},
138 {0x0000008f, 0x00001c0a},
139 {0x00000090, 0xff000001},
140 {0x00000094, 0x00101101},
141 {0x00000095, 0x00000fff},
142 {0x00000096, 0x00116fff},
143 {0x00000097, 0x60010000},
144 {0x00000098, 0x10010000},
145 {0x00000099, 0x00006000},
146 {0x0000009a, 0x00001000},
147 {0x0000009f, 0x00916a00}
148};
149
150int btc_mc_load_microcode(struct radeon_device *rdev)
151{
152 const __be32 *fw_data;
153 u32 mem_type, running, blackout = 0;
154 u32 *io_mc_regs;
155 int i;
156
157 if (!rdev->mc_fw)
158 return -EINVAL;
159
160 switch (rdev->family) {
161 case CHIP_BARTS:
162 io_mc_regs = (u32 *)&barts_io_mc_regs;
163 break;
164 case CHIP_TURKS:
165 io_mc_regs = (u32 *)&turks_io_mc_regs;
166 break;
167 case CHIP_CAICOS:
168 default:
169 io_mc_regs = (u32 *)&caicos_io_mc_regs;
170 break;
171 }
172
173 mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
174 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
175
176 if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
177 if (running) {
178 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
179 WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
180 }
181
182 /* reset the engine and set to writable */
183 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
184 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
185
186 /* load mc io regs */
187 for (i = 0; i < BTC_IO_MC_REGS_SIZE; i++) {
188 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
189 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
190 }
191 /* load the MC ucode */
192 fw_data = (const __be32 *)rdev->mc_fw->data;
193 for (i = 0; i < BTC_MC_UCODE_SIZE; i++)
194 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
195
196 /* put the engine back into the active state */
197 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
198 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
199 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
200
201 /* wait for training to complete */
202 while (!(RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD))
203 udelay(10);
204
205 if (running)
206 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
207 }
208
209 return 0;
210}
211
212int ni_init_microcode(struct radeon_device *rdev)
213{
214 struct platform_device *pdev;
215 const char *chip_name;
216 const char *rlc_chip_name;
217 size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
218 char fw_name[30];
219 int err;
220
221 DRM_DEBUG("\n");
222
223 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
224 err = IS_ERR(pdev);
225 if (err) {
226 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
227 return -EINVAL;
228 }
229
230 switch (rdev->family) {
231 case CHIP_BARTS:
232 chip_name = "BARTS";
233 rlc_chip_name = "BTC";
234 break;
235 case CHIP_TURKS:
236 chip_name = "TURKS";
237 rlc_chip_name = "BTC";
238 break;
239 case CHIP_CAICOS:
240 chip_name = "CAICOS";
241 rlc_chip_name = "BTC";
242 break;
243 default: BUG();
244 }
245
246 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
247 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
248 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
249 mc_req_size = BTC_MC_UCODE_SIZE * 4;
250
251 DRM_INFO("Loading %s Microcode\n", chip_name);
252
253 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
254 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
255 if (err)
256 goto out;
257 if (rdev->pfp_fw->size != pfp_req_size) {
258 printk(KERN_ERR
259 "ni_cp: Bogus length %zu in firmware \"%s\"\n",
260 rdev->pfp_fw->size, fw_name);
261 err = -EINVAL;
262 goto out;
263 }
264
265 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
266 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
267 if (err)
268 goto out;
269 if (rdev->me_fw->size != me_req_size) {
270 printk(KERN_ERR
271 "ni_cp: Bogus length %zu in firmware \"%s\"\n",
272 rdev->me_fw->size, fw_name);
273 err = -EINVAL;
274 }
275
276 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
277 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
278 if (err)
279 goto out;
280 if (rdev->rlc_fw->size != rlc_req_size) {
281 printk(KERN_ERR
282 "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
283 rdev->rlc_fw->size, fw_name);
284 err = -EINVAL;
285 }
286
287 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
288 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
289 if (err)
290 goto out;
291 if (rdev->mc_fw->size != mc_req_size) {
292 printk(KERN_ERR
293 "ni_mc: Bogus length %zu in firmware \"%s\"\n",
294 rdev->mc_fw->size, fw_name);
295 err = -EINVAL;
296 }
297out:
298 platform_device_unregister(pdev);
299
300 if (err) {
301 if (err != -EINVAL)
302 printk(KERN_ERR
303 "ni_cp: Failed to load firmware \"%s\"\n",
304 fw_name);
305 release_firmware(rdev->pfp_fw);
306 rdev->pfp_fw = NULL;
307 release_firmware(rdev->me_fw);
308 rdev->me_fw = NULL;
309 release_firmware(rdev->rlc_fw);
310 rdev->rlc_fw = NULL;
311 release_firmware(rdev->mc_fw);
312 rdev->mc_fw = NULL;
313 }
314 return err;
315}
316
diff --git a/drivers/gpu/drm/radeon/ni_reg.h b/drivers/gpu/drm/radeon/ni_reg.h
new file mode 100644
index 000000000000..5db7b7d6feb0
--- /dev/null
+++ b/drivers/gpu/drm/radeon/ni_reg.h
@@ -0,0 +1,86 @@
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#ifndef __NI_REG_H__
25#define __NI_REG_H__
26
27/* northern islands - DCE5 */
28
29#define NI_INPUT_GAMMA_CONTROL 0x6840
30# define NI_GRPH_INPUT_GAMMA_MODE(x) (((x) & 0x3) << 0)
31# define NI_INPUT_GAMMA_USE_LUT 0
32# define NI_INPUT_GAMMA_BYPASS 1
33# define NI_INPUT_GAMMA_SRGB_24 2
34# define NI_INPUT_GAMMA_XVYCC_222 3
35# define NI_OVL_INPUT_GAMMA_MODE(x) (((x) & 0x3) << 4)
36
37#define NI_PRESCALE_GRPH_CONTROL 0x68b4
38# define NI_GRPH_PRESCALE_BYPASS (1 << 4)
39
40#define NI_PRESCALE_OVL_CONTROL 0x68c4
41# define NI_OVL_PRESCALE_BYPASS (1 << 4)
42
43#define NI_INPUT_CSC_CONTROL 0x68d4
44# define NI_INPUT_CSC_GRPH_MODE(x) (((x) & 0x3) << 0)
45# define NI_INPUT_CSC_BYPASS 0
46# define NI_INPUT_CSC_PROG_COEFF 1
47# define NI_INPUT_CSC_PROG_SHARED_MATRIXA 2
48# define NI_INPUT_CSC_OVL_MODE(x) (((x) & 0x3) << 4)
49
50#define NI_OUTPUT_CSC_CONTROL 0x68f0
51# define NI_OUTPUT_CSC_GRPH_MODE(x) (((x) & 0x7) << 0)
52# define NI_OUTPUT_CSC_BYPASS 0
53# define NI_OUTPUT_CSC_TV_RGB 1
54# define NI_OUTPUT_CSC_YCBCR_601 2
55# define NI_OUTPUT_CSC_YCBCR_709 3
56# define NI_OUTPUT_CSC_PROG_COEFF 4
57# define NI_OUTPUT_CSC_PROG_SHARED_MATRIXB 5
58# define NI_OUTPUT_CSC_OVL_MODE(x) (((x) & 0x7) << 4)
59
60#define NI_DEGAMMA_CONTROL 0x6960
61# define NI_GRPH_DEGAMMA_MODE(x) (((x) & 0x3) << 0)
62# define NI_DEGAMMA_BYPASS 0
63# define NI_DEGAMMA_SRGB_24 1
64# define NI_DEGAMMA_XVYCC_222 2
65# define NI_OVL_DEGAMMA_MODE(x) (((x) & 0x3) << 4)
66# define NI_ICON_DEGAMMA_MODE(x) (((x) & 0x3) << 8)
67# define NI_CURSOR_DEGAMMA_MODE(x) (((x) & 0x3) << 12)
68
69#define NI_GAMUT_REMAP_CONTROL 0x6964
70# define NI_GRPH_GAMUT_REMAP_MODE(x) (((x) & 0x3) << 0)
71# define NI_GAMUT_REMAP_BYPASS 0
72# define NI_GAMUT_REMAP_PROG_COEFF 1
73# define NI_GAMUT_REMAP_PROG_SHARED_MATRIXA 2
74# define NI_GAMUT_REMAP_PROG_SHARED_MATRIXB 3
75# define NI_OVL_GAMUT_REMAP_MODE(x) (((x) & 0x3) << 4)
76
77#define NI_REGAMMA_CONTROL 0x6a80
78# define NI_GRPH_REGAMMA_MODE(x) (((x) & 0x7) << 0)
79# define NI_REGAMMA_BYPASS 0
80# define NI_REGAMMA_SRGB_24 1
81# define NI_REGAMMA_XVYCC_222 2
82# define NI_REGAMMA_PROG_A 3
83# define NI_REGAMMA_PROG_B 4
84# define NI_OVL_REGAMMA_MODE(x) (((x) & 0x7) << 4)
85
86#endif
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
new file mode 100644
index 000000000000..f7b445390e02
--- /dev/null
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -0,0 +1,41 @@
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#ifndef NI_H
25#define NI_H
26
27#define MC_SHARED_BLACKOUT_CNTL 0x20ac
28#define MC_SEQ_SUP_CNTL 0x28c8
29#define RUN_MASK (1 << 0)
30#define MC_SEQ_SUP_PGM 0x28cc
31#define MC_IO_PAD_CNTL_D0 0x29d0
32#define MEM_FALL_OUT_CMD (1 << 8)
33#define MC_SEQ_MISC0 0x2a00
34#define MC_SEQ_MISC0_GDDR5_SHIFT 28
35#define MC_SEQ_MISC0_GDDR5_MASK 0xf0000000
36#define MC_SEQ_MISC0_GDDR5_VALUE 5
37#define MC_SEQ_IO_DEBUG_INDEX 0x2a44
38#define MC_SEQ_IO_DEBUG_DATA 0x2a48
39
40#endif
41
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 8e10aa9f74b0..f637595b14e1 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -68,6 +68,56 @@ MODULE_FIRMWARE(FIRMWARE_R520);
68 * r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280 68 * r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280
69 */ 69 */
70 70
71void r100_pre_page_flip(struct radeon_device *rdev, int crtc)
72{
73 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
74 u32 tmp;
75
76 /* make sure flip is at vb rather than hb */
77 tmp = RREG32(RADEON_CRTC_OFFSET_CNTL + radeon_crtc->crtc_offset);
78 tmp &= ~RADEON_CRTC_OFFSET_FLIP_CNTL;
79 /* make sure pending bit is asserted */
80 tmp |= RADEON_CRTC_GUI_TRIG_OFFSET_LEFT_EN;
81 WREG32(RADEON_CRTC_OFFSET_CNTL + radeon_crtc->crtc_offset, tmp);
82
83 /* set pageflip to happen as late as possible in the vblank interval.
84 * same field for crtc1/2
85 */
86 tmp = RREG32(RADEON_CRTC_GEN_CNTL);
87 tmp &= ~RADEON_CRTC_VSTAT_MODE_MASK;
88 WREG32(RADEON_CRTC_GEN_CNTL, tmp);
89
90 /* enable the pflip int */
91 radeon_irq_kms_pflip_irq_get(rdev, crtc);
92}
93
94void r100_post_page_flip(struct radeon_device *rdev, int crtc)
95{
96 /* disable the pflip int */
97 radeon_irq_kms_pflip_irq_put(rdev, crtc);
98}
99
100u32 r100_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
101{
102 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
103 u32 tmp = ((u32)crtc_base) | RADEON_CRTC_OFFSET__OFFSET_LOCK;
104
105 /* Lock the graphics update lock */
106 /* update the scanout addresses */
107 WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp);
108
109 /* Wait for update_pending to go high. */
110 while (!(RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET));
111 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
112
113 /* Unlock the lock, so double-buffering can take place inside vblank */
114 tmp &= ~RADEON_CRTC_OFFSET__OFFSET_LOCK;
115 WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, tmp);
116
117 /* Return current update_pending status: */
118 return RREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset) & RADEON_CRTC_OFFSET__GUI_TRIG_OFFSET;
119}
120
71void r100_pm_get_dynpm_state(struct radeon_device *rdev) 121void r100_pm_get_dynpm_state(struct radeon_device *rdev)
72{ 122{
73 int i; 123 int i;
@@ -526,10 +576,12 @@ int r100_irq_set(struct radeon_device *rdev)
526 if (rdev->irq.gui_idle) { 576 if (rdev->irq.gui_idle) {
527 tmp |= RADEON_GUI_IDLE_MASK; 577 tmp |= RADEON_GUI_IDLE_MASK;
528 } 578 }
529 if (rdev->irq.crtc_vblank_int[0]) { 579 if (rdev->irq.crtc_vblank_int[0] ||
580 rdev->irq.pflip[0]) {
530 tmp |= RADEON_CRTC_VBLANK_MASK; 581 tmp |= RADEON_CRTC_VBLANK_MASK;
531 } 582 }
532 if (rdev->irq.crtc_vblank_int[1]) { 583 if (rdev->irq.crtc_vblank_int[1] ||
584 rdev->irq.pflip[1]) {
533 tmp |= RADEON_CRTC2_VBLANK_MASK; 585 tmp |= RADEON_CRTC2_VBLANK_MASK;
534 } 586 }
535 if (rdev->irq.hpd[0]) { 587 if (rdev->irq.hpd[0]) {
@@ -600,14 +652,22 @@ int r100_irq_process(struct radeon_device *rdev)
600 } 652 }
601 /* Vertical blank interrupts */ 653 /* Vertical blank interrupts */
602 if (status & RADEON_CRTC_VBLANK_STAT) { 654 if (status & RADEON_CRTC_VBLANK_STAT) {
603 drm_handle_vblank(rdev->ddev, 0); 655 if (rdev->irq.crtc_vblank_int[0]) {
604 rdev->pm.vblank_sync = true; 656 drm_handle_vblank(rdev->ddev, 0);
605 wake_up(&rdev->irq.vblank_queue); 657 rdev->pm.vblank_sync = true;
658 wake_up(&rdev->irq.vblank_queue);
659 }
660 if (rdev->irq.pflip[0])
661 radeon_crtc_handle_flip(rdev, 0);
606 } 662 }
607 if (status & RADEON_CRTC2_VBLANK_STAT) { 663 if (status & RADEON_CRTC2_VBLANK_STAT) {
608 drm_handle_vblank(rdev->ddev, 1); 664 if (rdev->irq.crtc_vblank_int[1]) {
609 rdev->pm.vblank_sync = true; 665 drm_handle_vblank(rdev->ddev, 1);
610 wake_up(&rdev->irq.vblank_queue); 666 rdev->pm.vblank_sync = true;
667 wake_up(&rdev->irq.vblank_queue);
668 }
669 if (rdev->irq.pflip[1])
670 radeon_crtc_handle_flip(rdev, 1);
611 } 671 }
612 if (status & RADEON_FP_DETECT_STAT) { 672 if (status & RADEON_FP_DETECT_STAT) {
613 queue_hotplug = true; 673 queue_hotplug = true;
@@ -622,7 +682,7 @@ int r100_irq_process(struct radeon_device *rdev)
622 /* reset gui idle ack. the status bit is broken */ 682 /* reset gui idle ack. the status bit is broken */
623 rdev->irq.gui_idle_acked = false; 683 rdev->irq.gui_idle_acked = false;
624 if (queue_hotplug) 684 if (queue_hotplug)
625 queue_work(rdev->wq, &rdev->hotplug_work); 685 schedule_work(&rdev->hotplug_work);
626 if (rdev->msi_enabled) { 686 if (rdev->msi_enabled) {
627 switch (rdev->family) { 687 switch (rdev->family) {
628 case CHIP_RS400: 688 case CHIP_RS400:
diff --git a/drivers/gpu/drm/radeon/r100d.h b/drivers/gpu/drm/radeon/r100d.h
index b121b6c678d4..eab91760fae0 100644
--- a/drivers/gpu/drm/radeon/r100d.h
+++ b/drivers/gpu/drm/radeon/r100d.h
@@ -551,7 +551,7 @@
551#define S_000360_CUR2_LOCK(x) (((x) & 0x1) << 31) 551#define S_000360_CUR2_LOCK(x) (((x) & 0x1) << 31)
552#define G_000360_CUR2_LOCK(x) (((x) >> 31) & 0x1) 552#define G_000360_CUR2_LOCK(x) (((x) >> 31) & 0x1)
553#define C_000360_CUR2_LOCK 0x7FFFFFFF 553#define C_000360_CUR2_LOCK 0x7FFFFFFF
554#define R_0003C2_GENMO_WT 0x0003C0 554#define R_0003C2_GENMO_WT 0x0003C2
555#define S_0003C2_GENMO_MONO_ADDRESS_B(x) (((x) & 0x1) << 0) 555#define S_0003C2_GENMO_MONO_ADDRESS_B(x) (((x) & 0x1) << 0)
556#define G_0003C2_GENMO_MONO_ADDRESS_B(x) (((x) >> 0) & 0x1) 556#define G_0003C2_GENMO_MONO_ADDRESS_B(x) (((x) >> 0) & 0x1)
557#define C_0003C2_GENMO_MONO_ADDRESS_B 0xFE 557#define C_0003C2_GENMO_MONO_ADDRESS_B 0xFE
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index cde1d3480d93..fae5e709f270 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -558,10 +558,7 @@ int rv370_get_pcie_lanes(struct radeon_device *rdev)
558 558
559 /* FIXME wait for idle */ 559 /* FIXME wait for idle */
560 560
561 if (rdev->family < CHIP_R600) 561 link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
562 link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
563 else
564 link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
565 562
566 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) { 563 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
567 case RADEON_PCIE_LC_LINK_WIDTH_X0: 564 case RADEON_PCIE_LC_LINK_WIDTH_X0:
@@ -745,6 +742,11 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
745 break; 742 break;
746 case 0x4E00: 743 case 0x4E00:
747 /* RB3D_CCTL */ 744 /* RB3D_CCTL */
745 if ((idx_value & (1 << 10)) && /* CMASK_ENABLE */
746 p->rdev->cmask_filp != p->filp) {
747 DRM_ERROR("Invalid RB3D_CCTL: Cannot enable CMASK.\n");
748 return -EINVAL;
749 }
748 track->num_cb = ((idx_value >> 5) & 0x3) + 1; 750 track->num_cb = ((idx_value >> 5) & 0x3) + 1;
749 break; 751 break;
750 case 0x4E38: 752 case 0x4E38:
@@ -787,6 +789,13 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
787 case 15: 789 case 15:
788 track->cb[i].cpp = 2; 790 track->cb[i].cpp = 2;
789 break; 791 break;
792 case 5:
793 if (p->rdev->family < CHIP_RV515) {
794 DRM_ERROR("Invalid color buffer format (%d)!\n",
795 ((idx_value >> 21) & 0xF));
796 return -EINVAL;
797 }
798 /* Pass through. */
790 case 6: 799 case 6:
791 track->cb[i].cpp = 4; 800 track->cb[i].cpp = 4;
792 break; 801 break;
@@ -1199,6 +1208,10 @@ static int r300_packet3_check(struct radeon_cs_parser *p,
1199 if (p->rdev->hyperz_filp != p->filp) 1208 if (p->rdev->hyperz_filp != p->filp)
1200 return -EINVAL; 1209 return -EINVAL;
1201 break; 1210 break;
1211 case PACKET3_3D_CLEAR_CMASK:
1212 if (p->rdev->cmask_filp != p->filp)
1213 return -EINVAL;
1214 break;
1202 case PACKET3_NOP: 1215 case PACKET3_NOP:
1203 break; 1216 break;
1204 default: 1217 default:
diff --git a/drivers/gpu/drm/radeon/r300d.h b/drivers/gpu/drm/radeon/r300d.h
index 0c036c60d9df..1f519a5ffb8c 100644
--- a/drivers/gpu/drm/radeon/r300d.h
+++ b/drivers/gpu/drm/radeon/r300d.h
@@ -54,6 +54,7 @@
54#define PACKET3_3D_DRAW_IMMD_2 0x35 54#define PACKET3_3D_DRAW_IMMD_2 0x35
55#define PACKET3_3D_DRAW_INDX_2 0x36 55#define PACKET3_3D_DRAW_INDX_2 0x36
56#define PACKET3_3D_CLEAR_HIZ 0x37 56#define PACKET3_3D_CLEAR_HIZ 0x37
57#define PACKET3_3D_CLEAR_CMASK 0x38
57#define PACKET3_BITBLT_MULTI 0x9B 58#define PACKET3_BITBLT_MULTI 0x9B
58 59
59#define PACKET0(reg, n) (CP_PACKET0 | \ 60#define PACKET0(reg, n) (CP_PACKET0 | \
diff --git a/drivers/gpu/drm/radeon/r500_reg.h b/drivers/gpu/drm/radeon/r500_reg.h
index 6ac1f604e29b..fc437059918f 100644
--- a/drivers/gpu/drm/radeon/r500_reg.h
+++ b/drivers/gpu/drm/radeon/r500_reg.h
@@ -355,6 +355,8 @@
355#define AVIVO_D1CRTC_FRAME_COUNT 0x60a4 355#define AVIVO_D1CRTC_FRAME_COUNT 0x60a4
356#define AVIVO_D1CRTC_STEREO_CONTROL 0x60c4 356#define AVIVO_D1CRTC_STEREO_CONTROL 0x60c4
357 357
358#define AVIVO_D1MODE_MASTER_UPDATE_MODE 0x60e4
359
358/* master controls */ 360/* master controls */
359#define AVIVO_DC_CRTC_MASTER_EN 0x60f8 361#define AVIVO_DC_CRTC_MASTER_EN 0x60f8
360#define AVIVO_DC_CRTC_TV_CONTROL 0x60fc 362#define AVIVO_DC_CRTC_TV_CONTROL 0x60fc
@@ -409,8 +411,10 @@
409#define AVIVO_D1GRPH_X_END 0x6134 411#define AVIVO_D1GRPH_X_END 0x6134
410#define AVIVO_D1GRPH_Y_END 0x6138 412#define AVIVO_D1GRPH_Y_END 0x6138
411#define AVIVO_D1GRPH_UPDATE 0x6144 413#define AVIVO_D1GRPH_UPDATE 0x6144
414# define AVIVO_D1GRPH_SURFACE_UPDATE_PENDING (1 << 2)
412# define AVIVO_D1GRPH_UPDATE_LOCK (1 << 16) 415# define AVIVO_D1GRPH_UPDATE_LOCK (1 << 16)
413#define AVIVO_D1GRPH_FLIP_CONTROL 0x6148 416#define AVIVO_D1GRPH_FLIP_CONTROL 0x6148
417# define AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN (1 << 0)
414 418
415#define AVIVO_D1CUR_CONTROL 0x6400 419#define AVIVO_D1CUR_CONTROL 0x6400
416# define AVIVO_D1CURSOR_EN (1 << 0) 420# define AVIVO_D1CURSOR_EN (1 << 0)
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 9c92db7c896b..6b50716267c0 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -83,6 +83,9 @@ MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
83MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin"); 83MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
84MODULE_FIRMWARE("radeon/CYPRESS_me.bin"); 84MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
85MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin"); 85MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
86MODULE_FIRMWARE("radeon/PALM_pfp.bin");
87MODULE_FIRMWARE("radeon/PALM_me.bin");
88MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
86 89
87int r600_debugfs_mc_info_init(struct radeon_device *rdev); 90int r600_debugfs_mc_info_init(struct radeon_device *rdev);
88 91
@@ -91,6 +94,7 @@ int r600_mc_wait_for_idle(struct radeon_device *rdev);
91void r600_gpu_init(struct radeon_device *rdev); 94void r600_gpu_init(struct radeon_device *rdev);
92void r600_fini(struct radeon_device *rdev); 95void r600_fini(struct radeon_device *rdev);
93void r600_irq_disable(struct radeon_device *rdev); 96void r600_irq_disable(struct radeon_device *rdev);
97static void r600_pcie_gen2_enable(struct radeon_device *rdev);
94 98
95/* get temperature in millidegrees */ 99/* get temperature in millidegrees */
96u32 rv6xx_get_temp(struct radeon_device *rdev) 100u32 rv6xx_get_temp(struct radeon_device *rdev)
@@ -1164,7 +1168,7 @@ static void r600_mc_program(struct radeon_device *rdev)
1164 * Note: GTT start, end, size should be initialized before calling this 1168 * Note: GTT start, end, size should be initialized before calling this
1165 * function on AGP platform. 1169 * function on AGP platform.
1166 */ 1170 */
1167void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) 1171static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1168{ 1172{
1169 u64 size_bf, size_af; 1173 u64 size_bf, size_af;
1170 1174
@@ -2009,6 +2013,10 @@ int r600_init_microcode(struct radeon_device *rdev)
2009 chip_name = "CYPRESS"; 2013 chip_name = "CYPRESS";
2010 rlc_chip_name = "CYPRESS"; 2014 rlc_chip_name = "CYPRESS";
2011 break; 2015 break;
2016 case CHIP_PALM:
2017 chip_name = "PALM";
2018 rlc_chip_name = "SUMO";
2019 break;
2012 default: BUG(); 2020 default: BUG();
2013 } 2021 }
2014 2022
@@ -2372,6 +2380,9 @@ int r600_startup(struct radeon_device *rdev)
2372{ 2380{
2373 int r; 2381 int r;
2374 2382
2383 /* enable pcie gen2 link */
2384 r600_pcie_gen2_enable(rdev);
2385
2375 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 2386 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2376 r = r600_init_microcode(rdev); 2387 r = r600_init_microcode(rdev);
2377 if (r) { 2388 if (r) {
@@ -2874,6 +2885,8 @@ static void r600_disable_interrupt_state(struct radeon_device *rdev)
2874 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 2885 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2875 WREG32(GRBM_INT_CNTL, 0); 2886 WREG32(GRBM_INT_CNTL, 0);
2876 WREG32(DxMODE_INT_MASK, 0); 2887 WREG32(DxMODE_INT_MASK, 0);
2888 WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
2889 WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
2877 if (ASIC_IS_DCE3(rdev)) { 2890 if (ASIC_IS_DCE3(rdev)) {
2878 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0); 2891 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
2879 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0); 2892 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
@@ -2998,6 +3011,7 @@ int r600_irq_set(struct radeon_device *rdev)
2998 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0; 3011 u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2999 u32 grbm_int_cntl = 0; 3012 u32 grbm_int_cntl = 0;
3000 u32 hdmi1, hdmi2; 3013 u32 hdmi1, hdmi2;
3014 u32 d1grph = 0, d2grph = 0;
3001 3015
3002 if (!rdev->irq.installed) { 3016 if (!rdev->irq.installed) {
3003 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 3017 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
@@ -3034,11 +3048,13 @@ int r600_irq_set(struct radeon_device *rdev)
3034 cp_int_cntl |= RB_INT_ENABLE; 3048 cp_int_cntl |= RB_INT_ENABLE;
3035 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 3049 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3036 } 3050 }
3037 if (rdev->irq.crtc_vblank_int[0]) { 3051 if (rdev->irq.crtc_vblank_int[0] ||
3052 rdev->irq.pflip[0]) {
3038 DRM_DEBUG("r600_irq_set: vblank 0\n"); 3053 DRM_DEBUG("r600_irq_set: vblank 0\n");
3039 mode_int |= D1MODE_VBLANK_INT_MASK; 3054 mode_int |= D1MODE_VBLANK_INT_MASK;
3040 } 3055 }
3041 if (rdev->irq.crtc_vblank_int[1]) { 3056 if (rdev->irq.crtc_vblank_int[1] ||
3057 rdev->irq.pflip[1]) {
3042 DRM_DEBUG("r600_irq_set: vblank 1\n"); 3058 DRM_DEBUG("r600_irq_set: vblank 1\n");
3043 mode_int |= D2MODE_VBLANK_INT_MASK; 3059 mode_int |= D2MODE_VBLANK_INT_MASK;
3044 } 3060 }
@@ -3081,6 +3097,8 @@ int r600_irq_set(struct radeon_device *rdev)
3081 3097
3082 WREG32(CP_INT_CNTL, cp_int_cntl); 3098 WREG32(CP_INT_CNTL, cp_int_cntl);
3083 WREG32(DxMODE_INT_MASK, mode_int); 3099 WREG32(DxMODE_INT_MASK, mode_int);
3100 WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
3101 WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
3084 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 3102 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3085 WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1); 3103 WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
3086 if (ASIC_IS_DCE3(rdev)) { 3104 if (ASIC_IS_DCE3(rdev)) {
@@ -3103,32 +3121,35 @@ int r600_irq_set(struct radeon_device *rdev)
3103 return 0; 3121 return 0;
3104} 3122}
3105 3123
3106static inline void r600_irq_ack(struct radeon_device *rdev, 3124static inline void r600_irq_ack(struct radeon_device *rdev)
3107 u32 *disp_int,
3108 u32 *disp_int_cont,
3109 u32 *disp_int_cont2)
3110{ 3125{
3111 u32 tmp; 3126 u32 tmp;
3112 3127
3113 if (ASIC_IS_DCE3(rdev)) { 3128 if (ASIC_IS_DCE3(rdev)) {
3114 *disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS); 3129 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3115 *disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE); 3130 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3116 *disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2); 3131 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3117 } else { 3132 } else {
3118 *disp_int = RREG32(DISP_INTERRUPT_STATUS); 3133 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3119 *disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 3134 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3120 *disp_int_cont2 = 0; 3135 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3121 } 3136 }
3122 3137 rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3123 if (*disp_int & LB_D1_VBLANK_INTERRUPT) 3138 rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3139
3140 if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3141 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3142 if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3143 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3144 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3124 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK); 3145 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3125 if (*disp_int & LB_D1_VLINE_INTERRUPT) 3146 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3126 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK); 3147 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3127 if (*disp_int & LB_D2_VBLANK_INTERRUPT) 3148 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3128 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK); 3149 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3129 if (*disp_int & LB_D2_VLINE_INTERRUPT) 3150 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3130 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK); 3151 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3131 if (*disp_int & DC_HPD1_INTERRUPT) { 3152 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3132 if (ASIC_IS_DCE3(rdev)) { 3153 if (ASIC_IS_DCE3(rdev)) {
3133 tmp = RREG32(DC_HPD1_INT_CONTROL); 3154 tmp = RREG32(DC_HPD1_INT_CONTROL);
3134 tmp |= DC_HPDx_INT_ACK; 3155 tmp |= DC_HPDx_INT_ACK;
@@ -3139,7 +3160,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
3139 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp); 3160 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3140 } 3161 }
3141 } 3162 }
3142 if (*disp_int & DC_HPD2_INTERRUPT) { 3163 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3143 if (ASIC_IS_DCE3(rdev)) { 3164 if (ASIC_IS_DCE3(rdev)) {
3144 tmp = RREG32(DC_HPD2_INT_CONTROL); 3165 tmp = RREG32(DC_HPD2_INT_CONTROL);
3145 tmp |= DC_HPDx_INT_ACK; 3166 tmp |= DC_HPDx_INT_ACK;
@@ -3150,7 +3171,7 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
3150 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp); 3171 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3151 } 3172 }
3152 } 3173 }
3153 if (*disp_int_cont & DC_HPD3_INTERRUPT) { 3174 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3154 if (ASIC_IS_DCE3(rdev)) { 3175 if (ASIC_IS_DCE3(rdev)) {
3155 tmp = RREG32(DC_HPD3_INT_CONTROL); 3176 tmp = RREG32(DC_HPD3_INT_CONTROL);
3156 tmp |= DC_HPDx_INT_ACK; 3177 tmp |= DC_HPDx_INT_ACK;
@@ -3161,18 +3182,18 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
3161 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp); 3182 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3162 } 3183 }
3163 } 3184 }
3164 if (*disp_int_cont & DC_HPD4_INTERRUPT) { 3185 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3165 tmp = RREG32(DC_HPD4_INT_CONTROL); 3186 tmp = RREG32(DC_HPD4_INT_CONTROL);
3166 tmp |= DC_HPDx_INT_ACK; 3187 tmp |= DC_HPDx_INT_ACK;
3167 WREG32(DC_HPD4_INT_CONTROL, tmp); 3188 WREG32(DC_HPD4_INT_CONTROL, tmp);
3168 } 3189 }
3169 if (ASIC_IS_DCE32(rdev)) { 3190 if (ASIC_IS_DCE32(rdev)) {
3170 if (*disp_int_cont2 & DC_HPD5_INTERRUPT) { 3191 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3171 tmp = RREG32(DC_HPD5_INT_CONTROL); 3192 tmp = RREG32(DC_HPD5_INT_CONTROL);
3172 tmp |= DC_HPDx_INT_ACK; 3193 tmp |= DC_HPDx_INT_ACK;
3173 WREG32(DC_HPD5_INT_CONTROL, tmp); 3194 WREG32(DC_HPD5_INT_CONTROL, tmp);
3174 } 3195 }
3175 if (*disp_int_cont2 & DC_HPD6_INTERRUPT) { 3196 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3176 tmp = RREG32(DC_HPD5_INT_CONTROL); 3197 tmp = RREG32(DC_HPD5_INT_CONTROL);
3177 tmp |= DC_HPDx_INT_ACK; 3198 tmp |= DC_HPDx_INT_ACK;
3178 WREG32(DC_HPD6_INT_CONTROL, tmp); 3199 WREG32(DC_HPD6_INT_CONTROL, tmp);
@@ -3194,12 +3215,10 @@ static inline void r600_irq_ack(struct radeon_device *rdev,
3194 3215
3195void r600_irq_disable(struct radeon_device *rdev) 3216void r600_irq_disable(struct radeon_device *rdev)
3196{ 3217{
3197 u32 disp_int, disp_int_cont, disp_int_cont2;
3198
3199 r600_disable_interrupts(rdev); 3218 r600_disable_interrupts(rdev);
3200 /* Wait and acknowledge irq */ 3219 /* Wait and acknowledge irq */
3201 mdelay(1); 3220 mdelay(1);
3202 r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2); 3221 r600_irq_ack(rdev);
3203 r600_disable_interrupt_state(rdev); 3222 r600_disable_interrupt_state(rdev);
3204} 3223}
3205 3224
@@ -3262,7 +3281,7 @@ int r600_irq_process(struct radeon_device *rdev)
3262 u32 wptr = r600_get_ih_wptr(rdev); 3281 u32 wptr = r600_get_ih_wptr(rdev);
3263 u32 rptr = rdev->ih.rptr; 3282 u32 rptr = rdev->ih.rptr;
3264 u32 src_id, src_data; 3283 u32 src_id, src_data;
3265 u32 ring_index, disp_int, disp_int_cont, disp_int_cont2; 3284 u32 ring_index;
3266 unsigned long flags; 3285 unsigned long flags;
3267 bool queue_hotplug = false; 3286 bool queue_hotplug = false;
3268 3287
@@ -3283,7 +3302,7 @@ int r600_irq_process(struct radeon_device *rdev)
3283 3302
3284restart_ih: 3303restart_ih:
3285 /* display interrupts */ 3304 /* display interrupts */
3286 r600_irq_ack(rdev, &disp_int, &disp_int_cont, &disp_int_cont2); 3305 r600_irq_ack(rdev);
3287 3306
3288 rdev->ih.wptr = wptr; 3307 rdev->ih.wptr = wptr;
3289 while (rptr != wptr) { 3308 while (rptr != wptr) {
@@ -3296,17 +3315,21 @@ restart_ih:
3296 case 1: /* D1 vblank/vline */ 3315 case 1: /* D1 vblank/vline */
3297 switch (src_data) { 3316 switch (src_data) {
3298 case 0: /* D1 vblank */ 3317 case 0: /* D1 vblank */
3299 if (disp_int & LB_D1_VBLANK_INTERRUPT) { 3318 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
3300 drm_handle_vblank(rdev->ddev, 0); 3319 if (rdev->irq.crtc_vblank_int[0]) {
3301 rdev->pm.vblank_sync = true; 3320 drm_handle_vblank(rdev->ddev, 0);
3302 wake_up(&rdev->irq.vblank_queue); 3321 rdev->pm.vblank_sync = true;
3303 disp_int &= ~LB_D1_VBLANK_INTERRUPT; 3322 wake_up(&rdev->irq.vblank_queue);
3323 }
3324 if (rdev->irq.pflip[0])
3325 radeon_crtc_handle_flip(rdev, 0);
3326 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3304 DRM_DEBUG("IH: D1 vblank\n"); 3327 DRM_DEBUG("IH: D1 vblank\n");
3305 } 3328 }
3306 break; 3329 break;
3307 case 1: /* D1 vline */ 3330 case 1: /* D1 vline */
3308 if (disp_int & LB_D1_VLINE_INTERRUPT) { 3331 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
3309 disp_int &= ~LB_D1_VLINE_INTERRUPT; 3332 rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3310 DRM_DEBUG("IH: D1 vline\n"); 3333 DRM_DEBUG("IH: D1 vline\n");
3311 } 3334 }
3312 break; 3335 break;
@@ -3318,17 +3341,21 @@ restart_ih:
3318 case 5: /* D2 vblank/vline */ 3341 case 5: /* D2 vblank/vline */
3319 switch (src_data) { 3342 switch (src_data) {
3320 case 0: /* D2 vblank */ 3343 case 0: /* D2 vblank */
3321 if (disp_int & LB_D2_VBLANK_INTERRUPT) { 3344 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
3322 drm_handle_vblank(rdev->ddev, 1); 3345 if (rdev->irq.crtc_vblank_int[1]) {
3323 rdev->pm.vblank_sync = true; 3346 drm_handle_vblank(rdev->ddev, 1);
3324 wake_up(&rdev->irq.vblank_queue); 3347 rdev->pm.vblank_sync = true;
3325 disp_int &= ~LB_D2_VBLANK_INTERRUPT; 3348 wake_up(&rdev->irq.vblank_queue);
3349 }
3350 if (rdev->irq.pflip[1])
3351 radeon_crtc_handle_flip(rdev, 1);
3352 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
3326 DRM_DEBUG("IH: D2 vblank\n"); 3353 DRM_DEBUG("IH: D2 vblank\n");
3327 } 3354 }
3328 break; 3355 break;
3329 case 1: /* D1 vline */ 3356 case 1: /* D1 vline */
3330 if (disp_int & LB_D2_VLINE_INTERRUPT) { 3357 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
3331 disp_int &= ~LB_D2_VLINE_INTERRUPT; 3358 rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
3332 DRM_DEBUG("IH: D2 vline\n"); 3359 DRM_DEBUG("IH: D2 vline\n");
3333 } 3360 }
3334 break; 3361 break;
@@ -3340,43 +3367,43 @@ restart_ih:
3340 case 19: /* HPD/DAC hotplug */ 3367 case 19: /* HPD/DAC hotplug */
3341 switch (src_data) { 3368 switch (src_data) {
3342 case 0: 3369 case 0:
3343 if (disp_int & DC_HPD1_INTERRUPT) { 3370 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3344 disp_int &= ~DC_HPD1_INTERRUPT; 3371 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
3345 queue_hotplug = true; 3372 queue_hotplug = true;
3346 DRM_DEBUG("IH: HPD1\n"); 3373 DRM_DEBUG("IH: HPD1\n");
3347 } 3374 }
3348 break; 3375 break;
3349 case 1: 3376 case 1:
3350 if (disp_int & DC_HPD2_INTERRUPT) { 3377 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3351 disp_int &= ~DC_HPD2_INTERRUPT; 3378 rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
3352 queue_hotplug = true; 3379 queue_hotplug = true;
3353 DRM_DEBUG("IH: HPD2\n"); 3380 DRM_DEBUG("IH: HPD2\n");
3354 } 3381 }
3355 break; 3382 break;
3356 case 4: 3383 case 4:
3357 if (disp_int_cont & DC_HPD3_INTERRUPT) { 3384 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3358 disp_int_cont &= ~DC_HPD3_INTERRUPT; 3385 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
3359 queue_hotplug = true; 3386 queue_hotplug = true;
3360 DRM_DEBUG("IH: HPD3\n"); 3387 DRM_DEBUG("IH: HPD3\n");
3361 } 3388 }
3362 break; 3389 break;
3363 case 5: 3390 case 5:
3364 if (disp_int_cont & DC_HPD4_INTERRUPT) { 3391 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3365 disp_int_cont &= ~DC_HPD4_INTERRUPT; 3392 rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
3366 queue_hotplug = true; 3393 queue_hotplug = true;
3367 DRM_DEBUG("IH: HPD4\n"); 3394 DRM_DEBUG("IH: HPD4\n");
3368 } 3395 }
3369 break; 3396 break;
3370 case 10: 3397 case 10:
3371 if (disp_int_cont2 & DC_HPD5_INTERRUPT) { 3398 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3372 disp_int_cont2 &= ~DC_HPD5_INTERRUPT; 3399 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
3373 queue_hotplug = true; 3400 queue_hotplug = true;
3374 DRM_DEBUG("IH: HPD5\n"); 3401 DRM_DEBUG("IH: HPD5\n");
3375 } 3402 }
3376 break; 3403 break;
3377 case 12: 3404 case 12:
3378 if (disp_int_cont2 & DC_HPD6_INTERRUPT) { 3405 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3379 disp_int_cont2 &= ~DC_HPD6_INTERRUPT; 3406 rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
3380 queue_hotplug = true; 3407 queue_hotplug = true;
3381 DRM_DEBUG("IH: HPD6\n"); 3408 DRM_DEBUG("IH: HPD6\n");
3382 } 3409 }
@@ -3419,7 +3446,7 @@ restart_ih:
3419 if (wptr != rdev->ih.wptr) 3446 if (wptr != rdev->ih.wptr)
3420 goto restart_ih; 3447 goto restart_ih;
3421 if (queue_hotplug) 3448 if (queue_hotplug)
3422 queue_work(rdev->wq, &rdev->hotplug_work); 3449 schedule_work(&rdev->hotplug_work);
3423 rdev->ih.rptr = rptr; 3450 rdev->ih.rptr = rptr;
3424 WREG32(IH_RB_RPTR, rdev->ih.rptr); 3451 WREG32(IH_RB_RPTR, rdev->ih.rptr);
3425 spin_unlock_irqrestore(&rdev->ih.lock, flags); 3452 spin_unlock_irqrestore(&rdev->ih.lock, flags);
@@ -3508,3 +3535,219 @@ void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
3508 } else 3535 } else
3509 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 3536 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3510} 3537}
3538
3539void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
3540{
3541 u32 link_width_cntl, mask, target_reg;
3542
3543 if (rdev->flags & RADEON_IS_IGP)
3544 return;
3545
3546 if (!(rdev->flags & RADEON_IS_PCIE))
3547 return;
3548
3549 /* x2 cards have a special sequence */
3550 if (ASIC_IS_X2(rdev))
3551 return;
3552
3553 /* FIXME wait for idle */
3554
3555 switch (lanes) {
3556 case 0:
3557 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
3558 break;
3559 case 1:
3560 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
3561 break;
3562 case 2:
3563 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
3564 break;
3565 case 4:
3566 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
3567 break;
3568 case 8:
3569 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
3570 break;
3571 case 12:
3572 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
3573 break;
3574 case 16:
3575 default:
3576 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
3577 break;
3578 }
3579
3580 link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
3581
3582 if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
3583 (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
3584 return;
3585
3586 if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
3587 return;
3588
3589 link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
3590 RADEON_PCIE_LC_RECONFIG_NOW |
3591 R600_PCIE_LC_RENEGOTIATE_EN |
3592 R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
3593 link_width_cntl |= mask;
3594
3595 WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3596
3597 /* some northbridges can renegotiate the link rather than requiring
3598 * a complete re-config.
3599 * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
3600 */
3601 if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
3602 link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
3603 else
3604 link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
3605
3606 WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
3607 RADEON_PCIE_LC_RECONFIG_NOW));
3608
3609 if (rdev->family >= CHIP_RV770)
3610 target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
3611 else
3612 target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
3613
3614 /* wait for lane set to complete */
3615 link_width_cntl = RREG32(target_reg);
3616 while (link_width_cntl == 0xffffffff)
3617 link_width_cntl = RREG32(target_reg);
3618
3619}
3620
3621int r600_get_pcie_lanes(struct radeon_device *rdev)
3622{
3623 u32 link_width_cntl;
3624
3625 if (rdev->flags & RADEON_IS_IGP)
3626 return 0;
3627
3628 if (!(rdev->flags & RADEON_IS_PCIE))
3629 return 0;
3630
3631 /* x2 cards have a special sequence */
3632 if (ASIC_IS_X2(rdev))
3633 return 0;
3634
3635 /* FIXME wait for idle */
3636
3637 link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
3638
3639 switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
3640 case RADEON_PCIE_LC_LINK_WIDTH_X0:
3641 return 0;
3642 case RADEON_PCIE_LC_LINK_WIDTH_X1:
3643 return 1;
3644 case RADEON_PCIE_LC_LINK_WIDTH_X2:
3645 return 2;
3646 case RADEON_PCIE_LC_LINK_WIDTH_X4:
3647 return 4;
3648 case RADEON_PCIE_LC_LINK_WIDTH_X8:
3649 return 8;
3650 case RADEON_PCIE_LC_LINK_WIDTH_X16:
3651 default:
3652 return 16;
3653 }
3654}
3655
3656static void r600_pcie_gen2_enable(struct radeon_device *rdev)
3657{
3658 u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
3659 u16 link_cntl2;
3660
3661 if (rdev->flags & RADEON_IS_IGP)
3662 return;
3663
3664 if (!(rdev->flags & RADEON_IS_PCIE))
3665 return;
3666
3667 /* x2 cards have a special sequence */
3668 if (ASIC_IS_X2(rdev))
3669 return;
3670
3671 /* only RV6xx+ chips are supported */
3672 if (rdev->family <= CHIP_R600)
3673 return;
3674
3675 /* 55 nm r6xx asics */
3676 if ((rdev->family == CHIP_RV670) ||
3677 (rdev->family == CHIP_RV620) ||
3678 (rdev->family == CHIP_RV635)) {
3679 /* advertise upconfig capability */
3680 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3681 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3682 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3683 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3684 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
3685 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
3686 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
3687 LC_RECONFIG_ARC_MISSING_ESCAPE);
3688 link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
3689 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3690 } else {
3691 link_width_cntl |= LC_UPCONFIGURE_DIS;
3692 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3693 }
3694 }
3695
3696 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3697 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3698 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3699
3700 /* 55 nm r6xx asics */
3701 if ((rdev->family == CHIP_RV670) ||
3702 (rdev->family == CHIP_RV620) ||
3703 (rdev->family == CHIP_RV635)) {
3704 WREG32(MM_CFGREGS_CNTL, 0x8);
3705 link_cntl2 = RREG32(0x4088);
3706 WREG32(MM_CFGREGS_CNTL, 0);
3707 /* not supported yet */
3708 if (link_cntl2 & SELECTABLE_DEEMPHASIS)
3709 return;
3710 }
3711
3712 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
3713 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
3714 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
3715 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
3716 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
3717 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3718
3719 tmp = RREG32(0x541c);
3720 WREG32(0x541c, tmp | 0x8);
3721 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
3722 link_cntl2 = RREG16(0x4088);
3723 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
3724 link_cntl2 |= 0x2;
3725 WREG16(0x4088, link_cntl2);
3726 WREG32(MM_CFGREGS_CNTL, 0);
3727
3728 if ((rdev->family == CHIP_RV670) ||
3729 (rdev->family == CHIP_RV620) ||
3730 (rdev->family == CHIP_RV635)) {
3731 training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
3732 training_cntl &= ~LC_POINT_7_PLUS_EN;
3733 WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
3734 } else {
3735 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3736 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3737 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3738 }
3739
3740 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3741 speed_cntl |= LC_GEN2_EN_STRAP;
3742 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3743
3744 } else {
3745 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3746 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3747 if (1)
3748 link_width_cntl |= LC_UPCONFIGURE_DIS;
3749 else
3750 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3751 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3752 }
3753}
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index bff4dc4f410f..a5d898b4bad2 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -728,6 +728,54 @@
728/* DCE 3.2 */ 728/* DCE 3.2 */
729# define DC_HPDx_EN (1 << 28) 729# define DC_HPDx_EN (1 << 28)
730 730
731#define D1GRPH_INTERRUPT_STATUS 0x6158
732#define D2GRPH_INTERRUPT_STATUS 0x6958
733# define DxGRPH_PFLIP_INT_OCCURRED (1 << 0)
734# define DxGRPH_PFLIP_INT_CLEAR (1 << 8)
735#define D1GRPH_INTERRUPT_CONTROL 0x615c
736#define D2GRPH_INTERRUPT_CONTROL 0x695c
737# define DxGRPH_PFLIP_INT_MASK (1 << 0)
738# define DxGRPH_PFLIP_INT_TYPE (1 << 8)
739
740/* PCIE link stuff */
741#define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */
742# define LC_POINT_7_PLUS_EN (1 << 6)
743#define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */
744# define LC_LINK_WIDTH_SHIFT 0
745# define LC_LINK_WIDTH_MASK 0x7
746# define LC_LINK_WIDTH_X0 0
747# define LC_LINK_WIDTH_X1 1
748# define LC_LINK_WIDTH_X2 2
749# define LC_LINK_WIDTH_X4 3
750# define LC_LINK_WIDTH_X8 4
751# define LC_LINK_WIDTH_X16 6
752# define LC_LINK_WIDTH_RD_SHIFT 4
753# define LC_LINK_WIDTH_RD_MASK 0x70
754# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7)
755# define LC_RECONFIG_NOW (1 << 8)
756# define LC_RENEGOTIATION_SUPPORT (1 << 9)
757# define LC_RENEGOTIATE_EN (1 << 10)
758# define LC_SHORT_RECONFIG_EN (1 << 11)
759# define LC_UPCONFIGURE_SUPPORT (1 << 12)
760# define LC_UPCONFIGURE_DIS (1 << 13)
761#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */
762# define LC_GEN2_EN_STRAP (1 << 0)
763# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1)
764# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 5)
765# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 6)
766# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8)
767# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3
768# define LC_CURRENT_DATA_RATE (1 << 11)
769# define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14)
770# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21)
771# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23)
772# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24)
773#define MM_CFGREGS_CNTL 0x544c
774# define MM_WR_TO_CFG_EN (1 << 3)
775#define LINK_CNTL2 0x88 /* F0 */
776# define TARGET_LINK_SPEED_MASK (0xf << 0)
777# define SELECTABLE_DEEMPHASIS (1 << 6)
778
731/* 779/*
732 * PM4 780 * PM4
733 */ 781 */
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 3a7095743d44..e9486630a467 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -69,6 +69,7 @@
69#include <ttm/ttm_bo_driver.h> 69#include <ttm/ttm_bo_driver.h>
70#include <ttm/ttm_placement.h> 70#include <ttm/ttm_placement.h>
71#include <ttm/ttm_module.h> 71#include <ttm/ttm_module.h>
72#include <ttm/ttm_execbuf_util.h>
72 73
73#include "radeon_family.h" 74#include "radeon_family.h"
74#include "radeon_mode.h" 75#include "radeon_mode.h"
@@ -180,6 +181,7 @@ void rs690_pm_info(struct radeon_device *rdev);
180extern u32 rv6xx_get_temp(struct radeon_device *rdev); 181extern u32 rv6xx_get_temp(struct radeon_device *rdev);
181extern u32 rv770_get_temp(struct radeon_device *rdev); 182extern u32 rv770_get_temp(struct radeon_device *rdev);
182extern u32 evergreen_get_temp(struct radeon_device *rdev); 183extern u32 evergreen_get_temp(struct radeon_device *rdev);
184extern u32 sumo_get_temp(struct radeon_device *rdev);
183 185
184/* 186/*
185 * Fences. 187 * Fences.
@@ -259,13 +261,12 @@ struct radeon_bo {
259}; 261};
260 262
261struct radeon_bo_list { 263struct radeon_bo_list {
262 struct list_head list; 264 struct ttm_validate_buffer tv;
263 struct radeon_bo *bo; 265 struct radeon_bo *bo;
264 uint64_t gpu_offset; 266 uint64_t gpu_offset;
265 unsigned rdomain; 267 unsigned rdomain;
266 unsigned wdomain; 268 unsigned wdomain;
267 u32 tiling_flags; 269 u32 tiling_flags;
268 bool reserved;
269}; 270};
270 271
271/* 272/*
@@ -377,11 +378,56 @@ void radeon_scratch_free(struct radeon_device *rdev, uint32_t reg);
377/* 378/*
378 * IRQS. 379 * IRQS.
379 */ 380 */
381
382struct radeon_unpin_work {
383 struct work_struct work;
384 struct radeon_device *rdev;
385 int crtc_id;
386 struct radeon_fence *fence;
387 struct drm_pending_vblank_event *event;
388 struct radeon_bo *old_rbo;
389 u64 new_crtc_base;
390};
391
392struct r500_irq_stat_regs {
393 u32 disp_int;
394};
395
396struct r600_irq_stat_regs {
397 u32 disp_int;
398 u32 disp_int_cont;
399 u32 disp_int_cont2;
400 u32 d1grph_int;
401 u32 d2grph_int;
402};
403
404struct evergreen_irq_stat_regs {
405 u32 disp_int;
406 u32 disp_int_cont;
407 u32 disp_int_cont2;
408 u32 disp_int_cont3;
409 u32 disp_int_cont4;
410 u32 disp_int_cont5;
411 u32 d1grph_int;
412 u32 d2grph_int;
413 u32 d3grph_int;
414 u32 d4grph_int;
415 u32 d5grph_int;
416 u32 d6grph_int;
417};
418
419union radeon_irq_stat_regs {
420 struct r500_irq_stat_regs r500;
421 struct r600_irq_stat_regs r600;
422 struct evergreen_irq_stat_regs evergreen;
423};
424
380struct radeon_irq { 425struct radeon_irq {
381 bool installed; 426 bool installed;
382 bool sw_int; 427 bool sw_int;
383 /* FIXME: use a define max crtc rather than hardcode it */ 428 /* FIXME: use a define max crtc rather than hardcode it */
384 bool crtc_vblank_int[6]; 429 bool crtc_vblank_int[6];
430 bool pflip[6];
385 wait_queue_head_t vblank_queue; 431 wait_queue_head_t vblank_queue;
386 /* FIXME: use defines for max hpd/dacs */ 432 /* FIXME: use defines for max hpd/dacs */
387 bool hpd[6]; 433 bool hpd[6];
@@ -392,12 +438,17 @@ struct radeon_irq {
392 bool hdmi[2]; 438 bool hdmi[2];
393 spinlock_t sw_lock; 439 spinlock_t sw_lock;
394 int sw_refcount; 440 int sw_refcount;
441 union radeon_irq_stat_regs stat_regs;
442 spinlock_t pflip_lock[6];
443 int pflip_refcount[6];
395}; 444};
396 445
397int radeon_irq_kms_init(struct radeon_device *rdev); 446int radeon_irq_kms_init(struct radeon_device *rdev);
398void radeon_irq_kms_fini(struct radeon_device *rdev); 447void radeon_irq_kms_fini(struct radeon_device *rdev);
399void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev); 448void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev);
400void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev); 449void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev);
450void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc);
451void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc);
401 452
402/* 453/*
403 * CP & ring. 454 * CP & ring.
@@ -687,6 +738,8 @@ enum radeon_int_thermal_type {
687 THERMAL_TYPE_RV6XX, 738 THERMAL_TYPE_RV6XX,
688 THERMAL_TYPE_RV770, 739 THERMAL_TYPE_RV770,
689 THERMAL_TYPE_EVERGREEN, 740 THERMAL_TYPE_EVERGREEN,
741 THERMAL_TYPE_SUMO,
742 THERMAL_TYPE_NI,
690}; 743};
691 744
692struct radeon_voltage { 745struct radeon_voltage {
@@ -770,6 +823,9 @@ struct radeon_pm {
770 u32 current_sclk; 823 u32 current_sclk;
771 u32 current_mclk; 824 u32 current_mclk;
772 u32 current_vddc; 825 u32 current_vddc;
826 u32 default_sclk;
827 u32 default_mclk;
828 u32 default_vddc;
773 struct radeon_i2c_chan *i2c_bus; 829 struct radeon_i2c_chan *i2c_bus;
774 /* selected pm method */ 830 /* selected pm method */
775 enum radeon_pm_method pm_method; 831 enum radeon_pm_method pm_method;
@@ -881,6 +937,10 @@ struct radeon_asic {
881 void (*pm_finish)(struct radeon_device *rdev); 937 void (*pm_finish)(struct radeon_device *rdev);
882 void (*pm_init_profile)(struct radeon_device *rdev); 938 void (*pm_init_profile)(struct radeon_device *rdev);
883 void (*pm_get_dynpm_state)(struct radeon_device *rdev); 939 void (*pm_get_dynpm_state)(struct radeon_device *rdev);
940 /* pageflipping */
941 void (*pre_page_flip)(struct radeon_device *rdev, int crtc);
942 u32 (*page_flip)(struct radeon_device *rdev, int crtc, u64 crtc_base);
943 void (*post_page_flip)(struct radeon_device *rdev, int crtc);
884}; 944};
885 945
886/* 946/*
@@ -975,6 +1035,7 @@ struct evergreen_asic {
975 unsigned tiling_npipes; 1035 unsigned tiling_npipes;
976 unsigned tiling_group_size; 1036 unsigned tiling_group_size;
977 unsigned tile_config; 1037 unsigned tile_config;
1038 struct r100_gpu_lockup lockup;
978}; 1039};
979 1040
980union radeon_asic_config { 1041union radeon_asic_config {
@@ -1091,11 +1152,11 @@ struct radeon_device {
1091 const struct firmware *me_fw; /* all family ME firmware */ 1152 const struct firmware *me_fw; /* all family ME firmware */
1092 const struct firmware *pfp_fw; /* r6/700 PFP firmware */ 1153 const struct firmware *pfp_fw; /* r6/700 PFP firmware */
1093 const struct firmware *rlc_fw; /* r6/700 RLC firmware */ 1154 const struct firmware *rlc_fw; /* r6/700 RLC firmware */
1155 const struct firmware *mc_fw; /* NI MC firmware */
1094 struct r600_blit r600_blit; 1156 struct r600_blit r600_blit;
1095 struct r700_vram_scratch vram_scratch; 1157 struct r700_vram_scratch vram_scratch;
1096 int msi_enabled; /* msi enabled */ 1158 int msi_enabled; /* msi enabled */
1097 struct r600_ih ih; /* r6/700 interrupt ring */ 1159 struct r600_ih ih; /* r6/700 interrupt ring */
1098 struct workqueue_struct *wq;
1099 struct work_struct hotplug_work; 1160 struct work_struct hotplug_work;
1100 int num_crtc; /* number of crtcs */ 1161 int num_crtc; /* number of crtcs */
1101 struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */ 1162 struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */
@@ -1110,10 +1171,10 @@ struct radeon_device {
1110 uint8_t audio_status_bits; 1171 uint8_t audio_status_bits;
1111 uint8_t audio_category_code; 1172 uint8_t audio_category_code;
1112 1173
1113 bool powered_down;
1114 struct notifier_block acpi_nb; 1174 struct notifier_block acpi_nb;
1115 /* only one userspace can use Hyperz features at a time */ 1175 /* only one userspace can use Hyperz features or CMASK at a time */
1116 struct drm_file *hyperz_filp; 1176 struct drm_file *hyperz_filp;
1177 struct drm_file *cmask_filp;
1117 /* i2c buses */ 1178 /* i2c buses */
1118 struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS]; 1179 struct radeon_i2c_chan *i2c_bus[RADEON_MAX_I2C_BUS];
1119}; 1180};
@@ -1188,6 +1249,8 @@ static inline void r100_io_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1188 */ 1249 */
1189#define RREG8(reg) readb(((void __iomem *)rdev->rmmio) + (reg)) 1250#define RREG8(reg) readb(((void __iomem *)rdev->rmmio) + (reg))
1190#define WREG8(reg, v) writeb(v, ((void __iomem *)rdev->rmmio) + (reg)) 1251#define WREG8(reg, v) writeb(v, ((void __iomem *)rdev->rmmio) + (reg))
1252#define RREG16(reg) readw(((void __iomem *)rdev->rmmio) + (reg))
1253#define WREG16(reg, v) writew(v, ((void __iomem *)rdev->rmmio) + (reg))
1191#define RREG32(reg) r100_mm_rreg(rdev, (reg)) 1254#define RREG32(reg) r100_mm_rreg(rdev, (reg))
1192#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg))) 1255#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", r100_mm_rreg(rdev, (reg)))
1193#define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v)) 1256#define WREG32(reg, v) r100_mm_wreg(rdev, (reg), (v))
@@ -1261,6 +1324,14 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
1261 (rdev->family == CHIP_RV410) || \ 1324 (rdev->family == CHIP_RV410) || \
1262 (rdev->family == CHIP_RS400) || \ 1325 (rdev->family == CHIP_RS400) || \
1263 (rdev->family == CHIP_RS480)) 1326 (rdev->family == CHIP_RS480))
1327#define ASIC_IS_X2(rdev) ((rdev->ddev->pdev->device == 0x9441) || \
1328 (rdev->ddev->pdev->device == 0x9443) || \
1329 (rdev->ddev->pdev->device == 0x944B) || \
1330 (rdev->ddev->pdev->device == 0x9506) || \
1331 (rdev->ddev->pdev->device == 0x9509) || \
1332 (rdev->ddev->pdev->device == 0x950F) || \
1333 (rdev->ddev->pdev->device == 0x689C) || \
1334 (rdev->ddev->pdev->device == 0x689D))
1264#define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600)) 1335#define ASIC_IS_AVIVO(rdev) ((rdev->family >= CHIP_RS600))
1265#define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600) || \ 1336#define ASIC_IS_DCE2(rdev) ((rdev->family == CHIP_RS600) || \
1266 (rdev->family == CHIP_RS690) || \ 1337 (rdev->family == CHIP_RS690) || \
@@ -1269,6 +1340,9 @@ void r100_pll_errata_after_index(struct radeon_device *rdev);
1269#define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620)) 1340#define ASIC_IS_DCE3(rdev) ((rdev->family >= CHIP_RV620))
1270#define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730)) 1341#define ASIC_IS_DCE32(rdev) ((rdev->family >= CHIP_RV730))
1271#define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR)) 1342#define ASIC_IS_DCE4(rdev) ((rdev->family >= CHIP_CEDAR))
1343#define ASIC_IS_DCE41(rdev) ((rdev->family >= CHIP_PALM) && \
1344 (rdev->flags & RADEON_IS_IGP))
1345#define ASIC_IS_DCE5(rdev) ((rdev->family >= CHIP_BARTS))
1272 1346
1273/* 1347/*
1274 * BIOS helpers. 1348 * BIOS helpers.
@@ -1344,6 +1418,9 @@ static inline void radeon_ring_write(struct radeon_device *rdev, uint32_t v)
1344#define radeon_pm_finish(rdev) (rdev)->asic->pm_finish((rdev)) 1418#define radeon_pm_finish(rdev) (rdev)->asic->pm_finish((rdev))
1345#define radeon_pm_init_profile(rdev) (rdev)->asic->pm_init_profile((rdev)) 1419#define radeon_pm_init_profile(rdev) (rdev)->asic->pm_init_profile((rdev))
1346#define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm_get_dynpm_state((rdev)) 1420#define radeon_pm_get_dynpm_state(rdev) (rdev)->asic->pm_get_dynpm_state((rdev))
1421#define radeon_pre_page_flip(rdev, crtc) rdev->asic->pre_page_flip((rdev), (crtc))
1422#define radeon_page_flip(rdev, crtc, base) rdev->asic->page_flip((rdev), (crtc), (base))
1423#define radeon_post_page_flip(rdev, crtc) rdev->asic->post_page_flip((rdev), (crtc))
1347 1424
1348/* Common functions */ 1425/* Common functions */
1349/* AGP */ 1426/* AGP */
@@ -1372,67 +1449,7 @@ extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc
1372extern int radeon_resume_kms(struct drm_device *dev); 1449extern int radeon_resume_kms(struct drm_device *dev);
1373extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state); 1450extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
1374 1451
1375/* r100,rv100,rs100,rv200,rs200,r200,rv250,rs300,rv280 */
1376extern void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
1377extern bool r100_gpu_cp_is_lockup(struct radeon_device *rdev, struct r100_gpu_lockup *lockup, struct radeon_cp *cp);
1378
1379/* rv200,rv250,rv280 */
1380extern void r200_set_safe_registers(struct radeon_device *rdev);
1381
1382/* r300,r350,rv350,rv370,rv380 */
1383extern void r300_set_reg_safe(struct radeon_device *rdev);
1384extern void r300_mc_program(struct radeon_device *rdev);
1385extern void r300_mc_init(struct radeon_device *rdev);
1386extern void r300_clock_startup(struct radeon_device *rdev);
1387extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
1388extern int rv370_pcie_gart_init(struct radeon_device *rdev);
1389extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
1390extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
1391extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
1392
1393/* r420,r423,rv410 */
1394extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
1395extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
1396extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
1397extern void r420_pipes_init(struct radeon_device *rdev);
1398
1399/* rv515 */
1400struct rv515_mc_save {
1401 u32 d1vga_control;
1402 u32 d2vga_control;
1403 u32 vga_render_control;
1404 u32 vga_hdp_control;
1405 u32 d1crtc_control;
1406 u32 d2crtc_control;
1407};
1408extern void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
1409extern void rv515_vga_render_disable(struct radeon_device *rdev);
1410extern void rv515_set_safe_registers(struct radeon_device *rdev);
1411extern void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
1412extern void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
1413extern void rv515_clock_startup(struct radeon_device *rdev);
1414extern void rv515_debugfs(struct radeon_device *rdev);
1415extern int rv515_suspend(struct radeon_device *rdev);
1416
1417/* rs400 */
1418extern int rs400_gart_init(struct radeon_device *rdev);
1419extern int rs400_gart_enable(struct radeon_device *rdev);
1420extern void rs400_gart_adjust_size(struct radeon_device *rdev);
1421extern void rs400_gart_disable(struct radeon_device *rdev);
1422extern void rs400_gart_fini(struct radeon_device *rdev);
1423
1424/* rs600 */
1425extern void rs600_set_safe_registers(struct radeon_device *rdev);
1426extern int rs600_irq_set(struct radeon_device *rdev);
1427extern void rs600_irq_disable(struct radeon_device *rdev);
1428
1429/* rs690, rs740 */
1430extern void rs690_line_buffer_adjust(struct radeon_device *rdev,
1431 struct drm_display_mode *mode1,
1432 struct drm_display_mode *mode2);
1433
1434/* r600, rv610, rv630, rv620, rv635, rv670, rs780, rs880 */ 1452/* r600, rv610, rv630, rv620, rv635, rv670, rs780, rs880 */
1435extern void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
1436extern bool r600_card_posted(struct radeon_device *rdev); 1453extern bool r600_card_posted(struct radeon_device *rdev);
1437extern void r600_cp_stop(struct radeon_device *rdev); 1454extern void r600_cp_stop(struct radeon_device *rdev);
1438extern int r600_cp_start(struct radeon_device *rdev); 1455extern int r600_cp_start(struct radeon_device *rdev);
@@ -1478,6 +1495,7 @@ extern void r600_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mo
1478extern int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder); 1495extern int r600_hdmi_buffer_status_changed(struct drm_encoder *encoder);
1479extern void r600_hdmi_update_audio_settings(struct drm_encoder *encoder); 1496extern void r600_hdmi_update_audio_settings(struct drm_encoder *encoder);
1480 1497
1498extern void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
1481extern void r700_cp_stop(struct radeon_device *rdev); 1499extern void r700_cp_stop(struct radeon_device *rdev);
1482extern void r700_cp_fini(struct radeon_device *rdev); 1500extern void r700_cp_fini(struct radeon_device *rdev);
1483extern void evergreen_disable_interrupt_state(struct radeon_device *rdev); 1501extern void evergreen_disable_interrupt_state(struct radeon_device *rdev);
@@ -1485,6 +1503,9 @@ extern int evergreen_irq_set(struct radeon_device *rdev);
1485extern int evergreen_blit_init(struct radeon_device *rdev); 1503extern int evergreen_blit_init(struct radeon_device *rdev);
1486extern void evergreen_blit_fini(struct radeon_device *rdev); 1504extern void evergreen_blit_fini(struct radeon_device *rdev);
1487 1505
1506extern int ni_init_microcode(struct radeon_device *rdev);
1507extern int btc_mc_load_microcode(struct radeon_device *rdev);
1508
1488/* radeon_acpi.c */ 1509/* radeon_acpi.c */
1489#if defined(CONFIG_ACPI) 1510#if defined(CONFIG_ACPI)
1490extern int radeon_acpi_init(struct radeon_device *rdev); 1511extern int radeon_acpi_init(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 64fb89ecbf74..3a1b16186224 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -94,7 +94,7 @@ static void radeon_register_accessor_init(struct radeon_device *rdev)
94 rdev->mc_rreg = &rs600_mc_rreg; 94 rdev->mc_rreg = &rs600_mc_rreg;
95 rdev->mc_wreg = &rs600_mc_wreg; 95 rdev->mc_wreg = &rs600_mc_wreg;
96 } 96 }
97 if ((rdev->family >= CHIP_R600) && (rdev->family <= CHIP_RV740)) { 97 if ((rdev->family >= CHIP_R600) && (rdev->family <= CHIP_HEMLOCK)) {
98 rdev->pciep_rreg = &r600_pciep_rreg; 98 rdev->pciep_rreg = &r600_pciep_rreg;
99 rdev->pciep_wreg = &r600_pciep_wreg; 99 rdev->pciep_wreg = &r600_pciep_wreg;
100 } 100 }
@@ -171,6 +171,9 @@ static struct radeon_asic r100_asic = {
171 .pm_finish = &r100_pm_finish, 171 .pm_finish = &r100_pm_finish,
172 .pm_init_profile = &r100_pm_init_profile, 172 .pm_init_profile = &r100_pm_init_profile,
173 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 173 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
174 .pre_page_flip = &r100_pre_page_flip,
175 .page_flip = &r100_page_flip,
176 .post_page_flip = &r100_post_page_flip,
174}; 177};
175 178
176static struct radeon_asic r200_asic = { 179static struct radeon_asic r200_asic = {
@@ -215,6 +218,9 @@ static struct radeon_asic r200_asic = {
215 .pm_finish = &r100_pm_finish, 218 .pm_finish = &r100_pm_finish,
216 .pm_init_profile = &r100_pm_init_profile, 219 .pm_init_profile = &r100_pm_init_profile,
217 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 220 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
221 .pre_page_flip = &r100_pre_page_flip,
222 .page_flip = &r100_page_flip,
223 .post_page_flip = &r100_post_page_flip,
218}; 224};
219 225
220static struct radeon_asic r300_asic = { 226static struct radeon_asic r300_asic = {
@@ -260,6 +266,9 @@ static struct radeon_asic r300_asic = {
260 .pm_finish = &r100_pm_finish, 266 .pm_finish = &r100_pm_finish,
261 .pm_init_profile = &r100_pm_init_profile, 267 .pm_init_profile = &r100_pm_init_profile,
262 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 268 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
269 .pre_page_flip = &r100_pre_page_flip,
270 .page_flip = &r100_page_flip,
271 .post_page_flip = &r100_post_page_flip,
263}; 272};
264 273
265static struct radeon_asic r300_asic_pcie = { 274static struct radeon_asic r300_asic_pcie = {
@@ -304,6 +313,9 @@ static struct radeon_asic r300_asic_pcie = {
304 .pm_finish = &r100_pm_finish, 313 .pm_finish = &r100_pm_finish,
305 .pm_init_profile = &r100_pm_init_profile, 314 .pm_init_profile = &r100_pm_init_profile,
306 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 315 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
316 .pre_page_flip = &r100_pre_page_flip,
317 .page_flip = &r100_page_flip,
318 .post_page_flip = &r100_post_page_flip,
307}; 319};
308 320
309static struct radeon_asic r420_asic = { 321static struct radeon_asic r420_asic = {
@@ -349,6 +361,9 @@ static struct radeon_asic r420_asic = {
349 .pm_finish = &r100_pm_finish, 361 .pm_finish = &r100_pm_finish,
350 .pm_init_profile = &r420_pm_init_profile, 362 .pm_init_profile = &r420_pm_init_profile,
351 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 363 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
364 .pre_page_flip = &r100_pre_page_flip,
365 .page_flip = &r100_page_flip,
366 .post_page_flip = &r100_post_page_flip,
352}; 367};
353 368
354static struct radeon_asic rs400_asic = { 369static struct radeon_asic rs400_asic = {
@@ -394,6 +409,9 @@ static struct radeon_asic rs400_asic = {
394 .pm_finish = &r100_pm_finish, 409 .pm_finish = &r100_pm_finish,
395 .pm_init_profile = &r100_pm_init_profile, 410 .pm_init_profile = &r100_pm_init_profile,
396 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 411 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
412 .pre_page_flip = &r100_pre_page_flip,
413 .page_flip = &r100_page_flip,
414 .post_page_flip = &r100_post_page_flip,
397}; 415};
398 416
399static struct radeon_asic rs600_asic = { 417static struct radeon_asic rs600_asic = {
@@ -439,6 +457,9 @@ static struct radeon_asic rs600_asic = {
439 .pm_finish = &rs600_pm_finish, 457 .pm_finish = &rs600_pm_finish,
440 .pm_init_profile = &r420_pm_init_profile, 458 .pm_init_profile = &r420_pm_init_profile,
441 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 459 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
460 .pre_page_flip = &rs600_pre_page_flip,
461 .page_flip = &rs600_page_flip,
462 .post_page_flip = &rs600_post_page_flip,
442}; 463};
443 464
444static struct radeon_asic rs690_asic = { 465static struct radeon_asic rs690_asic = {
@@ -484,6 +505,9 @@ static struct radeon_asic rs690_asic = {
484 .pm_finish = &rs600_pm_finish, 505 .pm_finish = &rs600_pm_finish,
485 .pm_init_profile = &r420_pm_init_profile, 506 .pm_init_profile = &r420_pm_init_profile,
486 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 507 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
508 .pre_page_flip = &rs600_pre_page_flip,
509 .page_flip = &rs600_page_flip,
510 .post_page_flip = &rs600_post_page_flip,
487}; 511};
488 512
489static struct radeon_asic rv515_asic = { 513static struct radeon_asic rv515_asic = {
@@ -529,6 +553,9 @@ static struct radeon_asic rv515_asic = {
529 .pm_finish = &rs600_pm_finish, 553 .pm_finish = &rs600_pm_finish,
530 .pm_init_profile = &r420_pm_init_profile, 554 .pm_init_profile = &r420_pm_init_profile,
531 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 555 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
556 .pre_page_flip = &rs600_pre_page_flip,
557 .page_flip = &rs600_page_flip,
558 .post_page_flip = &rs600_post_page_flip,
532}; 559};
533 560
534static struct radeon_asic r520_asic = { 561static struct radeon_asic r520_asic = {
@@ -574,6 +601,9 @@ static struct radeon_asic r520_asic = {
574 .pm_finish = &rs600_pm_finish, 601 .pm_finish = &rs600_pm_finish,
575 .pm_init_profile = &r420_pm_init_profile, 602 .pm_init_profile = &r420_pm_init_profile,
576 .pm_get_dynpm_state = &r100_pm_get_dynpm_state, 603 .pm_get_dynpm_state = &r100_pm_get_dynpm_state,
604 .pre_page_flip = &rs600_pre_page_flip,
605 .page_flip = &rs600_page_flip,
606 .post_page_flip = &rs600_post_page_flip,
577}; 607};
578 608
579static struct radeon_asic r600_asic = { 609static struct radeon_asic r600_asic = {
@@ -601,8 +631,8 @@ static struct radeon_asic r600_asic = {
601 .set_engine_clock = &radeon_atom_set_engine_clock, 631 .set_engine_clock = &radeon_atom_set_engine_clock,
602 .get_memory_clock = &radeon_atom_get_memory_clock, 632 .get_memory_clock = &radeon_atom_get_memory_clock,
603 .set_memory_clock = &radeon_atom_set_memory_clock, 633 .set_memory_clock = &radeon_atom_set_memory_clock,
604 .get_pcie_lanes = &rv370_get_pcie_lanes, 634 .get_pcie_lanes = &r600_get_pcie_lanes,
605 .set_pcie_lanes = NULL, 635 .set_pcie_lanes = &r600_set_pcie_lanes,
606 .set_clock_gating = NULL, 636 .set_clock_gating = NULL,
607 .set_surface_reg = r600_set_surface_reg, 637 .set_surface_reg = r600_set_surface_reg,
608 .clear_surface_reg = r600_clear_surface_reg, 638 .clear_surface_reg = r600_clear_surface_reg,
@@ -618,6 +648,9 @@ static struct radeon_asic r600_asic = {
618 .pm_finish = &rs600_pm_finish, 648 .pm_finish = &rs600_pm_finish,
619 .pm_init_profile = &r600_pm_init_profile, 649 .pm_init_profile = &r600_pm_init_profile,
620 .pm_get_dynpm_state = &r600_pm_get_dynpm_state, 650 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
651 .pre_page_flip = &rs600_pre_page_flip,
652 .page_flip = &rs600_page_flip,
653 .post_page_flip = &rs600_post_page_flip,
621}; 654};
622 655
623static struct radeon_asic rs780_asic = { 656static struct radeon_asic rs780_asic = {
@@ -662,6 +695,9 @@ static struct radeon_asic rs780_asic = {
662 .pm_finish = &rs600_pm_finish, 695 .pm_finish = &rs600_pm_finish,
663 .pm_init_profile = &rs780_pm_init_profile, 696 .pm_init_profile = &rs780_pm_init_profile,
664 .pm_get_dynpm_state = &r600_pm_get_dynpm_state, 697 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
698 .pre_page_flip = &rs600_pre_page_flip,
699 .page_flip = &rs600_page_flip,
700 .post_page_flip = &rs600_post_page_flip,
665}; 701};
666 702
667static struct radeon_asic rv770_asic = { 703static struct radeon_asic rv770_asic = {
@@ -689,8 +725,8 @@ static struct radeon_asic rv770_asic = {
689 .set_engine_clock = &radeon_atom_set_engine_clock, 725 .set_engine_clock = &radeon_atom_set_engine_clock,
690 .get_memory_clock = &radeon_atom_get_memory_clock, 726 .get_memory_clock = &radeon_atom_get_memory_clock,
691 .set_memory_clock = &radeon_atom_set_memory_clock, 727 .set_memory_clock = &radeon_atom_set_memory_clock,
692 .get_pcie_lanes = &rv370_get_pcie_lanes, 728 .get_pcie_lanes = &r600_get_pcie_lanes,
693 .set_pcie_lanes = NULL, 729 .set_pcie_lanes = &r600_set_pcie_lanes,
694 .set_clock_gating = &radeon_atom_set_clock_gating, 730 .set_clock_gating = &radeon_atom_set_clock_gating,
695 .set_surface_reg = r600_set_surface_reg, 731 .set_surface_reg = r600_set_surface_reg,
696 .clear_surface_reg = r600_clear_surface_reg, 732 .clear_surface_reg = r600_clear_surface_reg,
@@ -706,6 +742,9 @@ static struct radeon_asic rv770_asic = {
706 .pm_finish = &rs600_pm_finish, 742 .pm_finish = &rs600_pm_finish,
707 .pm_init_profile = &r600_pm_init_profile, 743 .pm_init_profile = &r600_pm_init_profile,
708 .pm_get_dynpm_state = &r600_pm_get_dynpm_state, 744 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
745 .pre_page_flip = &rs600_pre_page_flip,
746 .page_flip = &rv770_page_flip,
747 .post_page_flip = &rs600_post_page_flip,
709}; 748};
710 749
711static struct radeon_asic evergreen_asic = { 750static struct radeon_asic evergreen_asic = {
@@ -733,6 +772,95 @@ static struct radeon_asic evergreen_asic = {
733 .set_engine_clock = &radeon_atom_set_engine_clock, 772 .set_engine_clock = &radeon_atom_set_engine_clock,
734 .get_memory_clock = &radeon_atom_get_memory_clock, 773 .get_memory_clock = &radeon_atom_get_memory_clock,
735 .set_memory_clock = &radeon_atom_set_memory_clock, 774 .set_memory_clock = &radeon_atom_set_memory_clock,
775 .get_pcie_lanes = &r600_get_pcie_lanes,
776 .set_pcie_lanes = &r600_set_pcie_lanes,
777 .set_clock_gating = NULL,
778 .set_surface_reg = r600_set_surface_reg,
779 .clear_surface_reg = r600_clear_surface_reg,
780 .bandwidth_update = &evergreen_bandwidth_update,
781 .hpd_init = &evergreen_hpd_init,
782 .hpd_fini = &evergreen_hpd_fini,
783 .hpd_sense = &evergreen_hpd_sense,
784 .hpd_set_polarity = &evergreen_hpd_set_polarity,
785 .gui_idle = &r600_gui_idle,
786 .pm_misc = &evergreen_pm_misc,
787 .pm_prepare = &evergreen_pm_prepare,
788 .pm_finish = &evergreen_pm_finish,
789 .pm_init_profile = &r600_pm_init_profile,
790 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
791 .pre_page_flip = &evergreen_pre_page_flip,
792 .page_flip = &evergreen_page_flip,
793 .post_page_flip = &evergreen_post_page_flip,
794};
795
796static struct radeon_asic sumo_asic = {
797 .init = &evergreen_init,
798 .fini = &evergreen_fini,
799 .suspend = &evergreen_suspend,
800 .resume = &evergreen_resume,
801 .cp_commit = &r600_cp_commit,
802 .gpu_is_lockup = &evergreen_gpu_is_lockup,
803 .asic_reset = &evergreen_asic_reset,
804 .vga_set_state = &r600_vga_set_state,
805 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
806 .gart_set_page = &rs600_gart_set_page,
807 .ring_test = &r600_ring_test,
808 .ring_ib_execute = &r600_ring_ib_execute,
809 .irq_set = &evergreen_irq_set,
810 .irq_process = &evergreen_irq_process,
811 .get_vblank_counter = &evergreen_get_vblank_counter,
812 .fence_ring_emit = &r600_fence_ring_emit,
813 .cs_parse = &evergreen_cs_parse,
814 .copy_blit = &evergreen_copy_blit,
815 .copy_dma = &evergreen_copy_blit,
816 .copy = &evergreen_copy_blit,
817 .get_engine_clock = &radeon_atom_get_engine_clock,
818 .set_engine_clock = &radeon_atom_set_engine_clock,
819 .get_memory_clock = NULL,
820 .set_memory_clock = NULL,
821 .get_pcie_lanes = NULL,
822 .set_pcie_lanes = NULL,
823 .set_clock_gating = NULL,
824 .set_surface_reg = r600_set_surface_reg,
825 .clear_surface_reg = r600_clear_surface_reg,
826 .bandwidth_update = &evergreen_bandwidth_update,
827 .hpd_init = &evergreen_hpd_init,
828 .hpd_fini = &evergreen_hpd_fini,
829 .hpd_sense = &evergreen_hpd_sense,
830 .hpd_set_polarity = &evergreen_hpd_set_polarity,
831 .gui_idle = &r600_gui_idle,
832 .pm_misc = &evergreen_pm_misc,
833 .pm_prepare = &evergreen_pm_prepare,
834 .pm_finish = &evergreen_pm_finish,
835 .pm_init_profile = &rs780_pm_init_profile,
836 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
837};
838
839static struct radeon_asic btc_asic = {
840 .init = &evergreen_init,
841 .fini = &evergreen_fini,
842 .suspend = &evergreen_suspend,
843 .resume = &evergreen_resume,
844 .cp_commit = &r600_cp_commit,
845 .gpu_is_lockup = &evergreen_gpu_is_lockup,
846 .asic_reset = &evergreen_asic_reset,
847 .vga_set_state = &r600_vga_set_state,
848 .gart_tlb_flush = &evergreen_pcie_gart_tlb_flush,
849 .gart_set_page = &rs600_gart_set_page,
850 .ring_test = &r600_ring_test,
851 .ring_ib_execute = &r600_ring_ib_execute,
852 .irq_set = &evergreen_irq_set,
853 .irq_process = &evergreen_irq_process,
854 .get_vblank_counter = &evergreen_get_vblank_counter,
855 .fence_ring_emit = &r600_fence_ring_emit,
856 .cs_parse = &evergreen_cs_parse,
857 .copy_blit = &evergreen_copy_blit,
858 .copy_dma = &evergreen_copy_blit,
859 .copy = &evergreen_copy_blit,
860 .get_engine_clock = &radeon_atom_get_engine_clock,
861 .set_engine_clock = &radeon_atom_set_engine_clock,
862 .get_memory_clock = &radeon_atom_get_memory_clock,
863 .set_memory_clock = &radeon_atom_set_memory_clock,
736 .get_pcie_lanes = NULL, 864 .get_pcie_lanes = NULL,
737 .set_pcie_lanes = NULL, 865 .set_pcie_lanes = NULL,
738 .set_clock_gating = NULL, 866 .set_clock_gating = NULL,
@@ -749,6 +877,9 @@ static struct radeon_asic evergreen_asic = {
749 .pm_finish = &evergreen_pm_finish, 877 .pm_finish = &evergreen_pm_finish,
750 .pm_init_profile = &r600_pm_init_profile, 878 .pm_init_profile = &r600_pm_init_profile,
751 .pm_get_dynpm_state = &r600_pm_get_dynpm_state, 879 .pm_get_dynpm_state = &r600_pm_get_dynpm_state,
880 .pre_page_flip = &evergreen_pre_page_flip,
881 .page_flip = &evergreen_page_flip,
882 .post_page_flip = &evergreen_post_page_flip,
752}; 883};
753 884
754int radeon_asic_init(struct radeon_device *rdev) 885int radeon_asic_init(struct radeon_device *rdev)
@@ -835,6 +966,14 @@ int radeon_asic_init(struct radeon_device *rdev)
835 case CHIP_HEMLOCK: 966 case CHIP_HEMLOCK:
836 rdev->asic = &evergreen_asic; 967 rdev->asic = &evergreen_asic;
837 break; 968 break;
969 case CHIP_PALM:
970 rdev->asic = &sumo_asic;
971 break;
972 case CHIP_BARTS:
973 case CHIP_TURKS:
974 case CHIP_CAICOS:
975 rdev->asic = &btc_asic;
976 break;
838 default: 977 default:
839 /* FIXME: not supported yet */ 978 /* FIXME: not supported yet */
840 return -EINVAL; 979 return -EINVAL;
@@ -849,7 +988,9 @@ int radeon_asic_init(struct radeon_device *rdev)
849 if (rdev->flags & RADEON_SINGLE_CRTC) 988 if (rdev->flags & RADEON_SINGLE_CRTC)
850 rdev->num_crtc = 1; 989 rdev->num_crtc = 1;
851 else { 990 else {
852 if (ASIC_IS_DCE4(rdev)) 991 if (ASIC_IS_DCE41(rdev))
992 rdev->num_crtc = 2;
993 else if (ASIC_IS_DCE4(rdev))
853 rdev->num_crtc = 6; 994 rdev->num_crtc = 6;
854 else 995 else
855 rdev->num_crtc = 2; 996 rdev->num_crtc = 2;
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 740988244143..e01f07718539 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -102,6 +102,11 @@ int r100_pci_gart_enable(struct radeon_device *rdev);
102void r100_pci_gart_disable(struct radeon_device *rdev); 102void r100_pci_gart_disable(struct radeon_device *rdev);
103int r100_debugfs_mc_info_init(struct radeon_device *rdev); 103int r100_debugfs_mc_info_init(struct radeon_device *rdev);
104int r100_gui_wait_for_idle(struct radeon_device *rdev); 104int r100_gui_wait_for_idle(struct radeon_device *rdev);
105void r100_gpu_lockup_update(struct r100_gpu_lockup *lockup,
106 struct radeon_cp *cp);
107bool r100_gpu_cp_is_lockup(struct radeon_device *rdev,
108 struct r100_gpu_lockup *lockup,
109 struct radeon_cp *cp);
105void r100_ib_fini(struct radeon_device *rdev); 110void r100_ib_fini(struct radeon_device *rdev);
106int r100_ib_init(struct radeon_device *rdev); 111int r100_ib_init(struct radeon_device *rdev);
107void r100_irq_disable(struct radeon_device *rdev); 112void r100_irq_disable(struct radeon_device *rdev);
@@ -130,15 +135,19 @@ extern void r100_pm_prepare(struct radeon_device *rdev);
130extern void r100_pm_finish(struct radeon_device *rdev); 135extern void r100_pm_finish(struct radeon_device *rdev);
131extern void r100_pm_init_profile(struct radeon_device *rdev); 136extern void r100_pm_init_profile(struct radeon_device *rdev);
132extern void r100_pm_get_dynpm_state(struct radeon_device *rdev); 137extern void r100_pm_get_dynpm_state(struct radeon_device *rdev);
138extern void r100_pre_page_flip(struct radeon_device *rdev, int crtc);
139extern u32 r100_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
140extern void r100_post_page_flip(struct radeon_device *rdev, int crtc);
133 141
134/* 142/*
135 * r200,rv250,rs300,rv280 143 * r200,rv250,rs300,rv280
136 */ 144 */
137extern int r200_copy_dma(struct radeon_device *rdev, 145extern int r200_copy_dma(struct radeon_device *rdev,
138 uint64_t src_offset, 146 uint64_t src_offset,
139 uint64_t dst_offset, 147 uint64_t dst_offset,
140 unsigned num_pages, 148 unsigned num_pages,
141 struct radeon_fence *fence); 149 struct radeon_fence *fence);
150void r200_set_safe_registers(struct radeon_device *rdev);
142 151
143/* 152/*
144 * r300,r350,rv350,rv380 153 * r300,r350,rv350,rv380
@@ -159,6 +168,15 @@ extern uint32_t rv370_pcie_rreg(struct radeon_device *rdev, uint32_t reg);
159extern void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); 168extern void rv370_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
160extern void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes); 169extern void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes);
161extern int rv370_get_pcie_lanes(struct radeon_device *rdev); 170extern int rv370_get_pcie_lanes(struct radeon_device *rdev);
171extern void r300_set_reg_safe(struct radeon_device *rdev);
172extern void r300_mc_program(struct radeon_device *rdev);
173extern void r300_mc_init(struct radeon_device *rdev);
174extern void r300_clock_startup(struct radeon_device *rdev);
175extern int r300_mc_wait_for_idle(struct radeon_device *rdev);
176extern int rv370_pcie_gart_init(struct radeon_device *rdev);
177extern void rv370_pcie_gart_fini(struct radeon_device *rdev);
178extern int rv370_pcie_gart_enable(struct radeon_device *rdev);
179extern void rv370_pcie_gart_disable(struct radeon_device *rdev);
162 180
163/* 181/*
164 * r420,r423,rv410 182 * r420,r423,rv410
@@ -168,6 +186,10 @@ extern void r420_fini(struct radeon_device *rdev);
168extern int r420_suspend(struct radeon_device *rdev); 186extern int r420_suspend(struct radeon_device *rdev);
169extern int r420_resume(struct radeon_device *rdev); 187extern int r420_resume(struct radeon_device *rdev);
170extern void r420_pm_init_profile(struct radeon_device *rdev); 188extern void r420_pm_init_profile(struct radeon_device *rdev);
189extern u32 r420_mc_rreg(struct radeon_device *rdev, u32 reg);
190extern void r420_mc_wreg(struct radeon_device *rdev, u32 reg, u32 v);
191extern int r420_debugfs_pipes_info_init(struct radeon_device *rdev);
192extern void r420_pipes_init(struct radeon_device *rdev);
171 193
172/* 194/*
173 * rs400,rs480 195 * rs400,rs480
@@ -180,6 +202,12 @@ void rs400_gart_tlb_flush(struct radeon_device *rdev);
180int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); 202int rs400_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
181uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg); 203uint32_t rs400_mc_rreg(struct radeon_device *rdev, uint32_t reg);
182void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); 204void rs400_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
205int rs400_gart_init(struct radeon_device *rdev);
206int rs400_gart_enable(struct radeon_device *rdev);
207void rs400_gart_adjust_size(struct radeon_device *rdev);
208void rs400_gart_disable(struct radeon_device *rdev);
209void rs400_gart_fini(struct radeon_device *rdev);
210
183 211
184/* 212/*
185 * rs600. 213 * rs600.
@@ -191,6 +219,7 @@ extern int rs600_suspend(struct radeon_device *rdev);
191extern int rs600_resume(struct radeon_device *rdev); 219extern int rs600_resume(struct radeon_device *rdev);
192int rs600_irq_set(struct radeon_device *rdev); 220int rs600_irq_set(struct radeon_device *rdev);
193int rs600_irq_process(struct radeon_device *rdev); 221int rs600_irq_process(struct radeon_device *rdev);
222void rs600_irq_disable(struct radeon_device *rdev);
194u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc); 223u32 rs600_get_vblank_counter(struct radeon_device *rdev, int crtc);
195void rs600_gart_tlb_flush(struct radeon_device *rdev); 224void rs600_gart_tlb_flush(struct radeon_device *rdev);
196int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr); 225int rs600_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr);
@@ -205,6 +234,11 @@ void rs600_hpd_set_polarity(struct radeon_device *rdev,
205extern void rs600_pm_misc(struct radeon_device *rdev); 234extern void rs600_pm_misc(struct radeon_device *rdev);
206extern void rs600_pm_prepare(struct radeon_device *rdev); 235extern void rs600_pm_prepare(struct radeon_device *rdev);
207extern void rs600_pm_finish(struct radeon_device *rdev); 236extern void rs600_pm_finish(struct radeon_device *rdev);
237extern void rs600_pre_page_flip(struct radeon_device *rdev, int crtc);
238extern u32 rs600_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
239extern void rs600_post_page_flip(struct radeon_device *rdev, int crtc);
240void rs600_set_safe_registers(struct radeon_device *rdev);
241
208 242
209/* 243/*
210 * rs690,rs740 244 * rs690,rs740
@@ -216,10 +250,21 @@ int rs690_suspend(struct radeon_device *rdev);
216uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg); 250uint32_t rs690_mc_rreg(struct radeon_device *rdev, uint32_t reg);
217void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v); 251void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
218void rs690_bandwidth_update(struct radeon_device *rdev); 252void rs690_bandwidth_update(struct radeon_device *rdev);
253void rs690_line_buffer_adjust(struct radeon_device *rdev,
254 struct drm_display_mode *mode1,
255 struct drm_display_mode *mode2);
219 256
220/* 257/*
221 * rv515 258 * rv515
222 */ 259 */
260struct rv515_mc_save {
261 u32 d1vga_control;
262 u32 d2vga_control;
263 u32 vga_render_control;
264 u32 vga_hdp_control;
265 u32 d1crtc_control;
266 u32 d2crtc_control;
267};
223int rv515_init(struct radeon_device *rdev); 268int rv515_init(struct radeon_device *rdev);
224void rv515_fini(struct radeon_device *rdev); 269void rv515_fini(struct radeon_device *rdev);
225uint32_t rv515_mc_rreg(struct radeon_device *rdev, uint32_t reg); 270uint32_t rv515_mc_rreg(struct radeon_device *rdev, uint32_t reg);
@@ -230,6 +275,14 @@ void rv515_pcie_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
230void rv515_bandwidth_update(struct radeon_device *rdev); 275void rv515_bandwidth_update(struct radeon_device *rdev);
231int rv515_resume(struct radeon_device *rdev); 276int rv515_resume(struct radeon_device *rdev);
232int rv515_suspend(struct radeon_device *rdev); 277int rv515_suspend(struct radeon_device *rdev);
278void rv515_bandwidth_avivo_update(struct radeon_device *rdev);
279void rv515_vga_render_disable(struct radeon_device *rdev);
280void rv515_set_safe_registers(struct radeon_device *rdev);
281void rv515_mc_stop(struct radeon_device *rdev, struct rv515_mc_save *save);
282void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save);
283void rv515_clock_startup(struct radeon_device *rdev);
284void rv515_debugfs(struct radeon_device *rdev);
285
233 286
234/* 287/*
235 * r520,rv530,rv560,rv570,r580 288 * r520,rv530,rv560,rv570,r580
@@ -278,6 +331,8 @@ extern void r600_pm_misc(struct radeon_device *rdev);
278extern void r600_pm_init_profile(struct radeon_device *rdev); 331extern void r600_pm_init_profile(struct radeon_device *rdev);
279extern void rs780_pm_init_profile(struct radeon_device *rdev); 332extern void rs780_pm_init_profile(struct radeon_device *rdev);
280extern void r600_pm_get_dynpm_state(struct radeon_device *rdev); 333extern void r600_pm_get_dynpm_state(struct radeon_device *rdev);
334extern void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes);
335extern int r600_get_pcie_lanes(struct radeon_device *rdev);
281 336
282/* 337/*
283 * rv770,rv730,rv710,rv740 338 * rv770,rv730,rv710,rv740
@@ -287,6 +342,7 @@ void rv770_fini(struct radeon_device *rdev);
287int rv770_suspend(struct radeon_device *rdev); 342int rv770_suspend(struct radeon_device *rdev);
288int rv770_resume(struct radeon_device *rdev); 343int rv770_resume(struct radeon_device *rdev);
289extern void rv770_pm_misc(struct radeon_device *rdev); 344extern void rv770_pm_misc(struct radeon_device *rdev);
345extern u32 rv770_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
290 346
291/* 347/*
292 * evergreen 348 * evergreen
@@ -314,5 +370,8 @@ extern int evergreen_cs_parse(struct radeon_cs_parser *p);
314extern void evergreen_pm_misc(struct radeon_device *rdev); 370extern void evergreen_pm_misc(struct radeon_device *rdev);
315extern void evergreen_pm_prepare(struct radeon_device *rdev); 371extern void evergreen_pm_prepare(struct radeon_device *rdev);
316extern void evergreen_pm_finish(struct radeon_device *rdev); 372extern void evergreen_pm_finish(struct radeon_device *rdev);
373extern void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc);
374extern u32 evergreen_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
375extern void evergreen_post_page_flip(struct radeon_device *rdev, int crtc);
317 376
318#endif 377#endif
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index bc5a2c3382d9..1573202a6418 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -37,7 +37,7 @@ radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device,
37extern void radeon_link_encoder_connector(struct drm_device *dev); 37extern void radeon_link_encoder_connector(struct drm_device *dev);
38extern void 38extern void
39radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, 39radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum,
40 uint32_t supported_device); 40 uint32_t supported_device, u16 caps);
41 41
42/* from radeon_connector.c */ 42/* from radeon_connector.c */
43extern void 43extern void
@@ -313,7 +313,6 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
313 uint16_t *line_mux, 313 uint16_t *line_mux,
314 struct radeon_hpd *hpd) 314 struct radeon_hpd *hpd)
315{ 315{
316 struct radeon_device *rdev = dev->dev_private;
317 316
318 /* Asus M2A-VM HDMI board lists the DVI port as HDMI */ 317 /* Asus M2A-VM HDMI board lists the DVI port as HDMI */
319 if ((dev->pdev->device == 0x791e) && 318 if ((dev->pdev->device == 0x791e) &&
@@ -388,6 +387,17 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
388 *line_mux = 0x90; 387 *line_mux = 0x90;
389 } 388 }
390 389
390 /* mac rv630 */
391 if ((dev->pdev->device == 0x9588) &&
392 (dev->pdev->subsystem_vendor == 0x106b) &&
393 (dev->pdev->subsystem_device == 0x00a6)) {
394 if ((supported_device == ATOM_DEVICE_TV1_SUPPORT) &&
395 (*connector_type == DRM_MODE_CONNECTOR_DVII)) {
396 *connector_type = DRM_MODE_CONNECTOR_9PinDIN;
397 *line_mux = CONNECTOR_7PIN_DIN_ENUM_ID1;
398 }
399 }
400
391 /* ASUS HD 3600 XT board lists the DVI port as HDMI */ 401 /* ASUS HD 3600 XT board lists the DVI port as HDMI */
392 if ((dev->pdev->device == 0x9598) && 402 if ((dev->pdev->device == 0x9598) &&
393 (dev->pdev->subsystem_vendor == 0x1043) && 403 (dev->pdev->subsystem_vendor == 0x1043) &&
@@ -425,21 +435,23 @@ static bool radeon_atom_apply_quirks(struct drm_device *dev,
425 } 435 }
426 } 436 }
427 437
428 /* Acer laptop reports DVI-D as DVI-I and hpd pins reversed */ 438 /* Acer laptop (Acer TravelMate 5730G) has an HDMI port
439 * on the laptop and a DVI port on the docking station and
440 * both share the same encoder, hpd pin, and ddc line.
441 * So while the bios table is technically correct,
442 * we drop the DVI port here since xrandr has no concept of
443 * encoders and will try and drive both connectors
444 * with different crtcs which isn't possible on the hardware
445 * side and leaves no crtcs for LVDS or VGA.
446 */
429 if ((dev->pdev->device == 0x95c4) && 447 if ((dev->pdev->device == 0x95c4) &&
430 (dev->pdev->subsystem_vendor == 0x1025) && 448 (dev->pdev->subsystem_vendor == 0x1025) &&
431 (dev->pdev->subsystem_device == 0x013c)) { 449 (dev->pdev->subsystem_device == 0x013c)) {
432 struct radeon_gpio_rec gpio;
433
434 if ((*connector_type == DRM_MODE_CONNECTOR_DVII) && 450 if ((*connector_type == DRM_MODE_CONNECTOR_DVII) &&
435 (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) { 451 (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
436 gpio = radeon_lookup_gpio(rdev, 6); 452 /* actually it's a DVI-D port not DVI-I */
437 *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
438 *connector_type = DRM_MODE_CONNECTOR_DVID; 453 *connector_type = DRM_MODE_CONNECTOR_DVID;
439 } else if ((*connector_type == DRM_MODE_CONNECTOR_HDMIA) && 454 return false;
440 (supported_device == ATOM_DEVICE_DFP1_SUPPORT)) {
441 gpio = radeon_lookup_gpio(rdev, 7);
442 *hpd = radeon_atom_get_hpd_info_from_gpio(rdev, &gpio);
443 } 455 }
444 } 456 }
445 457
@@ -525,6 +537,7 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
525 u16 size, data_offset; 537 u16 size, data_offset;
526 u8 frev, crev; 538 u8 frev, crev;
527 ATOM_CONNECTOR_OBJECT_TABLE *con_obj; 539 ATOM_CONNECTOR_OBJECT_TABLE *con_obj;
540 ATOM_ENCODER_OBJECT_TABLE *enc_obj;
528 ATOM_OBJECT_TABLE *router_obj; 541 ATOM_OBJECT_TABLE *router_obj;
529 ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj; 542 ATOM_DISPLAY_OBJECT_PATH_TABLE *path_obj;
530 ATOM_OBJECT_HEADER *obj_header; 543 ATOM_OBJECT_HEADER *obj_header;
@@ -549,6 +562,9 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
549 con_obj = (ATOM_CONNECTOR_OBJECT_TABLE *) 562 con_obj = (ATOM_CONNECTOR_OBJECT_TABLE *)
550 (ctx->bios + data_offset + 563 (ctx->bios + data_offset +
551 le16_to_cpu(obj_header->usConnectorObjectTableOffset)); 564 le16_to_cpu(obj_header->usConnectorObjectTableOffset));
565 enc_obj = (ATOM_ENCODER_OBJECT_TABLE *)
566 (ctx->bios + data_offset +
567 le16_to_cpu(obj_header->usEncoderObjectTableOffset));
552 router_obj = (ATOM_OBJECT_TABLE *) 568 router_obj = (ATOM_OBJECT_TABLE *)
553 (ctx->bios + data_offset + 569 (ctx->bios + data_offset +
554 le16_to_cpu(obj_header->usRouterObjectTableOffset)); 570 le16_to_cpu(obj_header->usRouterObjectTableOffset));
@@ -654,14 +670,35 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
654 OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT; 670 OBJECT_TYPE_MASK) >> OBJECT_TYPE_SHIFT;
655 671
656 if (grph_obj_type == GRAPH_OBJECT_TYPE_ENCODER) { 672 if (grph_obj_type == GRAPH_OBJECT_TYPE_ENCODER) {
657 u16 encoder_obj = le16_to_cpu(path->usGraphicObjIds[j]); 673 for (k = 0; k < enc_obj->ucNumberOfObjects; k++) {
658 674 u16 encoder_obj = le16_to_cpu(enc_obj->asObjects[k].usObjectID);
659 radeon_add_atom_encoder(dev, 675 if (le16_to_cpu(path->usGraphicObjIds[j]) == encoder_obj) {
660 encoder_obj, 676 ATOM_COMMON_RECORD_HEADER *record = (ATOM_COMMON_RECORD_HEADER *)
661 le16_to_cpu 677 (ctx->bios + data_offset +
662 (path-> 678 le16_to_cpu(enc_obj->asObjects[k].usRecordOffset));
663 usDeviceTag)); 679 ATOM_ENCODER_CAP_RECORD *cap_record;
680 u16 caps = 0;
664 681
682 while (record->ucRecordType > 0 &&
683 record->ucRecordType <= ATOM_MAX_OBJECT_RECORD_NUMBER) {
684 switch (record->ucRecordType) {
685 case ATOM_ENCODER_CAP_RECORD_TYPE:
686 cap_record =(ATOM_ENCODER_CAP_RECORD *)
687 record;
688 caps = le16_to_cpu(cap_record->usEncoderCap);
689 break;
690 }
691 record = (ATOM_COMMON_RECORD_HEADER *)
692 ((char *)record + record->ucRecordSize);
693 }
694 radeon_add_atom_encoder(dev,
695 encoder_obj,
696 le16_to_cpu
697 (path->
698 usDeviceTag),
699 caps);
700 }
701 }
665 } else if (grph_obj_type == GRAPH_OBJECT_TYPE_ROUTER) { 702 } else if (grph_obj_type == GRAPH_OBJECT_TYPE_ROUTER) {
666 for (k = 0; k < router_obj->ucNumberOfObjects; k++) { 703 for (k = 0; k < router_obj->ucNumberOfObjects; k++) {
667 u16 router_obj_id = le16_to_cpu(router_obj->asObjects[k].usObjectID); 704 u16 router_obj_id = le16_to_cpu(router_obj->asObjects[k].usObjectID);
@@ -995,7 +1032,8 @@ bool radeon_get_atom_connector_info_from_supported_devices_table(struct
995 radeon_get_encoder_enum(dev, 1032 radeon_get_encoder_enum(dev,
996 (1 << i), 1033 (1 << i),
997 dac), 1034 dac),
998 (1 << i)); 1035 (1 << i),
1036 0);
999 else 1037 else
1000 radeon_add_legacy_encoder(dev, 1038 radeon_add_legacy_encoder(dev,
1001 radeon_get_encoder_enum(dev, 1039 radeon_get_encoder_enum(dev,
@@ -1074,6 +1112,7 @@ union firmware_info {
1074 ATOM_FIRMWARE_INFO_V1_3 info_13; 1112 ATOM_FIRMWARE_INFO_V1_3 info_13;
1075 ATOM_FIRMWARE_INFO_V1_4 info_14; 1113 ATOM_FIRMWARE_INFO_V1_4 info_14;
1076 ATOM_FIRMWARE_INFO_V2_1 info_21; 1114 ATOM_FIRMWARE_INFO_V2_1 info_21;
1115 ATOM_FIRMWARE_INFO_V2_2 info_22;
1077}; 1116};
1078 1117
1079bool radeon_atom_get_clock_info(struct drm_device *dev) 1118bool radeon_atom_get_clock_info(struct drm_device *dev)
@@ -1148,8 +1187,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
1148 *p2pll = *p1pll; 1187 *p2pll = *p1pll;
1149 1188
1150 /* system clock */ 1189 /* system clock */
1151 spll->reference_freq = 1190 if (ASIC_IS_DCE4(rdev))
1152 le16_to_cpu(firmware_info->info.usReferenceClock); 1191 spll->reference_freq =
1192 le16_to_cpu(firmware_info->info_21.usCoreReferenceClock);
1193 else
1194 spll->reference_freq =
1195 le16_to_cpu(firmware_info->info.usReferenceClock);
1153 spll->reference_div = 0; 1196 spll->reference_div = 0;
1154 1197
1155 spll->pll_out_min = 1198 spll->pll_out_min =
@@ -1171,8 +1214,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
1171 le16_to_cpu(firmware_info->info.usMaxEngineClockPLL_Input); 1214 le16_to_cpu(firmware_info->info.usMaxEngineClockPLL_Input);
1172 1215
1173 /* memory clock */ 1216 /* memory clock */
1174 mpll->reference_freq = 1217 if (ASIC_IS_DCE4(rdev))
1175 le16_to_cpu(firmware_info->info.usReferenceClock); 1218 mpll->reference_freq =
1219 le16_to_cpu(firmware_info->info_21.usMemoryReferenceClock);
1220 else
1221 mpll->reference_freq =
1222 le16_to_cpu(firmware_info->info.usReferenceClock);
1176 mpll->reference_div = 0; 1223 mpll->reference_div = 0;
1177 1224
1178 mpll->pll_out_min = 1225 mpll->pll_out_min =
@@ -1201,8 +1248,12 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
1201 if (ASIC_IS_DCE4(rdev)) { 1248 if (ASIC_IS_DCE4(rdev)) {
1202 rdev->clock.default_dispclk = 1249 rdev->clock.default_dispclk =
1203 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq); 1250 le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq);
1204 if (rdev->clock.default_dispclk == 0) 1251 if (rdev->clock.default_dispclk == 0) {
1205 rdev->clock.default_dispclk = 60000; /* 600 Mhz */ 1252 if (ASIC_IS_DCE5(rdev))
1253 rdev->clock.default_dispclk = 54000; /* 540 Mhz */
1254 else
1255 rdev->clock.default_dispclk = 60000; /* 600 Mhz */
1256 }
1206 rdev->clock.dp_extclk = 1257 rdev->clock.dp_extclk =
1207 le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq); 1258 le16_to_cpu(firmware_info->info_21.usUniphyDPModeExtClkFreq);
1208 } 1259 }
@@ -1337,6 +1388,43 @@ bool radeon_atombios_get_ppll_ss_info(struct radeon_device *rdev,
1337 return false; 1388 return false;
1338} 1389}
1339 1390
1391static void radeon_atombios_get_igp_ss_overrides(struct radeon_device *rdev,
1392 struct radeon_atom_ss *ss,
1393 int id)
1394{
1395 struct radeon_mode_info *mode_info = &rdev->mode_info;
1396 int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1397 u16 data_offset, size;
1398 struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 *igp_info;
1399 u8 frev, crev;
1400 u16 percentage = 0, rate = 0;
1401
1402 /* get any igp specific overrides */
1403 if (atom_parse_data_header(mode_info->atom_context, index, &size,
1404 &frev, &crev, &data_offset)) {
1405 igp_info = (struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 *)
1406 (mode_info->atom_context->bios + data_offset);
1407 switch (id) {
1408 case ASIC_INTERNAL_SS_ON_TMDS:
1409 percentage = le16_to_cpu(igp_info->usDVISSPercentage);
1410 rate = le16_to_cpu(igp_info->usDVISSpreadRateIn10Hz);
1411 break;
1412 case ASIC_INTERNAL_SS_ON_HDMI:
1413 percentage = le16_to_cpu(igp_info->usHDMISSPercentage);
1414 rate = le16_to_cpu(igp_info->usHDMISSpreadRateIn10Hz);
1415 break;
1416 case ASIC_INTERNAL_SS_ON_LVDS:
1417 percentage = le16_to_cpu(igp_info->usLvdsSSPercentage);
1418 rate = le16_to_cpu(igp_info->usLvdsSSpreadRateIn10Hz);
1419 break;
1420 }
1421 if (percentage)
1422 ss->percentage = percentage;
1423 if (rate)
1424 ss->rate = rate;
1425 }
1426}
1427
1340union asic_ss_info { 1428union asic_ss_info {
1341 struct _ATOM_ASIC_INTERNAL_SS_INFO info; 1429 struct _ATOM_ASIC_INTERNAL_SS_INFO info;
1342 struct _ATOM_ASIC_INTERNAL_SS_INFO_V2 info_2; 1430 struct _ATOM_ASIC_INTERNAL_SS_INFO_V2 info_2;
@@ -1401,6 +1489,8 @@ bool radeon_atombios_get_asic_ss_info(struct radeon_device *rdev,
1401 le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadSpectrumPercentage); 1489 le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadSpectrumPercentage);
1402 ss->type = ss_info->info_3.asSpreadSpectrum[i].ucSpreadSpectrumMode; 1490 ss->type = ss_info->info_3.asSpreadSpectrum[i].ucSpreadSpectrumMode;
1403 ss->rate = le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadRateIn10Hz); 1491 ss->rate = le16_to_cpu(ss_info->info_3.asSpreadSpectrum[i].usSpreadRateIn10Hz);
1492 if (rdev->flags & RADEON_IS_IGP)
1493 radeon_atombios_get_igp_ss_overrides(rdev, ss, id);
1404 return true; 1494 return true;
1405 } 1495 }
1406 } 1496 }
@@ -1477,6 +1567,9 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
1477 if (misc & ATOM_DOUBLE_CLOCK_MODE) 1567 if (misc & ATOM_DOUBLE_CLOCK_MODE)
1478 lvds->native_mode.flags |= DRM_MODE_FLAG_DBLSCAN; 1568 lvds->native_mode.flags |= DRM_MODE_FLAG_DBLSCAN;
1479 1569
1570 lvds->native_mode.width_mm = lvds_info->info.sLCDTiming.usImageHSize;
1571 lvds->native_mode.height_mm = lvds_info->info.sLCDTiming.usImageVSize;
1572
1480 /* set crtc values */ 1573 /* set crtc values */
1481 drm_mode_set_crtcinfo(&lvds->native_mode, CRTC_INTERLACE_HALVE_V); 1574 drm_mode_set_crtcinfo(&lvds->native_mode, CRTC_INTERLACE_HALVE_V);
1482 1575
@@ -1489,6 +1582,59 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct
1489 else 1582 else
1490 lvds->linkb = false; 1583 lvds->linkb = false;
1491 1584
1585 /* parse the lcd record table */
1586 if (lvds_info->info.usModePatchTableOffset) {
1587 ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record;
1588 ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record;
1589 bool bad_record = false;
1590 u8 *record = (u8 *)(mode_info->atom_context->bios +
1591 data_offset +
1592 lvds_info->info.usModePatchTableOffset);
1593 while (*record != ATOM_RECORD_END_TYPE) {
1594 switch (*record) {
1595 case LCD_MODE_PATCH_RECORD_MODE_TYPE:
1596 record += sizeof(ATOM_PATCH_RECORD_MODE);
1597 break;
1598 case LCD_RTS_RECORD_TYPE:
1599 record += sizeof(ATOM_LCD_RTS_RECORD);
1600 break;
1601 case LCD_CAP_RECORD_TYPE:
1602 record += sizeof(ATOM_LCD_MODE_CONTROL_CAP);
1603 break;
1604 case LCD_FAKE_EDID_PATCH_RECORD_TYPE:
1605 fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record;
1606 if (fake_edid_record->ucFakeEDIDLength) {
1607 struct edid *edid;
1608 int edid_size =
1609 max((int)EDID_LENGTH, (int)fake_edid_record->ucFakeEDIDLength);
1610 edid = kmalloc(edid_size, GFP_KERNEL);
1611 if (edid) {
1612 memcpy((u8 *)edid, (u8 *)&fake_edid_record->ucFakeEDIDString[0],
1613 fake_edid_record->ucFakeEDIDLength);
1614
1615 if (drm_edid_is_valid(edid))
1616 rdev->mode_info.bios_hardcoded_edid = edid;
1617 else
1618 kfree(edid);
1619 }
1620 }
1621 record += sizeof(ATOM_FAKE_EDID_PATCH_RECORD);
1622 break;
1623 case LCD_PANEL_RESOLUTION_RECORD_TYPE:
1624 panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record;
1625 lvds->native_mode.width_mm = panel_res_record->usHSize;
1626 lvds->native_mode.height_mm = panel_res_record->usVSize;
1627 record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD);
1628 break;
1629 default:
1630 DRM_ERROR("Bad LCD record %d\n", *record);
1631 bad_record = true;
1632 break;
1633 }
1634 if (bad_record)
1635 break;
1636 }
1637 }
1492 } 1638 }
1493 return lvds; 1639 return lvds;
1494} 1640}
@@ -1740,496 +1886,614 @@ static const char *pp_lib_thermal_controller_names[] = {
1740 "RV6xx", 1886 "RV6xx",
1741 "RV770", 1887 "RV770",
1742 "adt7473", 1888 "adt7473",
1889 "NONE",
1743 "External GPIO", 1890 "External GPIO",
1744 "Evergreen", 1891 "Evergreen",
1745 "adt7473 with internal", 1892 "emc2103",
1746 1893 "Sumo",
1894 "Northern Islands",
1747}; 1895};
1748 1896
1749union power_info { 1897union power_info {
1750 struct _ATOM_POWERPLAY_INFO info; 1898 struct _ATOM_POWERPLAY_INFO info;
1751 struct _ATOM_POWERPLAY_INFO_V2 info_2; 1899 struct _ATOM_POWERPLAY_INFO_V2 info_2;
1752 struct _ATOM_POWERPLAY_INFO_V3 info_3; 1900 struct _ATOM_POWERPLAY_INFO_V3 info_3;
1753 struct _ATOM_PPLIB_POWERPLAYTABLE info_4; 1901 struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1902 struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1903 struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1754}; 1904};
1755 1905
1756void radeon_atombios_get_power_modes(struct radeon_device *rdev) 1906union pplib_clock_info {
1907 struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1908 struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1909 struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1910 struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1911};
1912
1913union pplib_power_state {
1914 struct _ATOM_PPLIB_STATE v1;
1915 struct _ATOM_PPLIB_STATE_V2 v2;
1916};
1917
1918static void radeon_atombios_parse_misc_flags_1_3(struct radeon_device *rdev,
1919 int state_index,
1920 u32 misc, u32 misc2)
1921{
1922 rdev->pm.power_state[state_index].misc = misc;
1923 rdev->pm.power_state[state_index].misc2 = misc2;
1924 /* order matters! */
1925 if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
1926 rdev->pm.power_state[state_index].type =
1927 POWER_STATE_TYPE_POWERSAVE;
1928 if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
1929 rdev->pm.power_state[state_index].type =
1930 POWER_STATE_TYPE_BATTERY;
1931 if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
1932 rdev->pm.power_state[state_index].type =
1933 POWER_STATE_TYPE_BATTERY;
1934 if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
1935 rdev->pm.power_state[state_index].type =
1936 POWER_STATE_TYPE_BALANCED;
1937 if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
1938 rdev->pm.power_state[state_index].type =
1939 POWER_STATE_TYPE_PERFORMANCE;
1940 rdev->pm.power_state[state_index].flags &=
1941 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1942 }
1943 if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
1944 rdev->pm.power_state[state_index].type =
1945 POWER_STATE_TYPE_BALANCED;
1946 if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
1947 rdev->pm.power_state[state_index].type =
1948 POWER_STATE_TYPE_DEFAULT;
1949 rdev->pm.default_power_state_index = state_index;
1950 rdev->pm.power_state[state_index].default_clock_mode =
1951 &rdev->pm.power_state[state_index].clock_info[0];
1952 } else if (state_index == 0) {
1953 rdev->pm.power_state[state_index].clock_info[0].flags |=
1954 RADEON_PM_MODE_NO_DISPLAY;
1955 }
1956}
1957
1958static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev)
1757{ 1959{
1758 struct radeon_mode_info *mode_info = &rdev->mode_info; 1960 struct radeon_mode_info *mode_info = &rdev->mode_info;
1961 u32 misc, misc2 = 0;
1962 int num_modes = 0, i;
1963 int state_index = 0;
1964 struct radeon_i2c_bus_rec i2c_bus;
1965 union power_info *power_info;
1759 int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo); 1966 int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1760 u16 data_offset; 1967 u16 data_offset;
1761 u8 frev, crev; 1968 u8 frev, crev;
1762 u32 misc, misc2 = 0, sclk, mclk;
1763 union power_info *power_info;
1764 struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1765 struct _ATOM_PPLIB_STATE *power_state;
1766 int num_modes = 0, i, j;
1767 int state_index = 0, mode_index = 0;
1768 struct radeon_i2c_bus_rec i2c_bus;
1769
1770 rdev->pm.default_power_state_index = -1;
1771 1969
1772 if (atom_parse_data_header(mode_info->atom_context, index, NULL, 1970 if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1773 &frev, &crev, &data_offset)) { 1971 &frev, &crev, &data_offset))
1774 power_info = (union power_info *)(mode_info->atom_context->bios + data_offset); 1972 return state_index;
1775 if (frev < 4) { 1973 power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1776 /* add the i2c bus for thermal/fan chip */ 1974
1777 if (power_info->info.ucOverdriveThermalController > 0) { 1975 /* add the i2c bus for thermal/fan chip */
1778 DRM_INFO("Possible %s thermal controller at 0x%02x\n", 1976 if (power_info->info.ucOverdriveThermalController > 0) {
1779 thermal_controller_names[power_info->info.ucOverdriveThermalController], 1977 DRM_INFO("Possible %s thermal controller at 0x%02x\n",
1780 power_info->info.ucOverdriveControllerAddress >> 1); 1978 thermal_controller_names[power_info->info.ucOverdriveThermalController],
1781 i2c_bus = radeon_lookup_i2c_gpio(rdev, power_info->info.ucOverdriveI2cLine); 1979 power_info->info.ucOverdriveControllerAddress >> 1);
1782 rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus); 1980 i2c_bus = radeon_lookup_i2c_gpio(rdev, power_info->info.ucOverdriveI2cLine);
1783 if (rdev->pm.i2c_bus) { 1981 rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
1784 struct i2c_board_info info = { }; 1982 if (rdev->pm.i2c_bus) {
1785 const char *name = thermal_controller_names[power_info->info. 1983 struct i2c_board_info info = { };
1786 ucOverdriveThermalController]; 1984 const char *name = thermal_controller_names[power_info->info.
1787 info.addr = power_info->info.ucOverdriveControllerAddress >> 1; 1985 ucOverdriveThermalController];
1788 strlcpy(info.type, name, sizeof(info.type)); 1986 info.addr = power_info->info.ucOverdriveControllerAddress >> 1;
1789 i2c_new_device(&rdev->pm.i2c_bus->adapter, &info); 1987 strlcpy(info.type, name, sizeof(info.type));
1790 } 1988 i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
1989 }
1990 }
1991 num_modes = power_info->info.ucNumOfPowerModeEntries;
1992 if (num_modes > ATOM_MAX_NUMBEROF_POWER_BLOCK)
1993 num_modes = ATOM_MAX_NUMBEROF_POWER_BLOCK;
1994 /* last mode is usually default, array is low to high */
1995 for (i = 0; i < num_modes; i++) {
1996 rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE;
1997 switch (frev) {
1998 case 1:
1999 rdev->pm.power_state[state_index].num_clock_modes = 1;
2000 rdev->pm.power_state[state_index].clock_info[0].mclk =
2001 le16_to_cpu(power_info->info.asPowerPlayInfo[i].usMemoryClock);
2002 rdev->pm.power_state[state_index].clock_info[0].sclk =
2003 le16_to_cpu(power_info->info.asPowerPlayInfo[i].usEngineClock);
2004 /* skip invalid modes */
2005 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
2006 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
2007 continue;
2008 rdev->pm.power_state[state_index].pcie_lanes =
2009 power_info->info.asPowerPlayInfo[i].ucNumPciELanes;
2010 misc = le32_to_cpu(power_info->info.asPowerPlayInfo[i].ulMiscInfo);
2011 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
2012 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
2013 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
2014 VOLTAGE_GPIO;
2015 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
2016 radeon_lookup_gpio(rdev,
2017 power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex);
2018 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
2019 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
2020 true;
2021 else
2022 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
2023 false;
2024 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
2025 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
2026 VOLTAGE_VDDC;
2027 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
2028 power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex;
1791 } 2029 }
1792 num_modes = power_info->info.ucNumOfPowerModeEntries; 2030 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1793 if (num_modes > ATOM_MAX_NUMBEROF_POWER_BLOCK) 2031 radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, 0);
1794 num_modes = ATOM_MAX_NUMBEROF_POWER_BLOCK; 2032 state_index++;
1795 /* last mode is usually default, array is low to high */ 2033 break;
1796 for (i = 0; i < num_modes; i++) { 2034 case 2:
1797 rdev->pm.power_state[state_index].clock_info[0].voltage.type = VOLTAGE_NONE; 2035 rdev->pm.power_state[state_index].num_clock_modes = 1;
1798 switch (frev) { 2036 rdev->pm.power_state[state_index].clock_info[0].mclk =
1799 case 1: 2037 le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMemoryClock);
1800 rdev->pm.power_state[state_index].num_clock_modes = 1; 2038 rdev->pm.power_state[state_index].clock_info[0].sclk =
1801 rdev->pm.power_state[state_index].clock_info[0].mclk = 2039 le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulEngineClock);
1802 le16_to_cpu(power_info->info.asPowerPlayInfo[i].usMemoryClock); 2040 /* skip invalid modes */
1803 rdev->pm.power_state[state_index].clock_info[0].sclk = 2041 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
1804 le16_to_cpu(power_info->info.asPowerPlayInfo[i].usEngineClock); 2042 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
1805 /* skip invalid modes */ 2043 continue;
1806 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) || 2044 rdev->pm.power_state[state_index].pcie_lanes =
1807 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0)) 2045 power_info->info_2.asPowerPlayInfo[i].ucNumPciELanes;
1808 continue; 2046 misc = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo);
1809 rdev->pm.power_state[state_index].pcie_lanes = 2047 misc2 = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo2);
1810 power_info->info.asPowerPlayInfo[i].ucNumPciELanes; 2048 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
1811 misc = le32_to_cpu(power_info->info.asPowerPlayInfo[i].ulMiscInfo); 2049 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
1812 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) || 2050 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1813 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) { 2051 VOLTAGE_GPIO;
1814 rdev->pm.power_state[state_index].clock_info[0].voltage.type = 2052 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
1815 VOLTAGE_GPIO; 2053 radeon_lookup_gpio(rdev,
1816 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio = 2054 power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex);
1817 radeon_lookup_gpio(rdev, 2055 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
1818 power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex); 2056 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1819 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH) 2057 true;
1820 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high = 2058 else
1821 true; 2059 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1822 else 2060 false;
1823 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high = 2061 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
1824 false; 2062 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1825 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) { 2063 VOLTAGE_VDDC;
1826 rdev->pm.power_state[state_index].clock_info[0].voltage.type = 2064 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
1827 VOLTAGE_VDDC; 2065 power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex;
1828 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
1829 power_info->info.asPowerPlayInfo[i].ucVoltageDropIndex;
1830 }
1831 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1832 rdev->pm.power_state[state_index].misc = misc;
1833 /* order matters! */
1834 if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
1835 rdev->pm.power_state[state_index].type =
1836 POWER_STATE_TYPE_POWERSAVE;
1837 if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
1838 rdev->pm.power_state[state_index].type =
1839 POWER_STATE_TYPE_BATTERY;
1840 if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
1841 rdev->pm.power_state[state_index].type =
1842 POWER_STATE_TYPE_BATTERY;
1843 if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
1844 rdev->pm.power_state[state_index].type =
1845 POWER_STATE_TYPE_BALANCED;
1846 if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
1847 rdev->pm.power_state[state_index].type =
1848 POWER_STATE_TYPE_PERFORMANCE;
1849 rdev->pm.power_state[state_index].flags &=
1850 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1851 }
1852 if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
1853 rdev->pm.power_state[state_index].type =
1854 POWER_STATE_TYPE_DEFAULT;
1855 rdev->pm.default_power_state_index = state_index;
1856 rdev->pm.power_state[state_index].default_clock_mode =
1857 &rdev->pm.power_state[state_index].clock_info[0];
1858 rdev->pm.power_state[state_index].flags &=
1859 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1860 } else if (state_index == 0) {
1861 rdev->pm.power_state[state_index].clock_info[0].flags |=
1862 RADEON_PM_MODE_NO_DISPLAY;
1863 }
1864 state_index++;
1865 break;
1866 case 2:
1867 rdev->pm.power_state[state_index].num_clock_modes = 1;
1868 rdev->pm.power_state[state_index].clock_info[0].mclk =
1869 le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMemoryClock);
1870 rdev->pm.power_state[state_index].clock_info[0].sclk =
1871 le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulEngineClock);
1872 /* skip invalid modes */
1873 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
1874 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
1875 continue;
1876 rdev->pm.power_state[state_index].pcie_lanes =
1877 power_info->info_2.asPowerPlayInfo[i].ucNumPciELanes;
1878 misc = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo);
1879 misc2 = le32_to_cpu(power_info->info_2.asPowerPlayInfo[i].ulMiscInfo2);
1880 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
1881 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
1882 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1883 VOLTAGE_GPIO;
1884 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
1885 radeon_lookup_gpio(rdev,
1886 power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex);
1887 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
1888 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1889 true;
1890 else
1891 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1892 false;
1893 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
1894 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1895 VOLTAGE_VDDC;
1896 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
1897 power_info->info_2.asPowerPlayInfo[i].ucVoltageDropIndex;
1898 }
1899 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1900 rdev->pm.power_state[state_index].misc = misc;
1901 rdev->pm.power_state[state_index].misc2 = misc2;
1902 /* order matters! */
1903 if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
1904 rdev->pm.power_state[state_index].type =
1905 POWER_STATE_TYPE_POWERSAVE;
1906 if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
1907 rdev->pm.power_state[state_index].type =
1908 POWER_STATE_TYPE_BATTERY;
1909 if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
1910 rdev->pm.power_state[state_index].type =
1911 POWER_STATE_TYPE_BATTERY;
1912 if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
1913 rdev->pm.power_state[state_index].type =
1914 POWER_STATE_TYPE_BALANCED;
1915 if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
1916 rdev->pm.power_state[state_index].type =
1917 POWER_STATE_TYPE_PERFORMANCE;
1918 rdev->pm.power_state[state_index].flags &=
1919 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1920 }
1921 if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
1922 rdev->pm.power_state[state_index].type =
1923 POWER_STATE_TYPE_BALANCED;
1924 if (misc2 & ATOM_PM_MISCINFO2_MULTI_DISPLAY_SUPPORT)
1925 rdev->pm.power_state[state_index].flags &=
1926 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1927 if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
1928 rdev->pm.power_state[state_index].type =
1929 POWER_STATE_TYPE_DEFAULT;
1930 rdev->pm.default_power_state_index = state_index;
1931 rdev->pm.power_state[state_index].default_clock_mode =
1932 &rdev->pm.power_state[state_index].clock_info[0];
1933 rdev->pm.power_state[state_index].flags &=
1934 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1935 } else if (state_index == 0) {
1936 rdev->pm.power_state[state_index].clock_info[0].flags |=
1937 RADEON_PM_MODE_NO_DISPLAY;
1938 }
1939 state_index++;
1940 break;
1941 case 3:
1942 rdev->pm.power_state[state_index].num_clock_modes = 1;
1943 rdev->pm.power_state[state_index].clock_info[0].mclk =
1944 le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMemoryClock);
1945 rdev->pm.power_state[state_index].clock_info[0].sclk =
1946 le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulEngineClock);
1947 /* skip invalid modes */
1948 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
1949 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
1950 continue;
1951 rdev->pm.power_state[state_index].pcie_lanes =
1952 power_info->info_3.asPowerPlayInfo[i].ucNumPciELanes;
1953 misc = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo);
1954 misc2 = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo2);
1955 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
1956 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
1957 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1958 VOLTAGE_GPIO;
1959 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
1960 radeon_lookup_gpio(rdev,
1961 power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex);
1962 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
1963 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1964 true;
1965 else
1966 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
1967 false;
1968 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
1969 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
1970 VOLTAGE_VDDC;
1971 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
1972 power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex;
1973 if (misc2 & ATOM_PM_MISCINFO2_VDDCI_DYNAMIC_VOLTAGE_EN) {
1974 rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_enabled =
1975 true;
1976 rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_id =
1977 power_info->info_3.asPowerPlayInfo[i].ucVDDCI_VoltageDropIndex;
1978 }
1979 }
1980 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
1981 rdev->pm.power_state[state_index].misc = misc;
1982 rdev->pm.power_state[state_index].misc2 = misc2;
1983 /* order matters! */
1984 if (misc & ATOM_PM_MISCINFO_POWER_SAVING_MODE)
1985 rdev->pm.power_state[state_index].type =
1986 POWER_STATE_TYPE_POWERSAVE;
1987 if (misc & ATOM_PM_MISCINFO_DEFAULT_DC_STATE_ENTRY_TRUE)
1988 rdev->pm.power_state[state_index].type =
1989 POWER_STATE_TYPE_BATTERY;
1990 if (misc & ATOM_PM_MISCINFO_DEFAULT_LOW_DC_STATE_ENTRY_TRUE)
1991 rdev->pm.power_state[state_index].type =
1992 POWER_STATE_TYPE_BATTERY;
1993 if (misc & ATOM_PM_MISCINFO_LOAD_BALANCE_EN)
1994 rdev->pm.power_state[state_index].type =
1995 POWER_STATE_TYPE_BALANCED;
1996 if (misc & ATOM_PM_MISCINFO_3D_ACCELERATION_EN) {
1997 rdev->pm.power_state[state_index].type =
1998 POWER_STATE_TYPE_PERFORMANCE;
1999 rdev->pm.power_state[state_index].flags &=
2000 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2001 }
2002 if (misc2 & ATOM_PM_MISCINFO2_SYSTEM_AC_LITE_MODE)
2003 rdev->pm.power_state[state_index].type =
2004 POWER_STATE_TYPE_BALANCED;
2005 if (misc & ATOM_PM_MISCINFO_DRIVER_DEFAULT_MODE) {
2006 rdev->pm.power_state[state_index].type =
2007 POWER_STATE_TYPE_DEFAULT;
2008 rdev->pm.default_power_state_index = state_index;
2009 rdev->pm.power_state[state_index].default_clock_mode =
2010 &rdev->pm.power_state[state_index].clock_info[0];
2011 } else if (state_index == 0) {
2012 rdev->pm.power_state[state_index].clock_info[0].flags |=
2013 RADEON_PM_MODE_NO_DISPLAY;
2014 }
2015 state_index++;
2016 break;
2017 }
2018 } 2066 }
2019 /* last mode is usually default */ 2067 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2020 if (rdev->pm.default_power_state_index == -1) { 2068 radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, misc2);
2021 rdev->pm.power_state[state_index - 1].type = 2069 state_index++;
2022 POWER_STATE_TYPE_DEFAULT; 2070 break;
2023 rdev->pm.default_power_state_index = state_index - 1; 2071 case 3:
2024 rdev->pm.power_state[state_index - 1].default_clock_mode = 2072 rdev->pm.power_state[state_index].num_clock_modes = 1;
2025 &rdev->pm.power_state[state_index - 1].clock_info[0]; 2073 rdev->pm.power_state[state_index].clock_info[0].mclk =
2026 rdev->pm.power_state[state_index].flags &= 2074 le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMemoryClock);
2027 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY; 2075 rdev->pm.power_state[state_index].clock_info[0].sclk =
2028 rdev->pm.power_state[state_index].misc = 0; 2076 le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulEngineClock);
2029 rdev->pm.power_state[state_index].misc2 = 0; 2077 /* skip invalid modes */
2078 if ((rdev->pm.power_state[state_index].clock_info[0].mclk == 0) ||
2079 (rdev->pm.power_state[state_index].clock_info[0].sclk == 0))
2080 continue;
2081 rdev->pm.power_state[state_index].pcie_lanes =
2082 power_info->info_3.asPowerPlayInfo[i].ucNumPciELanes;
2083 misc = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo);
2084 misc2 = le32_to_cpu(power_info->info_3.asPowerPlayInfo[i].ulMiscInfo2);
2085 if ((misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_SUPPORT) ||
2086 (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)) {
2087 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
2088 VOLTAGE_GPIO;
2089 rdev->pm.power_state[state_index].clock_info[0].voltage.gpio =
2090 radeon_lookup_gpio(rdev,
2091 power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex);
2092 if (misc & ATOM_PM_MISCINFO_VOLTAGE_DROP_ACTIVE_HIGH)
2093 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
2094 true;
2095 else
2096 rdev->pm.power_state[state_index].clock_info[0].voltage.active_high =
2097 false;
2098 } else if (misc & ATOM_PM_MISCINFO_PROGRAM_VOLTAGE) {
2099 rdev->pm.power_state[state_index].clock_info[0].voltage.type =
2100 VOLTAGE_VDDC;
2101 rdev->pm.power_state[state_index].clock_info[0].voltage.vddc_id =
2102 power_info->info_3.asPowerPlayInfo[i].ucVoltageDropIndex;
2103 if (misc2 & ATOM_PM_MISCINFO2_VDDCI_DYNAMIC_VOLTAGE_EN) {
2104 rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_enabled =
2105 true;
2106 rdev->pm.power_state[state_index].clock_info[0].voltage.vddci_id =
2107 power_info->info_3.asPowerPlayInfo[i].ucVDDCI_VoltageDropIndex;
2108 }
2030 } 2109 }
2110 rdev->pm.power_state[state_index].flags = RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2111 radeon_atombios_parse_misc_flags_1_3(rdev, state_index, misc, misc2);
2112 state_index++;
2113 break;
2114 }
2115 }
2116 /* last mode is usually default */
2117 if (rdev->pm.default_power_state_index == -1) {
2118 rdev->pm.power_state[state_index - 1].type =
2119 POWER_STATE_TYPE_DEFAULT;
2120 rdev->pm.default_power_state_index = state_index - 1;
2121 rdev->pm.power_state[state_index - 1].default_clock_mode =
2122 &rdev->pm.power_state[state_index - 1].clock_info[0];
2123 rdev->pm.power_state[state_index].flags &=
2124 ~RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2125 rdev->pm.power_state[state_index].misc = 0;
2126 rdev->pm.power_state[state_index].misc2 = 0;
2127 }
2128 return state_index;
2129}
2130
2131static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *rdev,
2132 ATOM_PPLIB_THERMALCONTROLLER *controller)
2133{
2134 struct radeon_i2c_bus_rec i2c_bus;
2135
2136 /* add the i2c bus for thermal/fan chip */
2137 if (controller->ucType > 0) {
2138 if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) {
2139 DRM_INFO("Internal thermal controller %s fan control\n",
2140 (controller->ucFanParameters &
2141 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2142 rdev->pm.int_thermal_type = THERMAL_TYPE_RV6XX;
2143 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV770) {
2144 DRM_INFO("Internal thermal controller %s fan control\n",
2145 (controller->ucFanParameters &
2146 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2147 rdev->pm.int_thermal_type = THERMAL_TYPE_RV770;
2148 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EVERGREEN) {
2149 DRM_INFO("Internal thermal controller %s fan control\n",
2150 (controller->ucFanParameters &
2151 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2152 rdev->pm.int_thermal_type = THERMAL_TYPE_EVERGREEN;
2153 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_SUMO) {
2154 DRM_INFO("Internal thermal controller %s fan control\n",
2155 (controller->ucFanParameters &
2156 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2157 rdev->pm.int_thermal_type = THERMAL_TYPE_SUMO;
2158 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_NISLANDS) {
2159 DRM_INFO("Internal thermal controller %s fan control\n",
2160 (controller->ucFanParameters &
2161 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2162 rdev->pm.int_thermal_type = THERMAL_TYPE_NI;
2163 } else if ((controller->ucType ==
2164 ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
2165 (controller->ucType ==
2166 ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL) ||
2167 (controller->ucType ==
2168 ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL)) {
2169 DRM_INFO("Special thermal controller config\n");
2031 } else { 2170 } else {
2032 int fw_index = GetIndexIntoMasterTable(DATA, FirmwareInfo); 2171 DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n",
2033 uint8_t fw_frev, fw_crev; 2172 pp_lib_thermal_controller_names[controller->ucType],
2034 uint16_t fw_data_offset, vddc = 0; 2173 controller->ucI2cAddress >> 1,
2035 union firmware_info *firmware_info; 2174 (controller->ucFanParameters &
2036 ATOM_PPLIB_THERMALCONTROLLER *controller = &power_info->info_4.sThermalController; 2175 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2037 2176 i2c_bus = radeon_lookup_i2c_gpio(rdev, controller->ucI2cLine);
2038 if (atom_parse_data_header(mode_info->atom_context, fw_index, NULL, 2177 rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
2039 &fw_frev, &fw_crev, &fw_data_offset)) { 2178 if (rdev->pm.i2c_bus) {
2040 firmware_info = 2179 struct i2c_board_info info = { };
2041 (union firmware_info *)(mode_info->atom_context->bios + 2180 const char *name = pp_lib_thermal_controller_names[controller->ucType];
2042 fw_data_offset); 2181 info.addr = controller->ucI2cAddress >> 1;
2043 vddc = firmware_info->info_14.usBootUpVDDCVoltage; 2182 strlcpy(info.type, name, sizeof(info.type));
2183 i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
2044 } 2184 }
2185 }
2186 }
2187}
2045 2188
2046 /* add the i2c bus for thermal/fan chip */ 2189static u16 radeon_atombios_get_default_vddc(struct radeon_device *rdev)
2047 if (controller->ucType > 0) { 2190{
2048 if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV6xx) { 2191 struct radeon_mode_info *mode_info = &rdev->mode_info;
2049 DRM_INFO("Internal thermal controller %s fan control\n", 2192 int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
2050 (controller->ucFanParameters & 2193 u8 frev, crev;
2051 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with"); 2194 u16 data_offset;
2052 rdev->pm.int_thermal_type = THERMAL_TYPE_RV6XX; 2195 union firmware_info *firmware_info;
2053 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_RV770) { 2196 u16 vddc = 0;
2054 DRM_INFO("Internal thermal controller %s fan control\n",
2055 (controller->ucFanParameters &
2056 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2057 rdev->pm.int_thermal_type = THERMAL_TYPE_RV770;
2058 } else if (controller->ucType == ATOM_PP_THERMALCONTROLLER_EVERGREEN) {
2059 DRM_INFO("Internal thermal controller %s fan control\n",
2060 (controller->ucFanParameters &
2061 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2062 rdev->pm.int_thermal_type = THERMAL_TYPE_EVERGREEN;
2063 } else if ((controller->ucType ==
2064 ATOM_PP_THERMALCONTROLLER_EXTERNAL_GPIO) ||
2065 (controller->ucType ==
2066 ATOM_PP_THERMALCONTROLLER_ADT7473_WITH_INTERNAL)) {
2067 DRM_INFO("Special thermal controller config\n");
2068 } else {
2069 DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n",
2070 pp_lib_thermal_controller_names[controller->ucType],
2071 controller->ucI2cAddress >> 1,
2072 (controller->ucFanParameters &
2073 ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
2074 i2c_bus = radeon_lookup_i2c_gpio(rdev, controller->ucI2cLine);
2075 rdev->pm.i2c_bus = radeon_i2c_lookup(rdev, &i2c_bus);
2076 if (rdev->pm.i2c_bus) {
2077 struct i2c_board_info info = { };
2078 const char *name = pp_lib_thermal_controller_names[controller->ucType];
2079 info.addr = controller->ucI2cAddress >> 1;
2080 strlcpy(info.type, name, sizeof(info.type));
2081 i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
2082 }
2083 2197
2084 } 2198 if (atom_parse_data_header(mode_info->atom_context, index, NULL,
2085 } 2199 &frev, &crev, &data_offset)) {
2086 /* first mode is usually default, followed by low to high */ 2200 firmware_info =
2087 for (i = 0; i < power_info->info_4.ucNumStates; i++) { 2201 (union firmware_info *)(mode_info->atom_context->bios +
2088 mode_index = 0; 2202 data_offset);
2089 power_state = (struct _ATOM_PPLIB_STATE *) 2203 vddc = firmware_info->info_14.usBootUpVDDCVoltage;
2090 (mode_info->atom_context->bios + 2204 }
2091 data_offset + 2205
2092 le16_to_cpu(power_info->info_4.usStateArrayOffset) + 2206 return vddc;
2093 i * power_info->info_4.ucStateEntrySize); 2207}
2094 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *) 2208
2095 (mode_info->atom_context->bios + 2209static void radeon_atombios_parse_pplib_non_clock_info(struct radeon_device *rdev,
2096 data_offset + 2210 int state_index, int mode_index,
2097 le16_to_cpu(power_info->info_4.usNonClockInfoArrayOffset) + 2211 struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info)
2098 (power_state->ucNonClockStateIndex * 2212{
2099 power_info->info_4.ucNonClockSize)); 2213 int j;
2100 for (j = 0; j < (power_info->info_4.ucStateEntrySize - 1); j++) { 2214 u32 misc = le32_to_cpu(non_clock_info->ulCapsAndSettings);
2101 if (rdev->flags & RADEON_IS_IGP) { 2215 u32 misc2 = le16_to_cpu(non_clock_info->usClassification);
2102 struct _ATOM_PPLIB_RS780_CLOCK_INFO *clock_info = 2216 u16 vddc = radeon_atombios_get_default_vddc(rdev);
2103 (struct _ATOM_PPLIB_RS780_CLOCK_INFO *) 2217
2104 (mode_info->atom_context->bios + 2218 rdev->pm.power_state[state_index].misc = misc;
2105 data_offset + 2219 rdev->pm.power_state[state_index].misc2 = misc2;
2106 le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) + 2220 rdev->pm.power_state[state_index].pcie_lanes =
2107 (power_state->ucClockStateIndices[j] * 2221 ((misc & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >>
2108 power_info->info_4.ucClockInfoSize)); 2222 ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1;
2109 sclk = le16_to_cpu(clock_info->usLowEngineClockLow); 2223 switch (misc2 & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
2110 sclk |= clock_info->ucLowEngineClockHigh << 16; 2224 case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
2111 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk; 2225 rdev->pm.power_state[state_index].type =
2112 /* skip invalid modes */ 2226 POWER_STATE_TYPE_BATTERY;
2113 if (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0) 2227 break;
2114 continue; 2228 case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED:
2115 /* voltage works differently on IGPs */ 2229 rdev->pm.power_state[state_index].type =
2116 mode_index++; 2230 POWER_STATE_TYPE_BALANCED;
2117 } else if (ASIC_IS_DCE4(rdev)) { 2231 break;
2118 struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO *clock_info = 2232 case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
2119 (struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO *) 2233 rdev->pm.power_state[state_index].type =
2120 (mode_info->atom_context->bios + 2234 POWER_STATE_TYPE_PERFORMANCE;
2121 data_offset + 2235 break;
2122 le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) + 2236 case ATOM_PPLIB_CLASSIFICATION_UI_NONE:
2123 (power_state->ucClockStateIndices[j] * 2237 if (misc2 & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
2124 power_info->info_4.ucClockInfoSize)); 2238 rdev->pm.power_state[state_index].type =
2125 sclk = le16_to_cpu(clock_info->usEngineClockLow); 2239 POWER_STATE_TYPE_PERFORMANCE;
2126 sclk |= clock_info->ucEngineClockHigh << 16; 2240 break;
2127 mclk = le16_to_cpu(clock_info->usMemoryClockLow); 2241 }
2128 mclk |= clock_info->ucMemoryClockHigh << 16; 2242 rdev->pm.power_state[state_index].flags = 0;
2129 rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk; 2243 if (misc & ATOM_PPLIB_SINGLE_DISPLAY_ONLY)
2130 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk; 2244 rdev->pm.power_state[state_index].flags |=
2131 /* skip invalid modes */ 2245 RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2132 if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) || 2246 if (misc2 & ATOM_PPLIB_CLASSIFICATION_BOOT) {
2133 (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0)) 2247 rdev->pm.power_state[state_index].type =
2134 continue; 2248 POWER_STATE_TYPE_DEFAULT;
2135 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type = 2249 rdev->pm.default_power_state_index = state_index;
2136 VOLTAGE_SW; 2250 rdev->pm.power_state[state_index].default_clock_mode =
2137 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage = 2251 &rdev->pm.power_state[state_index].clock_info[mode_index - 1];
2138 clock_info->usVDDC; 2252 if (ASIC_IS_DCE5(rdev)) {
2139 /* XXX usVDDCI */ 2253 /* NI chips post without MC ucode, so default clocks are strobe mode only */
2140 mode_index++; 2254 rdev->pm.default_sclk = rdev->pm.power_state[state_index].clock_info[0].sclk;
2141 } else { 2255 rdev->pm.default_mclk = rdev->pm.power_state[state_index].clock_info[0].mclk;
2142 struct _ATOM_PPLIB_R600_CLOCK_INFO *clock_info = 2256 rdev->pm.default_vddc = rdev->pm.power_state[state_index].clock_info[0].voltage.voltage;
2143 (struct _ATOM_PPLIB_R600_CLOCK_INFO *) 2257 } else {
2144 (mode_info->atom_context->bios + 2258 /* patch the table values with the default slck/mclk from firmware info */
2145 data_offset + 2259 for (j = 0; j < mode_index; j++) {
2146 le16_to_cpu(power_info->info_4.usClockInfoArrayOffset) + 2260 rdev->pm.power_state[state_index].clock_info[j].mclk =
2147 (power_state->ucClockStateIndices[j] * 2261 rdev->clock.default_mclk;
2148 power_info->info_4.ucClockInfoSize)); 2262 rdev->pm.power_state[state_index].clock_info[j].sclk =
2149 sclk = le16_to_cpu(clock_info->usEngineClockLow); 2263 rdev->clock.default_sclk;
2150 sclk |= clock_info->ucEngineClockHigh << 16; 2264 if (vddc)
2151 mclk = le16_to_cpu(clock_info->usMemoryClockLow); 2265 rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
2152 mclk |= clock_info->ucMemoryClockHigh << 16; 2266 vddc;
2153 rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
2154 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
2155 /* skip invalid modes */
2156 if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) ||
2157 (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0))
2158 continue;
2159 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
2160 VOLTAGE_SW;
2161 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
2162 clock_info->usVDDC;
2163 mode_index++;
2164 }
2165 }
2166 rdev->pm.power_state[state_index].num_clock_modes = mode_index;
2167 if (mode_index) {
2168 misc = le32_to_cpu(non_clock_info->ulCapsAndSettings);
2169 misc2 = le16_to_cpu(non_clock_info->usClassification);
2170 rdev->pm.power_state[state_index].misc = misc;
2171 rdev->pm.power_state[state_index].misc2 = misc2;
2172 rdev->pm.power_state[state_index].pcie_lanes =
2173 ((misc & ATOM_PPLIB_PCIE_LINK_WIDTH_MASK) >>
2174 ATOM_PPLIB_PCIE_LINK_WIDTH_SHIFT) + 1;
2175 switch (misc2 & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
2176 case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
2177 rdev->pm.power_state[state_index].type =
2178 POWER_STATE_TYPE_BATTERY;
2179 break;
2180 case ATOM_PPLIB_CLASSIFICATION_UI_BALANCED:
2181 rdev->pm.power_state[state_index].type =
2182 POWER_STATE_TYPE_BALANCED;
2183 break;
2184 case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
2185 rdev->pm.power_state[state_index].type =
2186 POWER_STATE_TYPE_PERFORMANCE;
2187 break;
2188 case ATOM_PPLIB_CLASSIFICATION_UI_NONE:
2189 if (misc2 & ATOM_PPLIB_CLASSIFICATION_3DPERFORMANCE)
2190 rdev->pm.power_state[state_index].type =
2191 POWER_STATE_TYPE_PERFORMANCE;
2192 break;
2193 }
2194 rdev->pm.power_state[state_index].flags = 0;
2195 if (misc & ATOM_PPLIB_SINGLE_DISPLAY_ONLY)
2196 rdev->pm.power_state[state_index].flags |=
2197 RADEON_PM_STATE_SINGLE_DISPLAY_ONLY;
2198 if (misc2 & ATOM_PPLIB_CLASSIFICATION_BOOT) {
2199 rdev->pm.power_state[state_index].type =
2200 POWER_STATE_TYPE_DEFAULT;
2201 rdev->pm.default_power_state_index = state_index;
2202 rdev->pm.power_state[state_index].default_clock_mode =
2203 &rdev->pm.power_state[state_index].clock_info[mode_index - 1];
2204 /* patch the table values with the default slck/mclk from firmware info */
2205 for (j = 0; j < mode_index; j++) {
2206 rdev->pm.power_state[state_index].clock_info[j].mclk =
2207 rdev->clock.default_mclk;
2208 rdev->pm.power_state[state_index].clock_info[j].sclk =
2209 rdev->clock.default_sclk;
2210 if (vddc)
2211 rdev->pm.power_state[state_index].clock_info[j].voltage.voltage =
2212 vddc;
2213 }
2214 }
2215 state_index++;
2216 }
2217 }
2218 /* if multiple clock modes, mark the lowest as no display */
2219 for (i = 0; i < state_index; i++) {
2220 if (rdev->pm.power_state[i].num_clock_modes > 1)
2221 rdev->pm.power_state[i].clock_info[0].flags |=
2222 RADEON_PM_MODE_NO_DISPLAY;
2223 }
2224 /* first mode is usually default */
2225 if (rdev->pm.default_power_state_index == -1) {
2226 rdev->pm.power_state[0].type =
2227 POWER_STATE_TYPE_DEFAULT;
2228 rdev->pm.default_power_state_index = 0;
2229 rdev->pm.power_state[0].default_clock_mode =
2230 &rdev->pm.power_state[0].clock_info[0];
2231 } 2267 }
2232 } 2268 }
2269 }
2270}
2271
2272static bool radeon_atombios_parse_pplib_clock_info(struct radeon_device *rdev,
2273 int state_index, int mode_index,
2274 union pplib_clock_info *clock_info)
2275{
2276 u32 sclk, mclk;
2277
2278 if (rdev->flags & RADEON_IS_IGP) {
2279 if (rdev->family >= CHIP_PALM) {
2280 sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
2281 sclk |= clock_info->sumo.ucEngineClockHigh << 16;
2282 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
2283 } else {
2284 sclk = le16_to_cpu(clock_info->rs780.usLowEngineClockLow);
2285 sclk |= clock_info->rs780.ucLowEngineClockHigh << 16;
2286 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
2287 }
2288 } else if (ASIC_IS_DCE4(rdev)) {
2289 sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
2290 sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
2291 mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
2292 mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
2293 rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
2294 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
2295 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
2296 VOLTAGE_SW;
2297 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
2298 clock_info->evergreen.usVDDC;
2299 } else {
2300 sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
2301 sclk |= clock_info->r600.ucEngineClockHigh << 16;
2302 mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
2303 mclk |= clock_info->r600.ucMemoryClockHigh << 16;
2304 rdev->pm.power_state[state_index].clock_info[mode_index].mclk = mclk;
2305 rdev->pm.power_state[state_index].clock_info[mode_index].sclk = sclk;
2306 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.type =
2307 VOLTAGE_SW;
2308 rdev->pm.power_state[state_index].clock_info[mode_index].voltage.voltage =
2309 clock_info->r600.usVDDC;
2310 }
2311
2312 if (rdev->flags & RADEON_IS_IGP) {
2313 /* skip invalid modes */
2314 if (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0)
2315 return false;
2316 } else {
2317 /* skip invalid modes */
2318 if ((rdev->pm.power_state[state_index].clock_info[mode_index].mclk == 0) ||
2319 (rdev->pm.power_state[state_index].clock_info[mode_index].sclk == 0))
2320 return false;
2321 }
2322 return true;
2323}
2324
2325static int radeon_atombios_parse_power_table_4_5(struct radeon_device *rdev)
2326{
2327 struct radeon_mode_info *mode_info = &rdev->mode_info;
2328 struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
2329 union pplib_power_state *power_state;
2330 int i, j;
2331 int state_index = 0, mode_index = 0;
2332 union pplib_clock_info *clock_info;
2333 bool valid;
2334 union power_info *power_info;
2335 int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2336 u16 data_offset;
2337 u8 frev, crev;
2338
2339 if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
2340 &frev, &crev, &data_offset))
2341 return state_index;
2342 power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
2343
2344 radeon_atombios_add_pplib_thermal_controller(rdev, &power_info->pplib.sThermalController);
2345 /* first mode is usually default, followed by low to high */
2346 for (i = 0; i < power_info->pplib.ucNumStates; i++) {
2347 mode_index = 0;
2348 power_state = (union pplib_power_state *)
2349 (mode_info->atom_context->bios + data_offset +
2350 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
2351 i * power_info->pplib.ucStateEntrySize);
2352 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
2353 (mode_info->atom_context->bios + data_offset +
2354 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
2355 (power_state->v1.ucNonClockStateIndex *
2356 power_info->pplib.ucNonClockSize));
2357 for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
2358 clock_info = (union pplib_clock_info *)
2359 (mode_info->atom_context->bios + data_offset +
2360 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
2361 (power_state->v1.ucClockStateIndices[j] *
2362 power_info->pplib.ucClockInfoSize));
2363 valid = radeon_atombios_parse_pplib_clock_info(rdev,
2364 state_index, mode_index,
2365 clock_info);
2366 if (valid)
2367 mode_index++;
2368 }
2369 rdev->pm.power_state[state_index].num_clock_modes = mode_index;
2370 if (mode_index) {
2371 radeon_atombios_parse_pplib_non_clock_info(rdev, state_index, mode_index,
2372 non_clock_info);
2373 state_index++;
2374 }
2375 }
2376 /* if multiple clock modes, mark the lowest as no display */
2377 for (i = 0; i < state_index; i++) {
2378 if (rdev->pm.power_state[i].num_clock_modes > 1)
2379 rdev->pm.power_state[i].clock_info[0].flags |=
2380 RADEON_PM_MODE_NO_DISPLAY;
2381 }
2382 /* first mode is usually default */
2383 if (rdev->pm.default_power_state_index == -1) {
2384 rdev->pm.power_state[0].type =
2385 POWER_STATE_TYPE_DEFAULT;
2386 rdev->pm.default_power_state_index = 0;
2387 rdev->pm.power_state[0].default_clock_mode =
2388 &rdev->pm.power_state[0].clock_info[0];
2389 }
2390 return state_index;
2391}
2392
2393static int radeon_atombios_parse_power_table_6(struct radeon_device *rdev)
2394{
2395 struct radeon_mode_info *mode_info = &rdev->mode_info;
2396 struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
2397 union pplib_power_state *power_state;
2398 int i, j, non_clock_array_index, clock_array_index;
2399 int state_index = 0, mode_index = 0;
2400 union pplib_clock_info *clock_info;
2401 struct StateArray *state_array;
2402 struct ClockInfoArray *clock_info_array;
2403 struct NonClockInfoArray *non_clock_info_array;
2404 bool valid;
2405 union power_info *power_info;
2406 int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2407 u16 data_offset;
2408 u8 frev, crev;
2409
2410 if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
2411 &frev, &crev, &data_offset))
2412 return state_index;
2413 power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
2414
2415 radeon_atombios_add_pplib_thermal_controller(rdev, &power_info->pplib.sThermalController);
2416 state_array = (struct StateArray *)
2417 (mode_info->atom_context->bios + data_offset +
2418 power_info->pplib.usStateArrayOffset);
2419 clock_info_array = (struct ClockInfoArray *)
2420 (mode_info->atom_context->bios + data_offset +
2421 power_info->pplib.usClockInfoArrayOffset);
2422 non_clock_info_array = (struct NonClockInfoArray *)
2423 (mode_info->atom_context->bios + data_offset +
2424 power_info->pplib.usNonClockInfoArrayOffset);
2425 for (i = 0; i < state_array->ucNumEntries; i++) {
2426 mode_index = 0;
2427 power_state = (union pplib_power_state *)&state_array->states[i];
2428 /* XXX this might be an inagua bug... */
2429 non_clock_array_index = i; /* power_state->v2.nonClockInfoIndex */
2430 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
2431 &non_clock_info_array->nonClockInfo[non_clock_array_index];
2432 for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
2433 clock_array_index = power_state->v2.clockInfoIndex[j];
2434 /* XXX this might be an inagua bug... */
2435 if (clock_array_index >= clock_info_array->ucNumEntries)
2436 continue;
2437 clock_info = (union pplib_clock_info *)
2438 &clock_info_array->clockInfo[clock_array_index];
2439 valid = radeon_atombios_parse_pplib_clock_info(rdev,
2440 state_index, mode_index,
2441 clock_info);
2442 if (valid)
2443 mode_index++;
2444 }
2445 rdev->pm.power_state[state_index].num_clock_modes = mode_index;
2446 if (mode_index) {
2447 radeon_atombios_parse_pplib_non_clock_info(rdev, state_index, mode_index,
2448 non_clock_info);
2449 state_index++;
2450 }
2451 }
2452 /* if multiple clock modes, mark the lowest as no display */
2453 for (i = 0; i < state_index; i++) {
2454 if (rdev->pm.power_state[i].num_clock_modes > 1)
2455 rdev->pm.power_state[i].clock_info[0].flags |=
2456 RADEON_PM_MODE_NO_DISPLAY;
2457 }
2458 /* first mode is usually default */
2459 if (rdev->pm.default_power_state_index == -1) {
2460 rdev->pm.power_state[0].type =
2461 POWER_STATE_TYPE_DEFAULT;
2462 rdev->pm.default_power_state_index = 0;
2463 rdev->pm.power_state[0].default_clock_mode =
2464 &rdev->pm.power_state[0].clock_info[0];
2465 }
2466 return state_index;
2467}
2468
2469void radeon_atombios_get_power_modes(struct radeon_device *rdev)
2470{
2471 struct radeon_mode_info *mode_info = &rdev->mode_info;
2472 int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2473 u16 data_offset;
2474 u8 frev, crev;
2475 int state_index = 0;
2476
2477 rdev->pm.default_power_state_index = -1;
2478
2479 if (atom_parse_data_header(mode_info->atom_context, index, NULL,
2480 &frev, &crev, &data_offset)) {
2481 switch (frev) {
2482 case 1:
2483 case 2:
2484 case 3:
2485 state_index = radeon_atombios_parse_power_table_1_3(rdev);
2486 break;
2487 case 4:
2488 case 5:
2489 state_index = radeon_atombios_parse_power_table_4_5(rdev);
2490 break;
2491 case 6:
2492 state_index = radeon_atombios_parse_power_table_6(rdev);
2493 break;
2494 default:
2495 break;
2496 }
2233 } else { 2497 } else {
2234 /* add the default mode */ 2498 /* add the default mode */
2235 rdev->pm.power_state[state_index].type = 2499 rdev->pm.power_state[state_index].type =
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index 8f2c7b50dcf5..1aba85cad1a8 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -131,6 +131,45 @@ static bool radeon_atrm_get_bios(struct radeon_device *rdev)
131 return true; 131 return true;
132} 132}
133 133
134static bool ni_read_disabled_bios(struct radeon_device *rdev)
135{
136 u32 bus_cntl;
137 u32 d1vga_control;
138 u32 d2vga_control;
139 u32 vga_render_control;
140 u32 rom_cntl;
141 bool r;
142
143 bus_cntl = RREG32(R600_BUS_CNTL);
144 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
145 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
146 vga_render_control = RREG32(AVIVO_VGA_RENDER_CONTROL);
147 rom_cntl = RREG32(R600_ROM_CNTL);
148
149 /* enable the rom */
150 WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
151 /* Disable VGA mode */
152 WREG32(AVIVO_D1VGA_CONTROL,
153 (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
154 AVIVO_DVGA_CONTROL_TIMING_SELECT)));
155 WREG32(AVIVO_D2VGA_CONTROL,
156 (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
157 AVIVO_DVGA_CONTROL_TIMING_SELECT)));
158 WREG32(AVIVO_VGA_RENDER_CONTROL,
159 (vga_render_control & ~AVIVO_VGA_VSTATUS_CNTL_MASK));
160 WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
161
162 r = radeon_read_bios(rdev);
163
164 /* restore regs */
165 WREG32(R600_BUS_CNTL, bus_cntl);
166 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
167 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
168 WREG32(AVIVO_VGA_RENDER_CONTROL, vga_render_control);
169 WREG32(R600_ROM_CNTL, rom_cntl);
170 return r;
171}
172
134static bool r700_read_disabled_bios(struct radeon_device *rdev) 173static bool r700_read_disabled_bios(struct radeon_device *rdev)
135{ 174{
136 uint32_t viph_control; 175 uint32_t viph_control;
@@ -416,6 +455,8 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev)
416{ 455{
417 if (rdev->flags & RADEON_IS_IGP) 456 if (rdev->flags & RADEON_IS_IGP)
418 return igp_read_bios_from_vram(rdev); 457 return igp_read_bios_from_vram(rdev);
458 else if (rdev->family >= CHIP_BARTS)
459 return ni_read_disabled_bios(rdev);
419 else if (rdev->family >= CHIP_RV770) 460 else if (rdev->family >= CHIP_RV770)
420 return r700_read_disabled_bios(rdev); 461 return r700_read_disabled_bios(rdev);
421 else if (rdev->family >= CHIP_R600) 462 else if (rdev->family >= CHIP_R600)
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index 137b8075f6e7..591fcae8f224 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -471,8 +471,9 @@ bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev)
471 return true; 471 return true;
472} 472}
473 473
474/* this is used for atom LCDs as well */
474struct edid * 475struct edid *
475radeon_combios_get_hardcoded_edid(struct radeon_device *rdev) 476radeon_bios_get_hardcoded_edid(struct radeon_device *rdev)
476{ 477{
477 if (rdev->mode_info.bios_hardcoded_edid) 478 if (rdev->mode_info.bios_hardcoded_edid)
478 return rdev->mode_info.bios_hardcoded_edid; 479 return rdev->mode_info.bios_hardcoded_edid;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 8afaf7a7459e..22b7e3dc0eca 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -472,6 +472,9 @@ static int radeon_lvds_get_modes(struct drm_connector *connector)
472 if (mode) { 472 if (mode) {
473 ret = 1; 473 ret = 1;
474 drm_mode_probed_add(connector, mode); 474 drm_mode_probed_add(connector, mode);
475 /* add the width/height from vbios tables if available */
476 connector->display_info.width_mm = mode->width_mm;
477 connector->display_info.height_mm = mode->height_mm;
475 /* add scaled modes */ 478 /* add scaled modes */
476 radeon_add_common_modes(encoder, connector); 479 radeon_add_common_modes(encoder, connector);
477 } 480 }
@@ -1216,7 +1219,7 @@ radeon_add_atom_connector(struct drm_device *dev,
1216 if (ASIC_IS_AVIVO(rdev)) { 1219 if (ASIC_IS_AVIVO(rdev)) {
1217 drm_connector_attach_property(&radeon_connector->base, 1220 drm_connector_attach_property(&radeon_connector->base,
1218 rdev->mode_info.underscan_property, 1221 rdev->mode_info.underscan_property,
1219 UNDERSCAN_AUTO); 1222 UNDERSCAN_OFF);
1220 drm_connector_attach_property(&radeon_connector->base, 1223 drm_connector_attach_property(&radeon_connector->base,
1221 rdev->mode_info.underscan_hborder_property, 1224 rdev->mode_info.underscan_hborder_property,
1222 0); 1225 0);
@@ -1256,7 +1259,7 @@ radeon_add_atom_connector(struct drm_device *dev,
1256 if (ASIC_IS_AVIVO(rdev)) { 1259 if (ASIC_IS_AVIVO(rdev)) {
1257 drm_connector_attach_property(&radeon_connector->base, 1260 drm_connector_attach_property(&radeon_connector->base,
1258 rdev->mode_info.underscan_property, 1261 rdev->mode_info.underscan_property,
1259 UNDERSCAN_AUTO); 1262 UNDERSCAN_OFF);
1260 drm_connector_attach_property(&radeon_connector->base, 1263 drm_connector_attach_property(&radeon_connector->base,
1261 rdev->mode_info.underscan_hborder_property, 1264 rdev->mode_info.underscan_hborder_property,
1262 0); 1265 0);
@@ -1299,7 +1302,7 @@ radeon_add_atom_connector(struct drm_device *dev,
1299 if (ASIC_IS_AVIVO(rdev)) { 1302 if (ASIC_IS_AVIVO(rdev)) {
1300 drm_connector_attach_property(&radeon_connector->base, 1303 drm_connector_attach_property(&radeon_connector->base,
1301 rdev->mode_info.underscan_property, 1304 rdev->mode_info.underscan_property,
1302 UNDERSCAN_AUTO); 1305 UNDERSCAN_OFF);
1303 drm_connector_attach_property(&radeon_connector->base, 1306 drm_connector_attach_property(&radeon_connector->base,
1304 rdev->mode_info.underscan_hborder_property, 1307 rdev->mode_info.underscan_hborder_property,
1305 0); 1308 0);
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 6d64a2705f12..35b5eb8fbe2a 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -77,13 +77,13 @@ int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
77 p->relocs_ptr[i] = &p->relocs[i]; 77 p->relocs_ptr[i] = &p->relocs[i];
78 p->relocs[i].robj = p->relocs[i].gobj->driver_private; 78 p->relocs[i].robj = p->relocs[i].gobj->driver_private;
79 p->relocs[i].lobj.bo = p->relocs[i].robj; 79 p->relocs[i].lobj.bo = p->relocs[i].robj;
80 p->relocs[i].lobj.rdomain = r->read_domains;
81 p->relocs[i].lobj.wdomain = r->write_domain; 80 p->relocs[i].lobj.wdomain = r->write_domain;
81 p->relocs[i].lobj.rdomain = r->read_domains;
82 p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
82 p->relocs[i].handle = r->handle; 83 p->relocs[i].handle = r->handle;
83 p->relocs[i].flags = r->flags; 84 p->relocs[i].flags = r->flags;
84 INIT_LIST_HEAD(&p->relocs[i].lobj.list);
85 radeon_bo_list_add_object(&p->relocs[i].lobj, 85 radeon_bo_list_add_object(&p->relocs[i].lobj,
86 &p->validated); 86 &p->validated);
87 } 87 }
88 } 88 }
89 return radeon_bo_list_validate(&p->validated); 89 return radeon_bo_list_validate(&p->validated);
@@ -189,10 +189,13 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
189{ 189{
190 unsigned i; 190 unsigned i;
191 191
192 if (!error && parser->ib) { 192
193 radeon_bo_list_fence(&parser->validated, parser->ib->fence); 193 if (!error && parser->ib)
194 } 194 ttm_eu_fence_buffer_objects(&parser->validated,
195 radeon_bo_list_unreserve(&parser->validated); 195 parser->ib->fence);
196 else
197 ttm_eu_backoff_reservation(&parser->validated);
198
196 if (parser->relocs != NULL) { 199 if (parser->relocs != NULL) {
197 for (i = 0; i < parser->nrelocs; i++) { 200 for (i = 0; i < parser->nrelocs; i++) {
198 if (parser->relocs[i].gobj) 201 if (parser->relocs[i].gobj)
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 501966a13f48..26091d602b84 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -81,6 +81,10 @@ static const char radeon_family_name[][16] = {
81 "JUNIPER", 81 "JUNIPER",
82 "CYPRESS", 82 "CYPRESS",
83 "HEMLOCK", 83 "HEMLOCK",
84 "PALM",
85 "BARTS",
86 "TURKS",
87 "CAICOS",
84 "LAST", 88 "LAST",
85}; 89};
86 90
@@ -224,6 +228,11 @@ int radeon_wb_init(struct radeon_device *rdev)
224 rdev->wb.use_event = true; 228 rdev->wb.use_event = true;
225 } 229 }
226 } 230 }
231 /* always use writeback/events on NI */
232 if (ASIC_IS_DCE5(rdev)) {
233 rdev->wb.enabled = true;
234 rdev->wb.use_event = true;
235 }
227 236
228 dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis"); 237 dev_info(rdev->dev, "WB %sabled\n", rdev->wb.enabled ? "en" : "dis");
229 238
@@ -335,7 +344,12 @@ bool radeon_card_posted(struct radeon_device *rdev)
335 uint32_t reg; 344 uint32_t reg;
336 345
337 /* first check CRTCs */ 346 /* first check CRTCs */
338 if (ASIC_IS_DCE4(rdev)) { 347 if (ASIC_IS_DCE41(rdev)) {
348 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
349 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
350 if (reg & EVERGREEN_CRTC_MASTER_EN)
351 return true;
352 } else if (ASIC_IS_DCE4(rdev)) {
339 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) | 353 reg = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) |
340 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) | 354 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) |
341 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) | 355 RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) |
@@ -636,20 +650,20 @@ void radeon_check_arguments(struct radeon_device *rdev)
636static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) 650static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
637{ 651{
638 struct drm_device *dev = pci_get_drvdata(pdev); 652 struct drm_device *dev = pci_get_drvdata(pdev);
639 struct radeon_device *rdev = dev->dev_private;
640 pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; 653 pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
641 if (state == VGA_SWITCHEROO_ON) { 654 if (state == VGA_SWITCHEROO_ON) {
642 printk(KERN_INFO "radeon: switched on\n"); 655 printk(KERN_INFO "radeon: switched on\n");
643 /* don't suspend or resume card normally */ 656 /* don't suspend or resume card normally */
644 rdev->powered_down = false; 657 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
645 radeon_resume_kms(dev); 658 radeon_resume_kms(dev);
659 dev->switch_power_state = DRM_SWITCH_POWER_ON;
646 drm_kms_helper_poll_enable(dev); 660 drm_kms_helper_poll_enable(dev);
647 } else { 661 } else {
648 printk(KERN_INFO "radeon: switched off\n"); 662 printk(KERN_INFO "radeon: switched off\n");
649 drm_kms_helper_poll_disable(dev); 663 drm_kms_helper_poll_disable(dev);
664 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
650 radeon_suspend_kms(dev, pmm); 665 radeon_suspend_kms(dev, pmm);
651 /* don't suspend or resume card normally */ 666 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
652 rdev->powered_down = true;
653 } 667 }
654} 668}
655 669
@@ -704,11 +718,6 @@ int radeon_device_init(struct radeon_device *rdev,
704 init_waitqueue_head(&rdev->irq.vblank_queue); 718 init_waitqueue_head(&rdev->irq.vblank_queue);
705 init_waitqueue_head(&rdev->irq.idle_queue); 719 init_waitqueue_head(&rdev->irq.idle_queue);
706 720
707 /* setup workqueue */
708 rdev->wq = create_workqueue("radeon");
709 if (rdev->wq == NULL)
710 return -ENOMEM;
711
712 /* Set asic functions */ 721 /* Set asic functions */
713 r = radeon_asic_init(rdev); 722 r = radeon_asic_init(rdev);
714 if (r) 723 if (r)
@@ -773,6 +782,7 @@ int radeon_device_init(struct radeon_device *rdev,
773 vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode); 782 vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
774 vga_switcheroo_register_client(rdev->pdev, 783 vga_switcheroo_register_client(rdev->pdev,
775 radeon_switcheroo_set_state, 784 radeon_switcheroo_set_state,
785 NULL,
776 radeon_switcheroo_can_switch); 786 radeon_switcheroo_can_switch);
777 787
778 r = radeon_init(rdev); 788 r = radeon_init(rdev);
@@ -806,7 +816,6 @@ void radeon_device_fini(struct radeon_device *rdev)
806 /* evict vram memory */ 816 /* evict vram memory */
807 radeon_bo_evict_vram(rdev); 817 radeon_bo_evict_vram(rdev);
808 radeon_fini(rdev); 818 radeon_fini(rdev);
809 destroy_workqueue(rdev->wq);
810 vga_switcheroo_unregister_client(rdev->pdev); 819 vga_switcheroo_unregister_client(rdev->pdev);
811 vga_client_register(rdev->pdev, NULL, NULL, NULL); 820 vga_client_register(rdev->pdev, NULL, NULL, NULL);
812 if (rdev->rio_mem) 821 if (rdev->rio_mem)
@@ -835,7 +844,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
835 } 844 }
836 rdev = dev->dev_private; 845 rdev = dev->dev_private;
837 846
838 if (rdev->powered_down) 847 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
839 return 0; 848 return 0;
840 849
841 /* turn off display hw */ 850 /* turn off display hw */
@@ -893,7 +902,7 @@ int radeon_resume_kms(struct drm_device *dev)
893 struct drm_connector *connector; 902 struct drm_connector *connector;
894 struct radeon_device *rdev = dev->dev_private; 903 struct radeon_device *rdev = dev->dev_private;
895 904
896 if (rdev->powered_down) 905 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
897 return 0; 906 return 0;
898 907
899 acquire_console_sem(); 908 acquire_console_sem();
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 1df4dc6c063c..d26dabf878d9 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -68,7 +68,7 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc)
68 WREG32(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id); 68 WREG32(AVIVO_D1GRPH_LUT_SEL + radeon_crtc->crtc_offset, radeon_crtc->crtc_id);
69} 69}
70 70
71static void evergreen_crtc_load_lut(struct drm_crtc *crtc) 71static void dce4_crtc_load_lut(struct drm_crtc *crtc)
72{ 72{
73 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); 73 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
74 struct drm_device *dev = crtc->dev; 74 struct drm_device *dev = crtc->dev;
@@ -98,6 +98,66 @@ static void evergreen_crtc_load_lut(struct drm_crtc *crtc)
98 } 98 }
99} 99}
100 100
101static void dce5_crtc_load_lut(struct drm_crtc *crtc)
102{
103 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
104 struct drm_device *dev = crtc->dev;
105 struct radeon_device *rdev = dev->dev_private;
106 int i;
107
108 DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
109
110 WREG32(NI_INPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
111 (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
112 NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
113 WREG32(NI_PRESCALE_GRPH_CONTROL + radeon_crtc->crtc_offset,
114 NI_GRPH_PRESCALE_BYPASS);
115 WREG32(NI_PRESCALE_OVL_CONTROL + radeon_crtc->crtc_offset,
116 NI_OVL_PRESCALE_BYPASS);
117 WREG32(NI_INPUT_GAMMA_CONTROL + radeon_crtc->crtc_offset,
118 (NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
119 NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
120
121 WREG32(EVERGREEN_DC_LUT_CONTROL + radeon_crtc->crtc_offset, 0);
122
123 WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + radeon_crtc->crtc_offset, 0);
124 WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + radeon_crtc->crtc_offset, 0);
125 WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + radeon_crtc->crtc_offset, 0);
126
127 WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + radeon_crtc->crtc_offset, 0xffff);
128 WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + radeon_crtc->crtc_offset, 0xffff);
129 WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + radeon_crtc->crtc_offset, 0xffff);
130
131 WREG32(EVERGREEN_DC_LUT_RW_MODE + radeon_crtc->crtc_offset, 0);
132 WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
133
134 WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
135 for (i = 0; i < 256; i++) {
136 WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
137 (radeon_crtc->lut_r[i] << 20) |
138 (radeon_crtc->lut_g[i] << 10) |
139 (radeon_crtc->lut_b[i] << 0));
140 }
141
142 WREG32(NI_DEGAMMA_CONTROL + radeon_crtc->crtc_offset,
143 (NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
144 NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
145 NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
146 NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
147 WREG32(NI_GAMUT_REMAP_CONTROL + radeon_crtc->crtc_offset,
148 (NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
149 NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
150 WREG32(NI_REGAMMA_CONTROL + radeon_crtc->crtc_offset,
151 (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
152 NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
153 WREG32(NI_OUTPUT_CSC_CONTROL + radeon_crtc->crtc_offset,
154 (NI_OUTPUT_CSC_GRPH_MODE(NI_OUTPUT_CSC_BYPASS) |
155 NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
156 /* XXX match this to the depth of the crtc fmt block, move to modeset? */
157 WREG32(0x6940 + radeon_crtc->crtc_offset, 0);
158
159}
160
101static void legacy_crtc_load_lut(struct drm_crtc *crtc) 161static void legacy_crtc_load_lut(struct drm_crtc *crtc)
102{ 162{
103 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); 163 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
@@ -130,8 +190,10 @@ void radeon_crtc_load_lut(struct drm_crtc *crtc)
130 if (!crtc->enabled) 190 if (!crtc->enabled)
131 return; 191 return;
132 192
133 if (ASIC_IS_DCE4(rdev)) 193 if (ASIC_IS_DCE5(rdev))
134 evergreen_crtc_load_lut(crtc); 194 dce5_crtc_load_lut(crtc);
195 else if (ASIC_IS_DCE4(rdev))
196 dce4_crtc_load_lut(crtc);
135 else if (ASIC_IS_AVIVO(rdev)) 197 else if (ASIC_IS_AVIVO(rdev))
136 avivo_crtc_load_lut(crtc); 198 avivo_crtc_load_lut(crtc);
137 else 199 else
@@ -183,12 +245,272 @@ static void radeon_crtc_destroy(struct drm_crtc *crtc)
183 kfree(radeon_crtc); 245 kfree(radeon_crtc);
184} 246}
185 247
248/*
249 * Handle unpin events outside the interrupt handler proper.
250 */
251static void radeon_unpin_work_func(struct work_struct *__work)
252{
253 struct radeon_unpin_work *work =
254 container_of(__work, struct radeon_unpin_work, work);
255 int r;
256
257 /* unpin of the old buffer */
258 r = radeon_bo_reserve(work->old_rbo, false);
259 if (likely(r == 0)) {
260 r = radeon_bo_unpin(work->old_rbo);
261 if (unlikely(r != 0)) {
262 DRM_ERROR("failed to unpin buffer after flip\n");
263 }
264 radeon_bo_unreserve(work->old_rbo);
265 } else
266 DRM_ERROR("failed to reserve buffer after flip\n");
267 kfree(work);
268}
269
270void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
271{
272 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
273 struct radeon_unpin_work *work;
274 struct drm_pending_vblank_event *e;
275 struct timeval now;
276 unsigned long flags;
277 u32 update_pending;
278 int vpos, hpos;
279
280 spin_lock_irqsave(&rdev->ddev->event_lock, flags);
281 work = radeon_crtc->unpin_work;
282 if (work == NULL ||
283 !radeon_fence_signaled(work->fence)) {
284 spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
285 return;
286 }
287 /* New pageflip, or just completion of a previous one? */
288 if (!radeon_crtc->deferred_flip_completion) {
289 /* do the flip (mmio) */
290 update_pending = radeon_page_flip(rdev, crtc_id, work->new_crtc_base);
291 } else {
292 /* This is just a completion of a flip queued in crtc
293 * at last invocation. Make sure we go directly to
294 * completion routine.
295 */
296 update_pending = 0;
297 radeon_crtc->deferred_flip_completion = 0;
298 }
299
300 /* Has the pageflip already completed in crtc, or is it certain
301 * to complete in this vblank?
302 */
303 if (update_pending &&
304 (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id,
305 &vpos, &hpos)) &&
306 (vpos >=0) &&
307 (vpos < (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100)) {
308 /* crtc didn't flip in this target vblank interval,
309 * but flip is pending in crtc. It will complete it
310 * in next vblank interval, so complete the flip at
311 * next vblank irq.
312 */
313 radeon_crtc->deferred_flip_completion = 1;
314 spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
315 return;
316 }
317
318 /* Pageflip (will be) certainly completed in this vblank. Clean up. */
319 radeon_crtc->unpin_work = NULL;
320
321 /* wakeup userspace */
322 if (work->event) {
323 e = work->event;
324 e->event.sequence = drm_vblank_count_and_time(rdev->ddev, crtc_id, &now);
325 e->event.tv_sec = now.tv_sec;
326 e->event.tv_usec = now.tv_usec;
327 list_add_tail(&e->base.link, &e->base.file_priv->event_list);
328 wake_up_interruptible(&e->base.file_priv->event_wait);
329 }
330 spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
331
332 drm_vblank_put(rdev->ddev, radeon_crtc->crtc_id);
333 radeon_fence_unref(&work->fence);
334 radeon_post_page_flip(work->rdev, work->crtc_id);
335 schedule_work(&work->work);
336}
337
338static int radeon_crtc_page_flip(struct drm_crtc *crtc,
339 struct drm_framebuffer *fb,
340 struct drm_pending_vblank_event *event)
341{
342 struct drm_device *dev = crtc->dev;
343 struct radeon_device *rdev = dev->dev_private;
344 struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
345 struct radeon_framebuffer *old_radeon_fb;
346 struct radeon_framebuffer *new_radeon_fb;
347 struct drm_gem_object *obj;
348 struct radeon_bo *rbo;
349 struct radeon_fence *fence;
350 struct radeon_unpin_work *work;
351 unsigned long flags;
352 u32 tiling_flags, pitch_pixels;
353 u64 base;
354 int r;
355
356 work = kzalloc(sizeof *work, GFP_KERNEL);
357 if (work == NULL)
358 return -ENOMEM;
359
360 r = radeon_fence_create(rdev, &fence);
361 if (unlikely(r != 0)) {
362 kfree(work);
363 DRM_ERROR("flip queue: failed to create fence.\n");
364 return -ENOMEM;
365 }
366 work->event = event;
367 work->rdev = rdev;
368 work->crtc_id = radeon_crtc->crtc_id;
369 work->fence = radeon_fence_ref(fence);
370 old_radeon_fb = to_radeon_framebuffer(crtc->fb);
371 new_radeon_fb = to_radeon_framebuffer(fb);
372 /* schedule unpin of the old buffer */
373 obj = old_radeon_fb->obj;
374 rbo = obj->driver_private;
375 work->old_rbo = rbo;
376 INIT_WORK(&work->work, radeon_unpin_work_func);
377
378 /* We borrow the event spin lock for protecting unpin_work */
379 spin_lock_irqsave(&dev->event_lock, flags);
380 if (radeon_crtc->unpin_work) {
381 spin_unlock_irqrestore(&dev->event_lock, flags);
382 kfree(work);
383 radeon_fence_unref(&fence);
384
385 DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
386 return -EBUSY;
387 }
388 radeon_crtc->unpin_work = work;
389 radeon_crtc->deferred_flip_completion = 0;
390 spin_unlock_irqrestore(&dev->event_lock, flags);
391
392 /* pin the new buffer */
393 obj = new_radeon_fb->obj;
394 rbo = obj->driver_private;
395
396 DRM_DEBUG_DRIVER("flip-ioctl() cur_fbo = %p, cur_bbo = %p\n",
397 work->old_rbo, rbo);
398
399 r = radeon_bo_reserve(rbo, false);
400 if (unlikely(r != 0)) {
401 DRM_ERROR("failed to reserve new rbo buffer before flip\n");
402 goto pflip_cleanup;
403 }
404 r = radeon_bo_pin(rbo, RADEON_GEM_DOMAIN_VRAM, &base);
405 if (unlikely(r != 0)) {
406 radeon_bo_unreserve(rbo);
407 r = -EINVAL;
408 DRM_ERROR("failed to pin new rbo buffer before flip\n");
409 goto pflip_cleanup;
410 }
411 radeon_bo_get_tiling_flags(rbo, &tiling_flags, NULL);
412 radeon_bo_unreserve(rbo);
413
414 if (!ASIC_IS_AVIVO(rdev)) {
415 /* crtc offset is from display base addr not FB location */
416 base -= radeon_crtc->legacy_display_base_addr;
417 pitch_pixels = fb->pitch / (fb->bits_per_pixel / 8);
418
419 if (tiling_flags & RADEON_TILING_MACRO) {
420 if (ASIC_IS_R300(rdev)) {
421 base &= ~0x7ff;
422 } else {
423 int byteshift = fb->bits_per_pixel >> 4;
424 int tile_addr = (((crtc->y >> 3) * pitch_pixels + crtc->x) >> (8 - byteshift)) << 11;
425 base += tile_addr + ((crtc->x << byteshift) % 256) + ((crtc->y % 8) << 8);
426 }
427 } else {
428 int offset = crtc->y * pitch_pixels + crtc->x;
429 switch (fb->bits_per_pixel) {
430 case 8:
431 default:
432 offset *= 1;
433 break;
434 case 15:
435 case 16:
436 offset *= 2;
437 break;
438 case 24:
439 offset *= 3;
440 break;
441 case 32:
442 offset *= 4;
443 break;
444 }
445 base += offset;
446 }
447 base &= ~7;
448 }
449
450 spin_lock_irqsave(&dev->event_lock, flags);
451 work->new_crtc_base = base;
452 spin_unlock_irqrestore(&dev->event_lock, flags);
453
454 /* update crtc fb */
455 crtc->fb = fb;
456
457 r = drm_vblank_get(dev, radeon_crtc->crtc_id);
458 if (r) {
459 DRM_ERROR("failed to get vblank before flip\n");
460 goto pflip_cleanup1;
461 }
462
463 /* 32 ought to cover us */
464 r = radeon_ring_lock(rdev, 32);
465 if (r) {
466 DRM_ERROR("failed to lock the ring before flip\n");
467 goto pflip_cleanup2;
468 }
469
470 /* emit the fence */
471 radeon_fence_emit(rdev, fence);
472 /* set the proper interrupt */
473 radeon_pre_page_flip(rdev, radeon_crtc->crtc_id);
474 /* fire the ring */
475 radeon_ring_unlock_commit(rdev);
476
477 return 0;
478
479pflip_cleanup2:
480 drm_vblank_put(dev, radeon_crtc->crtc_id);
481
482pflip_cleanup1:
483 r = radeon_bo_reserve(rbo, false);
484 if (unlikely(r != 0)) {
485 DRM_ERROR("failed to reserve new rbo in error path\n");
486 goto pflip_cleanup;
487 }
488 r = radeon_bo_unpin(rbo);
489 if (unlikely(r != 0)) {
490 radeon_bo_unreserve(rbo);
491 r = -EINVAL;
492 DRM_ERROR("failed to unpin new rbo in error path\n");
493 goto pflip_cleanup;
494 }
495 radeon_bo_unreserve(rbo);
496
497pflip_cleanup:
498 spin_lock_irqsave(&dev->event_lock, flags);
499 radeon_crtc->unpin_work = NULL;
500 spin_unlock_irqrestore(&dev->event_lock, flags);
501 radeon_fence_unref(&fence);
502 kfree(work);
503
504 return r;
505}
506
186static const struct drm_crtc_funcs radeon_crtc_funcs = { 507static const struct drm_crtc_funcs radeon_crtc_funcs = {
187 .cursor_set = radeon_crtc_cursor_set, 508 .cursor_set = radeon_crtc_cursor_set,
188 .cursor_move = radeon_crtc_cursor_move, 509 .cursor_move = radeon_crtc_cursor_move,
189 .gamma_set = radeon_crtc_gamma_set, 510 .gamma_set = radeon_crtc_gamma_set,
190 .set_config = drm_crtc_helper_set_config, 511 .set_config = drm_crtc_helper_set_config,
191 .destroy = radeon_crtc_destroy, 512 .destroy = radeon_crtc_destroy,
513 .page_flip = radeon_crtc_page_flip,
192}; 514};
193 515
194static void radeon_crtc_init(struct drm_device *dev, int index) 516static void radeon_crtc_init(struct drm_device *dev, int index)
@@ -225,7 +547,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
225 radeon_legacy_init_crtc(dev, radeon_crtc); 547 radeon_legacy_init_crtc(dev, radeon_crtc);
226} 548}
227 549
228static const char *encoder_names[34] = { 550static const char *encoder_names[36] = {
229 "NONE", 551 "NONE",
230 "INTERNAL_LVDS", 552 "INTERNAL_LVDS",
231 "INTERNAL_TMDS1", 553 "INTERNAL_TMDS1",
@@ -260,6 +582,8 @@ static const char *encoder_names[34] = {
260 "INTERNAL_KLDSCP_LVTMA", 582 "INTERNAL_KLDSCP_LVTMA",
261 "INTERNAL_UNIPHY1", 583 "INTERNAL_UNIPHY1",
262 "INTERNAL_UNIPHY2", 584 "INTERNAL_UNIPHY2",
585 "NUTMEG",
586 "TRAVIS",
263}; 587};
264 588
265static const char *connector_names[15] = { 589static const char *connector_names[15] = {
@@ -417,9 +741,17 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
417 if (!radeon_connector->edid) { 741 if (!radeon_connector->edid) {
418 radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter); 742 radeon_connector->edid = drm_get_edid(&radeon_connector->base, &radeon_connector->ddc_bus->adapter);
419 } 743 }
420 /* some servers provide a hardcoded edid in rom for KVMs */ 744
421 if (!radeon_connector->edid) 745 if (!radeon_connector->edid) {
422 radeon_connector->edid = radeon_combios_get_hardcoded_edid(rdev); 746 if (rdev->is_atom_bios) {
747 /* some laptops provide a hardcoded edid in rom for LCDs */
748 if (((radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_LVDS) ||
749 (radeon_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)))
750 radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
751 } else
752 /* some servers provide a hardcoded edid in rom for KVMs */
753 radeon_connector->edid = radeon_bios_get_hardcoded_edid(rdev);
754 }
423 if (radeon_connector->edid) { 755 if (radeon_connector->edid) {
424 drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid); 756 drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
425 ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid); 757 ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
@@ -849,7 +1181,10 @@ int radeon_modeset_init(struct radeon_device *rdev)
849 1181
850 rdev->ddev->mode_config.funcs = (void *)&radeon_mode_funcs; 1182 rdev->ddev->mode_config.funcs = (void *)&radeon_mode_funcs;
851 1183
852 if (ASIC_IS_AVIVO(rdev)) { 1184 if (ASIC_IS_DCE5(rdev)) {
1185 rdev->ddev->mode_config.max_width = 16384;
1186 rdev->ddev->mode_config.max_height = 16384;
1187 } else if (ASIC_IS_AVIVO(rdev)) {
853 rdev->ddev->mode_config.max_width = 8192; 1188 rdev->ddev->mode_config.max_width = 8192;
854 rdev->ddev->mode_config.max_height = 8192; 1189 rdev->ddev->mode_config.max_height = 8192;
855 } else { 1190 } else {
@@ -1019,7 +1354,7 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
1019/* 1354/*
1020 * Retrieve current video scanout position of crtc on a given gpu. 1355 * Retrieve current video scanout position of crtc on a given gpu.
1021 * 1356 *
1022 * \param rdev Device to query. 1357 * \param dev Device to query.
1023 * \param crtc Crtc to query. 1358 * \param crtc Crtc to query.
1024 * \param *vpos Location where vertical scanout position should be stored. 1359 * \param *vpos Location where vertical scanout position should be stored.
1025 * \param *hpos Location where horizontal scanout position should go. 1360 * \param *hpos Location where horizontal scanout position should go.
@@ -1031,72 +1366,74 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
1031 * 1366 *
1032 * \return Flags, or'ed together as follows: 1367 * \return Flags, or'ed together as follows:
1033 * 1368 *
1034 * RADEON_SCANOUTPOS_VALID = Query successfull. 1369 * DRM_SCANOUTPOS_VALID = Query successfull.
1035 * RADEON_SCANOUTPOS_INVBL = Inside vblank. 1370 * DRM_SCANOUTPOS_INVBL = Inside vblank.
1036 * RADEON_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of 1371 * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of
1037 * this flag means that returned position may be offset by a constant but 1372 * this flag means that returned position may be offset by a constant but
1038 * unknown small number of scanlines wrt. real scanout position. 1373 * unknown small number of scanlines wrt. real scanout position.
1039 * 1374 *
1040 */ 1375 */
1041int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos, int *hpos) 1376int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int *hpos)
1042{ 1377{
1043 u32 stat_crtc = 0, vbl = 0, position = 0; 1378 u32 stat_crtc = 0, vbl = 0, position = 0;
1044 int vbl_start, vbl_end, vtotal, ret = 0; 1379 int vbl_start, vbl_end, vtotal, ret = 0;
1045 bool in_vbl = true; 1380 bool in_vbl = true;
1046 1381
1382 struct radeon_device *rdev = dev->dev_private;
1383
1047 if (ASIC_IS_DCE4(rdev)) { 1384 if (ASIC_IS_DCE4(rdev)) {
1048 if (crtc == 0) { 1385 if (crtc == 0) {
1049 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1386 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1050 EVERGREEN_CRTC0_REGISTER_OFFSET); 1387 EVERGREEN_CRTC0_REGISTER_OFFSET);
1051 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1388 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1052 EVERGREEN_CRTC0_REGISTER_OFFSET); 1389 EVERGREEN_CRTC0_REGISTER_OFFSET);
1053 ret |= RADEON_SCANOUTPOS_VALID; 1390 ret |= DRM_SCANOUTPOS_VALID;
1054 } 1391 }
1055 if (crtc == 1) { 1392 if (crtc == 1) {
1056 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1393 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1057 EVERGREEN_CRTC1_REGISTER_OFFSET); 1394 EVERGREEN_CRTC1_REGISTER_OFFSET);
1058 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1395 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1059 EVERGREEN_CRTC1_REGISTER_OFFSET); 1396 EVERGREEN_CRTC1_REGISTER_OFFSET);
1060 ret |= RADEON_SCANOUTPOS_VALID; 1397 ret |= DRM_SCANOUTPOS_VALID;
1061 } 1398 }
1062 if (crtc == 2) { 1399 if (crtc == 2) {
1063 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1400 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1064 EVERGREEN_CRTC2_REGISTER_OFFSET); 1401 EVERGREEN_CRTC2_REGISTER_OFFSET);
1065 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1402 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1066 EVERGREEN_CRTC2_REGISTER_OFFSET); 1403 EVERGREEN_CRTC2_REGISTER_OFFSET);
1067 ret |= RADEON_SCANOUTPOS_VALID; 1404 ret |= DRM_SCANOUTPOS_VALID;
1068 } 1405 }
1069 if (crtc == 3) { 1406 if (crtc == 3) {
1070 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1407 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1071 EVERGREEN_CRTC3_REGISTER_OFFSET); 1408 EVERGREEN_CRTC3_REGISTER_OFFSET);
1072 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1409 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1073 EVERGREEN_CRTC3_REGISTER_OFFSET); 1410 EVERGREEN_CRTC3_REGISTER_OFFSET);
1074 ret |= RADEON_SCANOUTPOS_VALID; 1411 ret |= DRM_SCANOUTPOS_VALID;
1075 } 1412 }
1076 if (crtc == 4) { 1413 if (crtc == 4) {
1077 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1414 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1078 EVERGREEN_CRTC4_REGISTER_OFFSET); 1415 EVERGREEN_CRTC4_REGISTER_OFFSET);
1079 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1416 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1080 EVERGREEN_CRTC4_REGISTER_OFFSET); 1417 EVERGREEN_CRTC4_REGISTER_OFFSET);
1081 ret |= RADEON_SCANOUTPOS_VALID; 1418 ret |= DRM_SCANOUTPOS_VALID;
1082 } 1419 }
1083 if (crtc == 5) { 1420 if (crtc == 5) {
1084 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + 1421 vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
1085 EVERGREEN_CRTC5_REGISTER_OFFSET); 1422 EVERGREEN_CRTC5_REGISTER_OFFSET);
1086 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + 1423 position = RREG32(EVERGREEN_CRTC_STATUS_POSITION +
1087 EVERGREEN_CRTC5_REGISTER_OFFSET); 1424 EVERGREEN_CRTC5_REGISTER_OFFSET);
1088 ret |= RADEON_SCANOUTPOS_VALID; 1425 ret |= DRM_SCANOUTPOS_VALID;
1089 } 1426 }
1090 } else if (ASIC_IS_AVIVO(rdev)) { 1427 } else if (ASIC_IS_AVIVO(rdev)) {
1091 if (crtc == 0) { 1428 if (crtc == 0) {
1092 vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END); 1429 vbl = RREG32(AVIVO_D1CRTC_V_BLANK_START_END);
1093 position = RREG32(AVIVO_D1CRTC_STATUS_POSITION); 1430 position = RREG32(AVIVO_D1CRTC_STATUS_POSITION);
1094 ret |= RADEON_SCANOUTPOS_VALID; 1431 ret |= DRM_SCANOUTPOS_VALID;
1095 } 1432 }
1096 if (crtc == 1) { 1433 if (crtc == 1) {
1097 vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END); 1434 vbl = RREG32(AVIVO_D2CRTC_V_BLANK_START_END);
1098 position = RREG32(AVIVO_D2CRTC_STATUS_POSITION); 1435 position = RREG32(AVIVO_D2CRTC_STATUS_POSITION);
1099 ret |= RADEON_SCANOUTPOS_VALID; 1436 ret |= DRM_SCANOUTPOS_VALID;
1100 } 1437 }
1101 } else { 1438 } else {
1102 /* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */ 1439 /* Pre-AVIVO: Different encoding of scanout pos and vblank interval. */
@@ -1112,7 +1449,7 @@ int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos,
1112 if (!(stat_crtc & 1)) 1449 if (!(stat_crtc & 1))
1113 in_vbl = false; 1450 in_vbl = false;
1114 1451
1115 ret |= RADEON_SCANOUTPOS_VALID; 1452 ret |= DRM_SCANOUTPOS_VALID;
1116 } 1453 }
1117 if (crtc == 1) { 1454 if (crtc == 1) {
1118 vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) & 1455 vbl = (RREG32(RADEON_CRTC2_V_TOTAL_DISP) &
@@ -1122,7 +1459,7 @@ int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos,
1122 if (!(stat_crtc & 1)) 1459 if (!(stat_crtc & 1))
1123 in_vbl = false; 1460 in_vbl = false;
1124 1461
1125 ret |= RADEON_SCANOUTPOS_VALID; 1462 ret |= DRM_SCANOUTPOS_VALID;
1126 } 1463 }
1127 } 1464 }
1128 1465
@@ -1133,13 +1470,13 @@ int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos,
1133 /* Valid vblank area boundaries from gpu retrieved? */ 1470 /* Valid vblank area boundaries from gpu retrieved? */
1134 if (vbl > 0) { 1471 if (vbl > 0) {
1135 /* Yes: Decode. */ 1472 /* Yes: Decode. */
1136 ret |= RADEON_SCANOUTPOS_ACCURATE; 1473 ret |= DRM_SCANOUTPOS_ACCURATE;
1137 vbl_start = vbl & 0x1fff; 1474 vbl_start = vbl & 0x1fff;
1138 vbl_end = (vbl >> 16) & 0x1fff; 1475 vbl_end = (vbl >> 16) & 0x1fff;
1139 } 1476 }
1140 else { 1477 else {
1141 /* No: Fake something reasonable which gives at least ok results. */ 1478 /* No: Fake something reasonable which gives at least ok results. */
1142 vbl_start = rdev->mode_info.crtcs[crtc]->base.mode.crtc_vdisplay; 1479 vbl_start = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vdisplay;
1143 vbl_end = 0; 1480 vbl_end = 0;
1144 } 1481 }
1145 1482
@@ -1155,7 +1492,7 @@ int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos,
1155 1492
1156 /* Inside "upper part" of vblank area? Apply corrective offset if so: */ 1493 /* Inside "upper part" of vblank area? Apply corrective offset if so: */
1157 if (in_vbl && (*vpos >= vbl_start)) { 1494 if (in_vbl && (*vpos >= vbl_start)) {
1158 vtotal = rdev->mode_info.crtcs[crtc]->base.mode.crtc_vtotal; 1495 vtotal = rdev->mode_info.crtcs[crtc]->base.hwmode.crtc_vtotal;
1159 *vpos = *vpos - vtotal; 1496 *vpos = *vpos - vtotal;
1160 } 1497 }
1161 1498
@@ -1164,7 +1501,7 @@ int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos,
1164 1501
1165 /* In vblank? */ 1502 /* In vblank? */
1166 if (in_vbl) 1503 if (in_vbl)
1167 ret |= RADEON_SCANOUTPOS_INVBL; 1504 ret |= DRM_SCANOUTPOS_INVBL;
1168 1505
1169 return ret; 1506 return ret;
1170} 1507}
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 60e689f2d048..be5cb4f28c29 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -48,9 +48,10 @@
48 * - 2.5.0 - add get accel 2 to work around ddx breakage for evergreen 48 * - 2.5.0 - add get accel 2 to work around ddx breakage for evergreen
49 * - 2.6.0 - add tiling config query (r6xx+), add initial HiZ support (r300->r500) 49 * - 2.6.0 - add tiling config query (r6xx+), add initial HiZ support (r300->r500)
50 * 2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs 50 * 2.7.0 - fixups for r600 2D tiling support. (no external ABI change), add eg dyn gpr regs
51 * 2.8.0 - pageflip support, r500 US_FORMAT regs. r500 ARGB2101010 colorbuf, r300->r500 CMASK
51 */ 52 */
52#define KMS_DRIVER_MAJOR 2 53#define KMS_DRIVER_MAJOR 2
53#define KMS_DRIVER_MINOR 7 54#define KMS_DRIVER_MINOR 8
54#define KMS_DRIVER_PATCHLEVEL 0 55#define KMS_DRIVER_PATCHLEVEL 0
55int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags); 56int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
56int radeon_driver_unload_kms(struct drm_device *dev); 57int radeon_driver_unload_kms(struct drm_device *dev);
@@ -66,6 +67,10 @@ int radeon_resume_kms(struct drm_device *dev);
66u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc); 67u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc);
67int radeon_enable_vblank_kms(struct drm_device *dev, int crtc); 68int radeon_enable_vblank_kms(struct drm_device *dev, int crtc);
68void radeon_disable_vblank_kms(struct drm_device *dev, int crtc); 69void radeon_disable_vblank_kms(struct drm_device *dev, int crtc);
70int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
71 int *max_error,
72 struct timeval *vblank_time,
73 unsigned flags);
69void radeon_driver_irq_preinstall_kms(struct drm_device *dev); 74void radeon_driver_irq_preinstall_kms(struct drm_device *dev);
70int radeon_driver_irq_postinstall_kms(struct drm_device *dev); 75int radeon_driver_irq_postinstall_kms(struct drm_device *dev);
71void radeon_driver_irq_uninstall_kms(struct drm_device *dev); 76void radeon_driver_irq_uninstall_kms(struct drm_device *dev);
@@ -74,6 +79,8 @@ int radeon_dma_ioctl_kms(struct drm_device *dev, void *data,
74 struct drm_file *file_priv); 79 struct drm_file *file_priv);
75int radeon_gem_object_init(struct drm_gem_object *obj); 80int radeon_gem_object_init(struct drm_gem_object *obj);
76void radeon_gem_object_free(struct drm_gem_object *obj); 81void radeon_gem_object_free(struct drm_gem_object *obj);
82extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
83 int *vpos, int *hpos);
77extern struct drm_ioctl_desc radeon_ioctls_kms[]; 84extern struct drm_ioctl_desc radeon_ioctls_kms[];
78extern int radeon_max_kms_ioctl; 85extern int radeon_max_kms_ioctl;
79int radeon_mmap(struct file *filp, struct vm_area_struct *vma); 86int radeon_mmap(struct file *filp, struct vm_area_struct *vma);
@@ -296,6 +303,8 @@ static struct drm_driver kms_driver = {
296 .get_vblank_counter = radeon_get_vblank_counter_kms, 303 .get_vblank_counter = radeon_get_vblank_counter_kms,
297 .enable_vblank = radeon_enable_vblank_kms, 304 .enable_vblank = radeon_enable_vblank_kms,
298 .disable_vblank = radeon_disable_vblank_kms, 305 .disable_vblank = radeon_disable_vblank_kms,
306 .get_vblank_timestamp = radeon_get_vblank_timestamp_kms,
307 .get_scanout_position = radeon_get_crtc_scanoutpos,
299#if defined(CONFIG_DEBUG_FS) 308#if defined(CONFIG_DEBUG_FS)
300 .debugfs_init = radeon_debugfs_init, 309 .debugfs_init = radeon_debugfs_init,
301 .debugfs_cleanup = radeon_debugfs_cleanup, 310 .debugfs_cleanup = radeon_debugfs_cleanup,
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index 041943df966b..8fd184286c0b 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -641,7 +641,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
641 switch (connector->connector_type) { 641 switch (connector->connector_type) {
642 case DRM_MODE_CONNECTOR_DVII: 642 case DRM_MODE_CONNECTOR_DVII:
643 case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */ 643 case DRM_MODE_CONNECTOR_HDMIB: /* HDMI-B is basically DL-DVI; analog works fine */
644 if (drm_detect_hdmi_monitor(radeon_connector->edid)) { 644 if (drm_detect_monitor_audio(radeon_connector->edid)) {
645 /* fix me */ 645 /* fix me */
646 if (ASIC_IS_DCE4(rdev)) 646 if (ASIC_IS_DCE4(rdev))
647 return ATOM_ENCODER_MODE_DVI; 647 return ATOM_ENCODER_MODE_DVI;
@@ -655,7 +655,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
655 case DRM_MODE_CONNECTOR_DVID: 655 case DRM_MODE_CONNECTOR_DVID:
656 case DRM_MODE_CONNECTOR_HDMIA: 656 case DRM_MODE_CONNECTOR_HDMIA:
657 default: 657 default:
658 if (drm_detect_hdmi_monitor(radeon_connector->edid)) { 658 if (drm_detect_monitor_audio(radeon_connector->edid)) {
659 /* fix me */ 659 /* fix me */
660 if (ASIC_IS_DCE4(rdev)) 660 if (ASIC_IS_DCE4(rdev))
661 return ATOM_ENCODER_MODE_DVI; 661 return ATOM_ENCODER_MODE_DVI;
@@ -673,7 +673,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
673 if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) || 673 if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
674 (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP)) 674 (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
675 return ATOM_ENCODER_MODE_DP; 675 return ATOM_ENCODER_MODE_DP;
676 else if (drm_detect_hdmi_monitor(radeon_connector->edid)) { 676 else if (drm_detect_monitor_audio(radeon_connector->edid)) {
677 /* fix me */ 677 /* fix me */
678 if (ASIC_IS_DCE4(rdev)) 678 if (ASIC_IS_DCE4(rdev))
679 return ATOM_ENCODER_MODE_DVI; 679 return ATOM_ENCODER_MODE_DVI;
@@ -712,8 +712,8 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
712 * - 2 DIG encoder blocks. 712 * - 2 DIG encoder blocks.
713 * DIG1/2 can drive UNIPHY0/1/2 link A or link B 713 * DIG1/2 can drive UNIPHY0/1/2 link A or link B
714 * 714 *
715 * DCE 4.0 715 * DCE 4.0/5.0
716 * - 3 DIG transmitter blocks UNPHY0/1/2 (links A and B). 716 * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B).
717 * Supports up to 6 digital outputs 717 * Supports up to 6 digital outputs
718 * - 6 DIG encoder blocks. 718 * - 6 DIG encoder blocks.
719 * - DIG to PHY mapping is hardcoded 719 * - DIG to PHY mapping is hardcoded
@@ -724,6 +724,12 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
724 * DIG5 drives UNIPHY2 link A, A+B 724 * DIG5 drives UNIPHY2 link A, A+B
725 * DIG6 drives UNIPHY2 link B 725 * DIG6 drives UNIPHY2 link B
726 * 726 *
727 * DCE 4.1
728 * - 3 DIG transmitter blocks UNIPHY0/1/2 (links A and B).
729 * Supports up to 6 digital outputs
730 * - 2 DIG encoder blocks.
731 * DIG1/2 can drive UNIPHY0/1/2 link A or link B
732 *
727 * Routing 733 * Routing
728 * crtc -> dig encoder -> UNIPHY/LVTMA (1 or 2 links) 734 * crtc -> dig encoder -> UNIPHY/LVTMA (1 or 2 links)
729 * Examples: 735 * Examples:
@@ -737,6 +743,7 @@ union dig_encoder_control {
737 DIG_ENCODER_CONTROL_PS_ALLOCATION v1; 743 DIG_ENCODER_CONTROL_PS_ALLOCATION v1;
738 DIG_ENCODER_CONTROL_PARAMETERS_V2 v2; 744 DIG_ENCODER_CONTROL_PARAMETERS_V2 v2;
739 DIG_ENCODER_CONTROL_PARAMETERS_V3 v3; 745 DIG_ENCODER_CONTROL_PARAMETERS_V3 v3;
746 DIG_ENCODER_CONTROL_PARAMETERS_V4 v4;
740}; 747};
741 748
742void 749void
@@ -752,6 +759,7 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
752 uint8_t frev, crev; 759 uint8_t frev, crev;
753 int dp_clock = 0; 760 int dp_clock = 0;
754 int dp_lane_count = 0; 761 int dp_lane_count = 0;
762 int hpd_id = RADEON_HPD_NONE;
755 763
756 if (connector) { 764 if (connector) {
757 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 765 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -760,6 +768,7 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
760 768
761 dp_clock = dig_connector->dp_clock; 769 dp_clock = dig_connector->dp_clock;
762 dp_lane_count = dig_connector->dp_lane_count; 770 dp_lane_count = dig_connector->dp_lane_count;
771 hpd_id = radeon_connector->hpd.hpd;
763 } 772 }
764 773
765 /* no dig encoder assigned */ 774 /* no dig encoder assigned */
@@ -784,19 +793,36 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action)
784 args.v1.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10); 793 args.v1.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
785 args.v1.ucEncoderMode = atombios_get_encoder_mode(encoder); 794 args.v1.ucEncoderMode = atombios_get_encoder_mode(encoder);
786 795
787 if (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) { 796 if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
788 if (dp_clock == 270000) 797 (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST))
789 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
790 args.v1.ucLaneNum = dp_lane_count; 798 args.v1.ucLaneNum = dp_lane_count;
791 } else if (radeon_encoder->pixel_clock > 165000) 799 else if (radeon_encoder->pixel_clock > 165000)
792 args.v1.ucLaneNum = 8; 800 args.v1.ucLaneNum = 8;
793 else 801 else
794 args.v1.ucLaneNum = 4; 802 args.v1.ucLaneNum = 4;
795 803
796 if (ASIC_IS_DCE4(rdev)) { 804 if (ASIC_IS_DCE5(rdev)) {
805 if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) ||
806 (args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP_MST)) {
807 if (dp_clock == 270000)
808 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
809 else if (dp_clock == 540000)
810 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ;
811 }
812 args.v4.acConfig.ucDigSel = dig->dig_encoder;
813 args.v4.ucBitPerColor = PANEL_8BIT_PER_COLOR;
814 if (hpd_id == RADEON_HPD_NONE)
815 args.v4.ucHPD_ID = 0;
816 else
817 args.v4.ucHPD_ID = hpd_id + 1;
818 } else if (ASIC_IS_DCE4(rdev)) {
819 if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
820 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
797 args.v3.acConfig.ucDigSel = dig->dig_encoder; 821 args.v3.acConfig.ucDigSel = dig->dig_encoder;
798 args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR; 822 args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR;
799 } else { 823 } else {
824 if ((args.v1.ucEncoderMode == ATOM_ENCODER_MODE_DP) && (dp_clock == 270000))
825 args.v1.ucConfig |= ATOM_ENCODER_CONFIG_DPLINKRATE_2_70GHZ;
800 switch (radeon_encoder->encoder_id) { 826 switch (radeon_encoder->encoder_id) {
801 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: 827 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
802 args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1; 828 args.v1.ucConfig = ATOM_ENCODER_CONFIG_V2_TRANSMITTER1;
@@ -823,6 +849,7 @@ union dig_transmitter_control {
823 DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1; 849 DIG_TRANSMITTER_CONTROL_PS_ALLOCATION v1;
824 DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2; 850 DIG_TRANSMITTER_CONTROL_PARAMETERS_V2 v2;
825 DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 v3; 851 DIG_TRANSMITTER_CONTROL_PARAMETERS_V3 v3;
852 DIG_TRANSMITTER_CONTROL_PARAMETERS_V4 v4;
826}; 853};
827 854
828void 855void
@@ -917,10 +944,18 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
917 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 944 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
918 pll_id = radeon_crtc->pll_id; 945 pll_id = radeon_crtc->pll_id;
919 } 946 }
920 if (is_dp && rdev->clock.dp_extclk) 947
921 args.v3.acConfig.ucRefClkSource = 2; /* external src */ 948 if (ASIC_IS_DCE5(rdev)) {
922 else 949 if (is_dp && rdev->clock.dp_extclk)
923 args.v3.acConfig.ucRefClkSource = pll_id; 950 args.v4.acConfig.ucRefClkSource = 3; /* external src */
951 else
952 args.v4.acConfig.ucRefClkSource = pll_id;
953 } else {
954 if (is_dp && rdev->clock.dp_extclk)
955 args.v3.acConfig.ucRefClkSource = 2; /* external src */
956 else
957 args.v3.acConfig.ucRefClkSource = pll_id;
958 }
924 959
925 switch (radeon_encoder->encoder_id) { 960 switch (radeon_encoder->encoder_id) {
926 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY: 961 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
@@ -1044,6 +1079,7 @@ atombios_set_edp_panel_power(struct drm_connector *connector, int action)
1044 1079
1045union external_encoder_control { 1080union external_encoder_control {
1046 EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION v1; 1081 EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION v1;
1082 EXTERNAL_ENCODER_CONTROL_PS_ALLOCATION_V3 v3;
1047}; 1083};
1048 1084
1049static void 1085static void
@@ -1054,6 +1090,7 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
1054 struct drm_device *dev = encoder->dev; 1090 struct drm_device *dev = encoder->dev;
1055 struct radeon_device *rdev = dev->dev_private; 1091 struct radeon_device *rdev = dev->dev_private;
1056 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1092 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1093 struct radeon_encoder *ext_radeon_encoder = to_radeon_encoder(ext_encoder);
1057 union external_encoder_control args; 1094 union external_encoder_control args;
1058 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1095 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1059 int index = GetIndexIntoMasterTable(COMMAND, ExternalEncoderControl); 1096 int index = GetIndexIntoMasterTable(COMMAND, ExternalEncoderControl);
@@ -1061,6 +1098,7 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
1061 int dp_clock = 0; 1098 int dp_clock = 0;
1062 int dp_lane_count = 0; 1099 int dp_lane_count = 0;
1063 int connector_object_id = 0; 1100 int connector_object_id = 0;
1101 u32 ext_enum = (ext_radeon_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT;
1064 1102
1065 if (connector) { 1103 if (connector) {
1066 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1104 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -1099,6 +1137,37 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
1099 else 1137 else
1100 args.v1.sDigEncoder.ucLaneNum = 4; 1138 args.v1.sDigEncoder.ucLaneNum = 4;
1101 break; 1139 break;
1140 case 3:
1141 args.v3.sExtEncoder.ucAction = action;
1142 if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT)
1143 args.v3.sExtEncoder.usConnectorId = connector_object_id;
1144 else
1145 args.v3.sExtEncoder.usPixelClock = cpu_to_le16(radeon_encoder->pixel_clock / 10);
1146 args.v3.sExtEncoder.ucEncoderMode = atombios_get_encoder_mode(encoder);
1147
1148 if (args.v3.sExtEncoder.ucEncoderMode == ATOM_ENCODER_MODE_DP) {
1149 if (dp_clock == 270000)
1150 args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
1151 else if (dp_clock == 540000)
1152 args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_DPLINKRATE_5_40GHZ;
1153 args.v3.sExtEncoder.ucLaneNum = dp_lane_count;
1154 } else if (radeon_encoder->pixel_clock > 165000)
1155 args.v3.sExtEncoder.ucLaneNum = 8;
1156 else
1157 args.v3.sExtEncoder.ucLaneNum = 4;
1158 switch (ext_enum) {
1159 case GRAPH_OBJECT_ENUM_ID1:
1160 args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER1;
1161 break;
1162 case GRAPH_OBJECT_ENUM_ID2:
1163 args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER2;
1164 break;
1165 case GRAPH_OBJECT_ENUM_ID3:
1166 args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER3;
1167 break;
1168 }
1169 args.v3.sExtEncoder.ucBitPerColor = PANEL_8BIT_PER_COLOR;
1170 break;
1102 default: 1171 default:
1103 DRM_ERROR("Unknown table version: %d, %d\n", frev, crev); 1172 DRM_ERROR("Unknown table version: %d, %d\n", frev, crev);
1104 return; 1173 return;
@@ -1158,6 +1227,8 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
1158 DISPLAY_DEVICE_OUTPUT_CONTROL_PS_ALLOCATION args; 1227 DISPLAY_DEVICE_OUTPUT_CONTROL_PS_ALLOCATION args;
1159 int index = 0; 1228 int index = 0;
1160 bool is_dig = false; 1229 bool is_dig = false;
1230 bool is_dce5_dac = false;
1231 bool is_dce5_dvo = false;
1161 1232
1162 memset(&args, 0, sizeof(args)); 1233 memset(&args, 0, sizeof(args));
1163 1234
@@ -1180,7 +1251,9 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
1180 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl); 1251 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
1181 break; 1252 break;
1182 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1: 1253 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
1183 if (ASIC_IS_DCE3(rdev)) 1254 if (ASIC_IS_DCE5(rdev))
1255 is_dce5_dvo = true;
1256 else if (ASIC_IS_DCE3(rdev))
1184 is_dig = true; 1257 is_dig = true;
1185 else 1258 else
1186 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl); 1259 index = GetIndexIntoMasterTable(COMMAND, DVOOutputControl);
@@ -1196,12 +1269,16 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
1196 break; 1269 break;
1197 case ENCODER_OBJECT_ID_INTERNAL_DAC1: 1270 case ENCODER_OBJECT_ID_INTERNAL_DAC1:
1198 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1: 1271 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1:
1199 if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT)) 1272 if (ASIC_IS_DCE5(rdev))
1200 index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl); 1273 is_dce5_dac = true;
1201 else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT)) 1274 else {
1202 index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl); 1275 if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
1203 else 1276 index = GetIndexIntoMasterTable(COMMAND, TV1OutputControl);
1204 index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl); 1277 else if (radeon_encoder->active_device & (ATOM_DEVICE_CV_SUPPORT))
1278 index = GetIndexIntoMasterTable(COMMAND, CV1OutputControl);
1279 else
1280 index = GetIndexIntoMasterTable(COMMAND, DAC1OutputControl);
1281 }
1205 break; 1282 break;
1206 case ENCODER_OBJECT_ID_INTERNAL_DAC2: 1283 case ENCODER_OBJECT_ID_INTERNAL_DAC2:
1207 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2: 1284 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2:
@@ -1260,6 +1337,28 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
1260 atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0); 1337 atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
1261 break; 1338 break;
1262 } 1339 }
1340 } else if (is_dce5_dac) {
1341 switch (mode) {
1342 case DRM_MODE_DPMS_ON:
1343 atombios_dac_setup(encoder, ATOM_ENABLE);
1344 break;
1345 case DRM_MODE_DPMS_STANDBY:
1346 case DRM_MODE_DPMS_SUSPEND:
1347 case DRM_MODE_DPMS_OFF:
1348 atombios_dac_setup(encoder, ATOM_DISABLE);
1349 break;
1350 }
1351 } else if (is_dce5_dvo) {
1352 switch (mode) {
1353 case DRM_MODE_DPMS_ON:
1354 atombios_dvo_setup(encoder, ATOM_ENABLE);
1355 break;
1356 case DRM_MODE_DPMS_STANDBY:
1357 case DRM_MODE_DPMS_SUSPEND:
1358 case DRM_MODE_DPMS_OFF:
1359 atombios_dvo_setup(encoder, ATOM_DISABLE);
1360 break;
1361 }
1263 } else { 1362 } else {
1264 switch (mode) { 1363 switch (mode) {
1265 case DRM_MODE_DPMS_ON: 1364 case DRM_MODE_DPMS_ON:
@@ -1289,12 +1388,18 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
1289 switch (mode) { 1388 switch (mode) {
1290 case DRM_MODE_DPMS_ON: 1389 case DRM_MODE_DPMS_ON:
1291 default: 1390 default:
1292 action = ATOM_ENABLE; 1391 if (ASIC_IS_DCE41(rdev))
1392 action = EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT;
1393 else
1394 action = ATOM_ENABLE;
1293 break; 1395 break;
1294 case DRM_MODE_DPMS_STANDBY: 1396 case DRM_MODE_DPMS_STANDBY:
1295 case DRM_MODE_DPMS_SUSPEND: 1397 case DRM_MODE_DPMS_SUSPEND:
1296 case DRM_MODE_DPMS_OFF: 1398 case DRM_MODE_DPMS_OFF:
1297 action = ATOM_DISABLE; 1399 if (ASIC_IS_DCE41(rdev))
1400 action = EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT;
1401 else
1402 action = ATOM_DISABLE;
1298 break; 1403 break;
1299 } 1404 }
1300 atombios_external_encoder_setup(encoder, ext_encoder, action); 1405 atombios_external_encoder_setup(encoder, ext_encoder, action);
@@ -1483,27 +1588,35 @@ static int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder)
1483 struct radeon_encoder_atom_dig *dig; 1588 struct radeon_encoder_atom_dig *dig;
1484 uint32_t dig_enc_in_use = 0; 1589 uint32_t dig_enc_in_use = 0;
1485 1590
1591 /* DCE4/5 */
1486 if (ASIC_IS_DCE4(rdev)) { 1592 if (ASIC_IS_DCE4(rdev)) {
1487 dig = radeon_encoder->enc_priv; 1593 dig = radeon_encoder->enc_priv;
1488 switch (radeon_encoder->encoder_id) { 1594 if (ASIC_IS_DCE41(rdev)) {
1489 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
1490 if (dig->linkb) 1595 if (dig->linkb)
1491 return 1; 1596 return 1;
1492 else 1597 else
1493 return 0; 1598 return 0;
1494 break; 1599 } else {
1495 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1: 1600 switch (radeon_encoder->encoder_id) {
1496 if (dig->linkb) 1601 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
1497 return 3; 1602 if (dig->linkb)
1498 else 1603 return 1;
1499 return 2; 1604 else
1500 break; 1605 return 0;
1501 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2: 1606 break;
1502 if (dig->linkb) 1607 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
1503 return 5; 1608 if (dig->linkb)
1504 else 1609 return 3;
1505 return 4; 1610 else
1506 break; 1611 return 2;
1612 break;
1613 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
1614 if (dig->linkb)
1615 return 5;
1616 else
1617 return 4;
1618 break;
1619 }
1507 } 1620 }
1508 } 1621 }
1509 1622
@@ -1610,7 +1723,13 @@ radeon_atom_encoder_mode_set(struct drm_encoder *encoder,
1610 } 1723 }
1611 1724
1612 if (ext_encoder) { 1725 if (ext_encoder) {
1613 atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE); 1726 if (ASIC_IS_DCE41(rdev)) {
1727 atombios_external_encoder_setup(encoder, ext_encoder,
1728 EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT);
1729 atombios_external_encoder_setup(encoder, ext_encoder,
1730 EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP);
1731 } else
1732 atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
1614 } 1733 }
1615 1734
1616 atombios_apply_encoder_quirks(encoder, adjusted_mode); 1735 atombios_apply_encoder_quirks(encoder, adjusted_mode);
@@ -1927,7 +2046,10 @@ radeon_atombios_set_dig_info(struct radeon_encoder *radeon_encoder)
1927} 2046}
1928 2047
1929void 2048void
1930radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t supported_device) 2049radeon_add_atom_encoder(struct drm_device *dev,
2050 uint32_t encoder_enum,
2051 uint32_t supported_device,
2052 u16 caps)
1931{ 2053{
1932 struct radeon_device *rdev = dev->dev_private; 2054 struct radeon_device *rdev = dev->dev_private;
1933 struct drm_encoder *encoder; 2055 struct drm_encoder *encoder;
@@ -1970,6 +2092,7 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
1970 radeon_encoder->rmx_type = RMX_OFF; 2092 radeon_encoder->rmx_type = RMX_OFF;
1971 radeon_encoder->underscan_type = UNDERSCAN_OFF; 2093 radeon_encoder->underscan_type = UNDERSCAN_OFF;
1972 radeon_encoder->is_ext_encoder = false; 2094 radeon_encoder->is_ext_encoder = false;
2095 radeon_encoder->caps = caps;
1973 2096
1974 switch (radeon_encoder->encoder_id) { 2097 switch (radeon_encoder->encoder_id) {
1975 case ENCODER_OBJECT_ID_INTERNAL_LVDS: 2098 case ENCODER_OBJECT_ID_INTERNAL_LVDS:
@@ -2029,6 +2152,8 @@ radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum, uint32_t
2029 case ENCODER_OBJECT_ID_TITFP513: 2152 case ENCODER_OBJECT_ID_TITFP513:
2030 case ENCODER_OBJECT_ID_VT1623: 2153 case ENCODER_OBJECT_ID_VT1623:
2031 case ENCODER_OBJECT_ID_HDMI_SI1930: 2154 case ENCODER_OBJECT_ID_HDMI_SI1930:
2155 case ENCODER_OBJECT_ID_TRAVIS:
2156 case ENCODER_OBJECT_ID_NUTMEG:
2032 /* these are handled by the primary encoders */ 2157 /* these are handled by the primary encoders */
2033 radeon_encoder->is_ext_encoder = true; 2158 radeon_encoder->is_ext_encoder = true;
2034 if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) 2159 if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
diff --git a/drivers/gpu/drm/radeon/radeon_family.h b/drivers/gpu/drm/radeon/radeon_family.h
index e329066dcabd..1ca55eb09ad3 100644
--- a/drivers/gpu/drm/radeon/radeon_family.h
+++ b/drivers/gpu/drm/radeon/radeon_family.h
@@ -80,6 +80,10 @@ enum radeon_family {
80 CHIP_JUNIPER, 80 CHIP_JUNIPER,
81 CHIP_CYPRESS, 81 CHIP_CYPRESS,
82 CHIP_HEMLOCK, 82 CHIP_HEMLOCK,
83 CHIP_PALM,
84 CHIP_BARTS,
85 CHIP_TURKS,
86 CHIP_CAICOS,
83 CHIP_LAST, 87 CHIP_LAST,
84}; 88};
85 89
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 6abea32be5e8..ca32e9c1e91d 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -225,8 +225,6 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
225 225
226 strcpy(info->fix.id, "radeondrmfb"); 226 strcpy(info->fix.id, "radeondrmfb");
227 227
228 drm_fb_helper_fill_fix(info, fb->pitch, fb->depth);
229
230 info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; 228 info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
231 info->fbops = &radeonfb_ops; 229 info->fbops = &radeonfb_ops;
232 230
@@ -247,8 +245,6 @@ static int radeonfb_create(struct radeon_fbdev *rfbdev,
247 info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base; 245 info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base;
248 info->apertures->ranges[0].size = rdev->mc.aper_size; 246 info->apertures->ranges[0].size = rdev->mc.aper_size;
249 247
250 info->fix.mmio_start = 0;
251 info->fix.mmio_len = 0;
252 info->pixmap.size = 64*1024; 248 info->pixmap.size = 64*1024;
253 info->pixmap.buf_align = 8; 249 info->pixmap.buf_align = 8;
254 info->pixmap.access_align = 32; 250 info->pixmap.access_align = 32;
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index daacb281dfaf..171b0b2e3a64 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -38,6 +38,7 @@
38#include "drm.h" 38#include "drm.h"
39#include "radeon_reg.h" 39#include "radeon_reg.h"
40#include "radeon.h" 40#include "radeon.h"
41#include "radeon_trace.h"
41 42
42int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) 43int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
43{ 44{
@@ -57,6 +58,7 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
57 } else 58 } else
58 radeon_fence_ring_emit(rdev, fence); 59 radeon_fence_ring_emit(rdev, fence);
59 60
61 trace_radeon_fence_emit(rdev->ddev, fence->seq);
60 fence->emited = true; 62 fence->emited = true;
61 list_del(&fence->list); 63 list_del(&fence->list);
62 list_add_tail(&fence->list, &rdev->fence_drv.emited); 64 list_add_tail(&fence->list, &rdev->fence_drv.emited);
@@ -213,6 +215,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
213retry: 215retry:
214 /* save current sequence used to check for GPU lockup */ 216 /* save current sequence used to check for GPU lockup */
215 seq = rdev->fence_drv.last_seq; 217 seq = rdev->fence_drv.last_seq;
218 trace_radeon_fence_wait_begin(rdev->ddev, seq);
216 if (intr) { 219 if (intr) {
217 radeon_irq_kms_sw_irq_get(rdev); 220 radeon_irq_kms_sw_irq_get(rdev);
218 r = wait_event_interruptible_timeout(rdev->fence_drv.queue, 221 r = wait_event_interruptible_timeout(rdev->fence_drv.queue,
@@ -227,6 +230,7 @@ retry:
227 radeon_fence_signaled(fence), timeout); 230 radeon_fence_signaled(fence), timeout);
228 radeon_irq_kms_sw_irq_put(rdev); 231 radeon_irq_kms_sw_irq_put(rdev);
229 } 232 }
233 trace_radeon_fence_wait_end(rdev->ddev, seq);
230 if (unlikely(!radeon_fence_signaled(fence))) { 234 if (unlikely(!radeon_fence_signaled(fence))) {
231 /* we were interrupted for some reason and fence isn't 235 /* we were interrupted for some reason and fence isn't
232 * isn't signaled yet, resume wait 236 * isn't signaled yet, resume wait
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index a108c7ed14f5..a289646e8aa4 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -64,15 +64,15 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
64 struct radeon_device *rdev = dev->dev_private; 64 struct radeon_device *rdev = dev->dev_private;
65 unsigned i; 65 unsigned i;
66 66
67 INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
68
69 /* Disable *all* interrupts */ 67 /* Disable *all* interrupts */
70 rdev->irq.sw_int = false; 68 rdev->irq.sw_int = false;
71 rdev->irq.gui_idle = false; 69 rdev->irq.gui_idle = false;
72 for (i = 0; i < rdev->num_crtc; i++) 70 for (i = 0; i < rdev->num_crtc; i++)
73 rdev->irq.crtc_vblank_int[i] = false; 71 rdev->irq.crtc_vblank_int[i] = false;
74 for (i = 0; i < 6; i++) 72 for (i = 0; i < 6; i++) {
75 rdev->irq.hpd[i] = false; 73 rdev->irq.hpd[i] = false;
74 rdev->irq.pflip[i] = false;
75 }
76 radeon_irq_set(rdev); 76 radeon_irq_set(rdev);
77 /* Clear bits */ 77 /* Clear bits */
78 radeon_irq_process(rdev); 78 radeon_irq_process(rdev);
@@ -101,8 +101,10 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
101 rdev->irq.gui_idle = false; 101 rdev->irq.gui_idle = false;
102 for (i = 0; i < rdev->num_crtc; i++) 102 for (i = 0; i < rdev->num_crtc; i++)
103 rdev->irq.crtc_vblank_int[i] = false; 103 rdev->irq.crtc_vblank_int[i] = false;
104 for (i = 0; i < 6; i++) 104 for (i = 0; i < 6; i++) {
105 rdev->irq.hpd[i] = false; 105 rdev->irq.hpd[i] = false;
106 rdev->irq.pflip[i] = false;
107 }
106 radeon_irq_set(rdev); 108 radeon_irq_set(rdev);
107} 109}
108 110
@@ -110,6 +112,8 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
110{ 112{
111 int r = 0; 113 int r = 0;
112 114
115 INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func);
116
113 spin_lock_init(&rdev->irq.sw_lock); 117 spin_lock_init(&rdev->irq.sw_lock);
114 r = drm_vblank_init(rdev->ddev, rdev->num_crtc); 118 r = drm_vblank_init(rdev->ddev, rdev->num_crtc);
115 if (r) { 119 if (r) {
@@ -121,7 +125,7 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
121 * chips. Disable MSI on them for now. 125 * chips. Disable MSI on them for now.
122 */ 126 */
123 if ((rdev->family >= CHIP_RV380) && 127 if ((rdev->family >= CHIP_RV380) &&
124 (!(rdev->flags & RADEON_IS_IGP)) && 128 ((!(rdev->flags & RADEON_IS_IGP)) || (rdev->family >= CHIP_PALM)) &&
125 (!(rdev->flags & RADEON_IS_AGP))) { 129 (!(rdev->flags & RADEON_IS_AGP))) {
126 int ret = pci_enable_msi(rdev->pdev); 130 int ret = pci_enable_msi(rdev->pdev);
127 if (!ret) { 131 if (!ret) {
@@ -148,6 +152,7 @@ void radeon_irq_kms_fini(struct radeon_device *rdev)
148 if (rdev->msi_enabled) 152 if (rdev->msi_enabled)
149 pci_disable_msi(rdev->pdev); 153 pci_disable_msi(rdev->pdev);
150 } 154 }
155 flush_work_sync(&rdev->hotplug_work);
151} 156}
152 157
153void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev) 158void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev)
@@ -175,3 +180,34 @@ void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev)
175 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags); 180 spin_unlock_irqrestore(&rdev->irq.sw_lock, irqflags);
176} 181}
177 182
183void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc)
184{
185 unsigned long irqflags;
186
187 if (crtc < 0 || crtc >= rdev->num_crtc)
188 return;
189
190 spin_lock_irqsave(&rdev->irq.pflip_lock[crtc], irqflags);
191 if (rdev->ddev->irq_enabled && (++rdev->irq.pflip_refcount[crtc] == 1)) {
192 rdev->irq.pflip[crtc] = true;
193 radeon_irq_set(rdev);
194 }
195 spin_unlock_irqrestore(&rdev->irq.pflip_lock[crtc], irqflags);
196}
197
198void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc)
199{
200 unsigned long irqflags;
201
202 if (crtc < 0 || crtc >= rdev->num_crtc)
203 return;
204
205 spin_lock_irqsave(&rdev->irq.pflip_lock[crtc], irqflags);
206 BUG_ON(rdev->ddev->irq_enabled && rdev->irq.pflip_refcount[crtc] <= 0);
207 if (rdev->ddev->irq_enabled && (--rdev->irq.pflip_refcount[crtc] == 0)) {
208 rdev->irq.pflip[crtc] = false;
209 radeon_irq_set(rdev);
210 }
211 spin_unlock_irqrestore(&rdev->irq.pflip_lock[crtc], irqflags);
212}
213
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 8fbbe1c6ebbd..28a53e4a925f 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -96,9 +96,27 @@ out:
96 return r; 96 return r;
97} 97}
98 98
99static void radeon_set_filp_rights(struct drm_device *dev,
100 struct drm_file **owner,
101 struct drm_file *applier,
102 uint32_t *value)
103{
104 mutex_lock(&dev->struct_mutex);
105 if (*value == 1) {
106 /* wants rights */
107 if (!*owner)
108 *owner = applier;
109 } else if (*value == 0) {
110 /* revokes rights */
111 if (*owner == applier)
112 *owner = NULL;
113 }
114 *value = *owner == applier ? 1 : 0;
115 mutex_unlock(&dev->struct_mutex);
116}
99 117
100/* 118/*
101 * Userspace get informations ioctl 119 * Userspace get information ioctl
102 */ 120 */
103int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 121int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
104{ 122{
@@ -173,18 +191,15 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
173 DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", value); 191 DRM_DEBUG_KMS("WANT_HYPERZ: invalid value %d\n", value);
174 return -EINVAL; 192 return -EINVAL;
175 } 193 }
176 mutex_lock(&dev->struct_mutex); 194 radeon_set_filp_rights(dev, &rdev->hyperz_filp, filp, &value);
177 if (value == 1) { 195 break;
178 /* wants hyper-z */ 196 case RADEON_INFO_WANT_CMASK:
179 if (!rdev->hyperz_filp) 197 /* The same logic as Hyper-Z. */
180 rdev->hyperz_filp = filp; 198 if (value >= 2) {
181 } else if (value == 0) { 199 DRM_DEBUG_KMS("WANT_CMASK: invalid value %d\n", value);
182 /* revokes hyper-z */ 200 return -EINVAL;
183 if (rdev->hyperz_filp == filp)
184 rdev->hyperz_filp = NULL;
185 } 201 }
186 value = rdev->hyperz_filp == filp ? 1 : 0; 202 radeon_set_filp_rights(dev, &rdev->cmask_filp, filp, &value);
187 mutex_unlock(&dev->struct_mutex);
188 break; 203 break;
189 default: 204 default:
190 DRM_DEBUG_KMS("Invalid request %d\n", info->request); 205 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
@@ -203,10 +218,6 @@ int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
203 */ 218 */
204int radeon_driver_firstopen_kms(struct drm_device *dev) 219int radeon_driver_firstopen_kms(struct drm_device *dev)
205{ 220{
206 struct radeon_device *rdev = dev->dev_private;
207
208 if (rdev->powered_down)
209 return -EINVAL;
210 return 0; 221 return 0;
211} 222}
212 223
@@ -277,6 +288,27 @@ void radeon_disable_vblank_kms(struct drm_device *dev, int crtc)
277 radeon_irq_set(rdev); 288 radeon_irq_set(rdev);
278} 289}
279 290
291int radeon_get_vblank_timestamp_kms(struct drm_device *dev, int crtc,
292 int *max_error,
293 struct timeval *vblank_time,
294 unsigned flags)
295{
296 struct drm_crtc *drmcrtc;
297 struct radeon_device *rdev = dev->dev_private;
298
299 if (crtc < 0 || crtc >= dev->num_crtcs) {
300 DRM_ERROR("Invalid crtc %d\n", crtc);
301 return -EINVAL;
302 }
303
304 /* Get associated drm_crtc: */
305 drmcrtc = &rdev->mode_info.crtcs[crtc]->base;
306
307 /* Helper routine in DRM core does all the work: */
308 return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc, max_error,
309 vblank_time, flags,
310 drmcrtc);
311}
280 312
281/* 313/*
282 * IOCTL. 314 * IOCTL.
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index e301c6f9e059..12bdeab91c86 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -277,6 +277,9 @@ struct radeon_crtc {
277 fixed20_12 hsc; 277 fixed20_12 hsc;
278 struct drm_display_mode native_mode; 278 struct drm_display_mode native_mode;
279 int pll_id; 279 int pll_id;
280 /* page flipping */
281 struct radeon_unpin_work *unpin_work;
282 int deferred_flip_completion;
280}; 283};
281 284
282struct radeon_encoder_primary_dac { 285struct radeon_encoder_primary_dac {
@@ -376,6 +379,7 @@ struct radeon_encoder {
376 int hdmi_audio_workaround; 379 int hdmi_audio_workaround;
377 int hdmi_buffer_status; 380 int hdmi_buffer_status;
378 bool is_ext_encoder; 381 bool is_ext_encoder;
382 u16 caps;
379}; 383};
380 384
381struct radeon_connector_atom_dig { 385struct radeon_connector_atom_dig {
@@ -442,10 +446,6 @@ struct radeon_framebuffer {
442 struct drm_gem_object *obj; 446 struct drm_gem_object *obj;
443}; 447};
444 448
445/* radeon_get_crtc_scanoutpos() return flags */
446#define RADEON_SCANOUTPOS_VALID (1 << 0)
447#define RADEON_SCANOUTPOS_INVBL (1 << 1)
448#define RADEON_SCANOUTPOS_ACCURATE (1 << 2)
449 449
450extern enum radeon_tv_std 450extern enum radeon_tv_std
451radeon_combios_get_tv_info(struct radeon_device *rdev); 451radeon_combios_get_tv_info(struct radeon_device *rdev);
@@ -562,11 +562,12 @@ extern int radeon_crtc_cursor_set(struct drm_crtc *crtc,
562extern int radeon_crtc_cursor_move(struct drm_crtc *crtc, 562extern int radeon_crtc_cursor_move(struct drm_crtc *crtc,
563 int x, int y); 563 int x, int y);
564 564
565extern int radeon_get_crtc_scanoutpos(struct radeon_device *rdev, int crtc, int *vpos, int *hpos); 565extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
566 int *vpos, int *hpos);
566 567
567extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev); 568extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev);
568extern struct edid * 569extern struct edid *
569radeon_combios_get_hardcoded_edid(struct radeon_device *rdev); 570radeon_bios_get_hardcoded_edid(struct radeon_device *rdev);
570extern bool radeon_atom_get_clock_info(struct drm_device *dev); 571extern bool radeon_atom_get_clock_info(struct drm_device *dev);
571extern bool radeon_combios_get_clock_info(struct drm_device *dev); 572extern bool radeon_combios_get_clock_info(struct drm_device *dev);
572extern struct radeon_encoder_atom_dig * 573extern struct radeon_encoder_atom_dig *
@@ -662,4 +663,7 @@ int radeon_fbdev_total_size(struct radeon_device *rdev);
662bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj); 663bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
663 664
664void radeon_fb_output_poll_changed(struct radeon_device *rdev); 665void radeon_fb_output_poll_changed(struct radeon_device *rdev);
666
667void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id);
668
665#endif 669#endif
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index a598d0049aa5..7d6b8e88f746 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -34,6 +34,7 @@
34#include <drm/drmP.h> 34#include <drm/drmP.h>
35#include "radeon_drm.h" 35#include "radeon_drm.h"
36#include "radeon.h" 36#include "radeon.h"
37#include "radeon_trace.h"
37 38
38 39
39int radeon_ttm_init(struct radeon_device *rdev); 40int radeon_ttm_init(struct radeon_device *rdev);
@@ -146,6 +147,7 @@ retry:
146 list_add_tail(&bo->list, &rdev->gem.objects); 147 list_add_tail(&bo->list, &rdev->gem.objects);
147 mutex_unlock(&bo->rdev->gem.mutex); 148 mutex_unlock(&bo->rdev->gem.mutex);
148 } 149 }
150 trace_radeon_bo_create(bo);
149 return 0; 151 return 0;
150} 152}
151 153
@@ -302,34 +304,9 @@ void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
302 struct list_head *head) 304 struct list_head *head)
303{ 305{
304 if (lobj->wdomain) { 306 if (lobj->wdomain) {
305 list_add(&lobj->list, head); 307 list_add(&lobj->tv.head, head);
306 } else { 308 } else {
307 list_add_tail(&lobj->list, head); 309 list_add_tail(&lobj->tv.head, head);
308 }
309}
310
311int radeon_bo_list_reserve(struct list_head *head)
312{
313 struct radeon_bo_list *lobj;
314 int r;
315
316 list_for_each_entry(lobj, head, list){
317 r = radeon_bo_reserve(lobj->bo, false);
318 if (unlikely(r != 0))
319 return r;
320 lobj->reserved = true;
321 }
322 return 0;
323}
324
325void radeon_bo_list_unreserve(struct list_head *head)
326{
327 struct radeon_bo_list *lobj;
328
329 list_for_each_entry(lobj, head, list) {
330 /* only unreserve object we successfully reserved */
331 if (lobj->reserved && radeon_bo_is_reserved(lobj->bo))
332 radeon_bo_unreserve(lobj->bo);
333 } 310 }
334} 311}
335 312
@@ -340,14 +317,11 @@ int radeon_bo_list_validate(struct list_head *head)
340 u32 domain; 317 u32 domain;
341 int r; 318 int r;
342 319
343 list_for_each_entry(lobj, head, list) { 320 r = ttm_eu_reserve_buffers(head);
344 lobj->reserved = false;
345 }
346 r = radeon_bo_list_reserve(head);
347 if (unlikely(r != 0)) { 321 if (unlikely(r != 0)) {
348 return r; 322 return r;
349 } 323 }
350 list_for_each_entry(lobj, head, list) { 324 list_for_each_entry(lobj, head, tv.head) {
351 bo = lobj->bo; 325 bo = lobj->bo;
352 if (!bo->pin_count) { 326 if (!bo->pin_count) {
353 domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain; 327 domain = lobj->wdomain ? lobj->wdomain : lobj->rdomain;
@@ -370,25 +344,6 @@ int radeon_bo_list_validate(struct list_head *head)
370 return 0; 344 return 0;
371} 345}
372 346
373void radeon_bo_list_fence(struct list_head *head, void *fence)
374{
375 struct radeon_bo_list *lobj;
376 struct radeon_bo *bo;
377 struct radeon_fence *old_fence = NULL;
378
379 list_for_each_entry(lobj, head, list) {
380 bo = lobj->bo;
381 spin_lock(&bo->tbo.lock);
382 old_fence = (struct radeon_fence *)bo->tbo.sync_obj;
383 bo->tbo.sync_obj = radeon_fence_ref(fence);
384 bo->tbo.sync_obj_arg = NULL;
385 spin_unlock(&bo->tbo.lock);
386 if (old_fence) {
387 radeon_fence_unref(&old_fence);
388 }
389 }
390}
391
392int radeon_bo_fbdev_mmap(struct radeon_bo *bo, 347int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
393 struct vm_area_struct *vma) 348 struct vm_area_struct *vma)
394{ 349{
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index d143702b244a..22d4c237dea5 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -126,12 +126,12 @@ static inline int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type,
126 r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0); 126 r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0);
127 if (unlikely(r != 0)) 127 if (unlikely(r != 0))
128 return r; 128 return r;
129 spin_lock(&bo->tbo.lock); 129 spin_lock(&bo->tbo.bdev->fence_lock);
130 if (mem_type) 130 if (mem_type)
131 *mem_type = bo->tbo.mem.mem_type; 131 *mem_type = bo->tbo.mem.mem_type;
132 if (bo->tbo.sync_obj) 132 if (bo->tbo.sync_obj)
133 r = ttm_bo_wait(&bo->tbo, true, true, no_wait); 133 r = ttm_bo_wait(&bo->tbo, true, true, no_wait);
134 spin_unlock(&bo->tbo.lock); 134 spin_unlock(&bo->tbo.bdev->fence_lock);
135 ttm_bo_unreserve(&bo->tbo); 135 ttm_bo_unreserve(&bo->tbo);
136 return r; 136 return r;
137} 137}
@@ -152,10 +152,7 @@ extern int radeon_bo_init(struct radeon_device *rdev);
152extern void radeon_bo_fini(struct radeon_device *rdev); 152extern void radeon_bo_fini(struct radeon_device *rdev);
153extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj, 153extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
154 struct list_head *head); 154 struct list_head *head);
155extern int radeon_bo_list_reserve(struct list_head *head);
156extern void radeon_bo_list_unreserve(struct list_head *head);
157extern int radeon_bo_list_validate(struct list_head *head); 155extern int radeon_bo_list_validate(struct list_head *head);
158extern void radeon_bo_list_fence(struct list_head *head, void *fence);
159extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo, 156extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
160 struct vm_area_struct *vma); 157 struct vm_area_struct *vma);
161extern int radeon_bo_set_tiling_flags(struct radeon_bo *bo, 158extern int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 8c9b2ef32c68..3b1b2bf9cdd5 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -167,13 +167,13 @@ static void radeon_set_power_state(struct radeon_device *rdev)
167 if (radeon_gui_idle(rdev)) { 167 if (radeon_gui_idle(rdev)) {
168 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index]. 168 sclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
169 clock_info[rdev->pm.requested_clock_mode_index].sclk; 169 clock_info[rdev->pm.requested_clock_mode_index].sclk;
170 if (sclk > rdev->clock.default_sclk) 170 if (sclk > rdev->pm.default_sclk)
171 sclk = rdev->clock.default_sclk; 171 sclk = rdev->pm.default_sclk;
172 172
173 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index]. 173 mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
174 clock_info[rdev->pm.requested_clock_mode_index].mclk; 174 clock_info[rdev->pm.requested_clock_mode_index].mclk;
175 if (mclk > rdev->clock.default_mclk) 175 if (mclk > rdev->pm.default_mclk)
176 mclk = rdev->clock.default_mclk; 176 mclk = rdev->pm.default_mclk;
177 177
178 /* upvolt before raising clocks, downvolt after lowering clocks */ 178 /* upvolt before raising clocks, downvolt after lowering clocks */
179 if (sclk < rdev->pm.current_sclk) 179 if (sclk < rdev->pm.current_sclk)
@@ -405,20 +405,13 @@ static ssize_t radeon_set_pm_method(struct device *dev,
405 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT; 405 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
406 mutex_unlock(&rdev->pm.mutex); 406 mutex_unlock(&rdev->pm.mutex);
407 } else if (strncmp("profile", buf, strlen("profile")) == 0) { 407 } else if (strncmp("profile", buf, strlen("profile")) == 0) {
408 bool flush_wq = false;
409
410 mutex_lock(&rdev->pm.mutex); 408 mutex_lock(&rdev->pm.mutex);
411 if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
412 cancel_delayed_work(&rdev->pm.dynpm_idle_work);
413 flush_wq = true;
414 }
415 /* disable dynpm */ 409 /* disable dynpm */
416 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED; 410 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
417 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE; 411 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
418 rdev->pm.pm_method = PM_METHOD_PROFILE; 412 rdev->pm.pm_method = PM_METHOD_PROFILE;
419 mutex_unlock(&rdev->pm.mutex); 413 mutex_unlock(&rdev->pm.mutex);
420 if (flush_wq) 414 cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
421 flush_workqueue(rdev->wq);
422 } else { 415 } else {
423 DRM_ERROR("invalid power method!\n"); 416 DRM_ERROR("invalid power method!\n");
424 goto fail; 417 goto fail;
@@ -447,8 +440,12 @@ static ssize_t radeon_hwmon_show_temp(struct device *dev,
447 temp = rv770_get_temp(rdev); 440 temp = rv770_get_temp(rdev);
448 break; 441 break;
449 case THERMAL_TYPE_EVERGREEN: 442 case THERMAL_TYPE_EVERGREEN:
443 case THERMAL_TYPE_NI:
450 temp = evergreen_get_temp(rdev); 444 temp = evergreen_get_temp(rdev);
451 break; 445 break;
446 case THERMAL_TYPE_SUMO:
447 temp = sumo_get_temp(rdev);
448 break;
452 default: 449 default:
453 temp = 0; 450 temp = 0;
454 break; 451 break;
@@ -487,6 +484,7 @@ static int radeon_hwmon_init(struct radeon_device *rdev)
487 case THERMAL_TYPE_RV6XX: 484 case THERMAL_TYPE_RV6XX:
488 case THERMAL_TYPE_RV770: 485 case THERMAL_TYPE_RV770:
489 case THERMAL_TYPE_EVERGREEN: 486 case THERMAL_TYPE_EVERGREEN:
487 case THERMAL_TYPE_SUMO:
490 rdev->pm.int_hwmon_dev = hwmon_device_register(rdev->dev); 488 rdev->pm.int_hwmon_dev = hwmon_device_register(rdev->dev);
491 if (IS_ERR(rdev->pm.int_hwmon_dev)) { 489 if (IS_ERR(rdev->pm.int_hwmon_dev)) {
492 err = PTR_ERR(rdev->pm.int_hwmon_dev); 490 err = PTR_ERR(rdev->pm.int_hwmon_dev);
@@ -520,34 +518,39 @@ static void radeon_hwmon_fini(struct radeon_device *rdev)
520 518
521void radeon_pm_suspend(struct radeon_device *rdev) 519void radeon_pm_suspend(struct radeon_device *rdev)
522{ 520{
523 bool flush_wq = false;
524
525 mutex_lock(&rdev->pm.mutex); 521 mutex_lock(&rdev->pm.mutex);
526 if (rdev->pm.pm_method == PM_METHOD_DYNPM) { 522 if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
527 cancel_delayed_work(&rdev->pm.dynpm_idle_work);
528 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE) 523 if (rdev->pm.dynpm_state == DYNPM_STATE_ACTIVE)
529 rdev->pm.dynpm_state = DYNPM_STATE_SUSPENDED; 524 rdev->pm.dynpm_state = DYNPM_STATE_SUSPENDED;
530 flush_wq = true;
531 } 525 }
532 mutex_unlock(&rdev->pm.mutex); 526 mutex_unlock(&rdev->pm.mutex);
533 if (flush_wq) 527
534 flush_workqueue(rdev->wq); 528 cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
535} 529}
536 530
537void radeon_pm_resume(struct radeon_device *rdev) 531void radeon_pm_resume(struct radeon_device *rdev)
538{ 532{
533 /* set up the default clocks if the MC ucode is loaded */
534 if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
535 if (rdev->pm.default_vddc)
536 radeon_atom_set_voltage(rdev, rdev->pm.default_vddc);
537 if (rdev->pm.default_sclk)
538 radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
539 if (rdev->pm.default_mclk)
540 radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
541 }
539 /* asic init will reset the default power state */ 542 /* asic init will reset the default power state */
540 mutex_lock(&rdev->pm.mutex); 543 mutex_lock(&rdev->pm.mutex);
541 rdev->pm.current_power_state_index = rdev->pm.default_power_state_index; 544 rdev->pm.current_power_state_index = rdev->pm.default_power_state_index;
542 rdev->pm.current_clock_mode_index = 0; 545 rdev->pm.current_clock_mode_index = 0;
543 rdev->pm.current_sclk = rdev->clock.default_sclk; 546 rdev->pm.current_sclk = rdev->pm.default_sclk;
544 rdev->pm.current_mclk = rdev->clock.default_mclk; 547 rdev->pm.current_mclk = rdev->pm.default_mclk;
545 rdev->pm.current_vddc = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage; 548 rdev->pm.current_vddc = rdev->pm.power_state[rdev->pm.default_power_state_index].clock_info[0].voltage.voltage;
546 if (rdev->pm.pm_method == PM_METHOD_DYNPM 549 if (rdev->pm.pm_method == PM_METHOD_DYNPM
547 && rdev->pm.dynpm_state == DYNPM_STATE_SUSPENDED) { 550 && rdev->pm.dynpm_state == DYNPM_STATE_SUSPENDED) {
548 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE; 551 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
549 queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work, 552 schedule_delayed_work(&rdev->pm.dynpm_idle_work,
550 msecs_to_jiffies(RADEON_IDLE_LOOP_MS)); 553 msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
551 } 554 }
552 mutex_unlock(&rdev->pm.mutex); 555 mutex_unlock(&rdev->pm.mutex);
553 radeon_pm_compute_clocks(rdev); 556 radeon_pm_compute_clocks(rdev);
@@ -564,6 +567,8 @@ int radeon_pm_init(struct radeon_device *rdev)
564 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE; 567 rdev->pm.dynpm_planned_action = DYNPM_ACTION_NONE;
565 rdev->pm.dynpm_can_upclock = true; 568 rdev->pm.dynpm_can_upclock = true;
566 rdev->pm.dynpm_can_downclock = true; 569 rdev->pm.dynpm_can_downclock = true;
570 rdev->pm.default_sclk = rdev->clock.default_sclk;
571 rdev->pm.default_mclk = rdev->clock.default_mclk;
567 rdev->pm.current_sclk = rdev->clock.default_sclk; 572 rdev->pm.current_sclk = rdev->clock.default_sclk;
568 rdev->pm.current_mclk = rdev->clock.default_mclk; 573 rdev->pm.current_mclk = rdev->clock.default_mclk;
569 rdev->pm.int_thermal_type = THERMAL_TYPE_NONE; 574 rdev->pm.int_thermal_type = THERMAL_TYPE_NONE;
@@ -575,12 +580,24 @@ int radeon_pm_init(struct radeon_device *rdev)
575 radeon_combios_get_power_modes(rdev); 580 radeon_combios_get_power_modes(rdev);
576 radeon_pm_print_states(rdev); 581 radeon_pm_print_states(rdev);
577 radeon_pm_init_profile(rdev); 582 radeon_pm_init_profile(rdev);
583 /* set up the default clocks if the MC ucode is loaded */
584 if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
585 if (rdev->pm.default_vddc)
586 radeon_atom_set_voltage(rdev, rdev->pm.default_vddc);
587 if (rdev->pm.default_sclk)
588 radeon_set_engine_clock(rdev, rdev->pm.default_sclk);
589 if (rdev->pm.default_mclk)
590 radeon_set_memory_clock(rdev, rdev->pm.default_mclk);
591 }
578 } 592 }
579 593
580 /* set up the internal thermal sensor if applicable */ 594 /* set up the internal thermal sensor if applicable */
581 ret = radeon_hwmon_init(rdev); 595 ret = radeon_hwmon_init(rdev);
582 if (ret) 596 if (ret)
583 return ret; 597 return ret;
598
599 INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
600
584 if (rdev->pm.num_power_states > 1) { 601 if (rdev->pm.num_power_states > 1) {
585 /* where's the best place to put these? */ 602 /* where's the best place to put these? */
586 ret = device_create_file(rdev->dev, &dev_attr_power_profile); 603 ret = device_create_file(rdev->dev, &dev_attr_power_profile);
@@ -594,8 +611,6 @@ int radeon_pm_init(struct radeon_device *rdev)
594 rdev->acpi_nb.notifier_call = radeon_acpi_event; 611 rdev->acpi_nb.notifier_call = radeon_acpi_event;
595 register_acpi_notifier(&rdev->acpi_nb); 612 register_acpi_notifier(&rdev->acpi_nb);
596#endif 613#endif
597 INIT_DELAYED_WORK(&rdev->pm.dynpm_idle_work, radeon_dynpm_idle_work_handler);
598
599 if (radeon_debugfs_pm_init(rdev)) { 614 if (radeon_debugfs_pm_init(rdev)) {
600 DRM_ERROR("Failed to register debugfs file for PM!\n"); 615 DRM_ERROR("Failed to register debugfs file for PM!\n");
601 } 616 }
@@ -609,25 +624,20 @@ int radeon_pm_init(struct radeon_device *rdev)
609void radeon_pm_fini(struct radeon_device *rdev) 624void radeon_pm_fini(struct radeon_device *rdev)
610{ 625{
611 if (rdev->pm.num_power_states > 1) { 626 if (rdev->pm.num_power_states > 1) {
612 bool flush_wq = false;
613
614 mutex_lock(&rdev->pm.mutex); 627 mutex_lock(&rdev->pm.mutex);
615 if (rdev->pm.pm_method == PM_METHOD_PROFILE) { 628 if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
616 rdev->pm.profile = PM_PROFILE_DEFAULT; 629 rdev->pm.profile = PM_PROFILE_DEFAULT;
617 radeon_pm_update_profile(rdev); 630 radeon_pm_update_profile(rdev);
618 radeon_pm_set_clocks(rdev); 631 radeon_pm_set_clocks(rdev);
619 } else if (rdev->pm.pm_method == PM_METHOD_DYNPM) { 632 } else if (rdev->pm.pm_method == PM_METHOD_DYNPM) {
620 /* cancel work */
621 cancel_delayed_work(&rdev->pm.dynpm_idle_work);
622 flush_wq = true;
623 /* reset default clocks */ 633 /* reset default clocks */
624 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED; 634 rdev->pm.dynpm_state = DYNPM_STATE_DISABLED;
625 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT; 635 rdev->pm.dynpm_planned_action = DYNPM_ACTION_DEFAULT;
626 radeon_pm_set_clocks(rdev); 636 radeon_pm_set_clocks(rdev);
627 } 637 }
628 mutex_unlock(&rdev->pm.mutex); 638 mutex_unlock(&rdev->pm.mutex);
629 if (flush_wq) 639
630 flush_workqueue(rdev->wq); 640 cancel_delayed_work_sync(&rdev->pm.dynpm_idle_work);
631 641
632 device_remove_file(rdev->dev, &dev_attr_power_profile); 642 device_remove_file(rdev->dev, &dev_attr_power_profile);
633 device_remove_file(rdev->dev, &dev_attr_power_method); 643 device_remove_file(rdev->dev, &dev_attr_power_method);
@@ -686,12 +696,12 @@ void radeon_pm_compute_clocks(struct radeon_device *rdev)
686 radeon_pm_get_dynpm_state(rdev); 696 radeon_pm_get_dynpm_state(rdev);
687 radeon_pm_set_clocks(rdev); 697 radeon_pm_set_clocks(rdev);
688 698
689 queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work, 699 schedule_delayed_work(&rdev->pm.dynpm_idle_work,
690 msecs_to_jiffies(RADEON_IDLE_LOOP_MS)); 700 msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
691 } else if (rdev->pm.dynpm_state == DYNPM_STATE_PAUSED) { 701 } else if (rdev->pm.dynpm_state == DYNPM_STATE_PAUSED) {
692 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE; 702 rdev->pm.dynpm_state = DYNPM_STATE_ACTIVE;
693 queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work, 703 schedule_delayed_work(&rdev->pm.dynpm_idle_work,
694 msecs_to_jiffies(RADEON_IDLE_LOOP_MS)); 704 msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
695 DRM_DEBUG_DRIVER("radeon: dynamic power management activated\n"); 705 DRM_DEBUG_DRIVER("radeon: dynamic power management activated\n");
696 } 706 }
697 } else { /* count == 0 */ 707 } else { /* count == 0 */
@@ -720,9 +730,9 @@ static bool radeon_pm_in_vbl(struct radeon_device *rdev)
720 */ 730 */
721 for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) { 731 for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) {
722 if (rdev->pm.active_crtcs & (1 << crtc)) { 732 if (rdev->pm.active_crtcs & (1 << crtc)) {
723 vbl_status = radeon_get_crtc_scanoutpos(rdev, crtc, &vpos, &hpos); 733 vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, &vpos, &hpos);
724 if ((vbl_status & RADEON_SCANOUTPOS_VALID) && 734 if ((vbl_status & DRM_SCANOUTPOS_VALID) &&
725 !(vbl_status & RADEON_SCANOUTPOS_INVBL)) 735 !(vbl_status & DRM_SCANOUTPOS_INVBL))
726 in_vbl = false; 736 in_vbl = false;
727 } 737 }
728 } 738 }
@@ -796,8 +806,8 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work)
796 radeon_pm_set_clocks(rdev); 806 radeon_pm_set_clocks(rdev);
797 } 807 }
798 808
799 queue_delayed_work(rdev->wq, &rdev->pm.dynpm_idle_work, 809 schedule_delayed_work(&rdev->pm.dynpm_idle_work,
800 msecs_to_jiffies(RADEON_IDLE_LOOP_MS)); 810 msecs_to_jiffies(RADEON_IDLE_LOOP_MS));
801 } 811 }
802 mutex_unlock(&rdev->pm.mutex); 812 mutex_unlock(&rdev->pm.mutex);
803 ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched); 813 ttm_bo_unlock_delayed_workqueue(&rdev->mman.bdev, resched);
@@ -814,9 +824,9 @@ static int radeon_debugfs_pm_info(struct seq_file *m, void *data)
814 struct drm_device *dev = node->minor->dev; 824 struct drm_device *dev = node->minor->dev;
815 struct radeon_device *rdev = dev->dev_private; 825 struct radeon_device *rdev = dev->dev_private;
816 826
817 seq_printf(m, "default engine clock: %u0 kHz\n", rdev->clock.default_sclk); 827 seq_printf(m, "default engine clock: %u0 kHz\n", rdev->pm.default_sclk);
818 seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev)); 828 seq_printf(m, "current engine clock: %u0 kHz\n", radeon_get_engine_clock(rdev));
819 seq_printf(m, "default memory clock: %u0 kHz\n", rdev->clock.default_mclk); 829 seq_printf(m, "default memory clock: %u0 kHz\n", rdev->pm.default_mclk);
820 if (rdev->asic->get_memory_clock) 830 if (rdev->asic->get_memory_clock)
821 seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev)); 831 seq_printf(m, "current memory clock: %u0 kHz\n", radeon_get_memory_clock(rdev));
822 if (rdev->pm.current_vddc) 832 if (rdev->pm.current_vddc)
diff --git a/drivers/gpu/drm/radeon/radeon_reg.h b/drivers/gpu/drm/radeon/radeon_reg.h
index 64928814de53..3cd4dace57c7 100644
--- a/drivers/gpu/drm/radeon/radeon_reg.h
+++ b/drivers/gpu/drm/radeon/radeon_reg.h
@@ -55,6 +55,7 @@
55#include "r500_reg.h" 55#include "r500_reg.h"
56#include "r600_reg.h" 56#include "r600_reg.h"
57#include "evergreen_reg.h" 57#include "evergreen_reg.h"
58#include "ni_reg.h"
58 59
59#define RADEON_MC_AGP_LOCATION 0x014c 60#define RADEON_MC_AGP_LOCATION 0x014c
60#define RADEON_MC_AGP_START_MASK 0x0000FFFF 61#define RADEON_MC_AGP_START_MASK 0x0000FFFF
@@ -320,6 +321,15 @@
320# define RADEON_PCIE_LC_RECONFIG_NOW (1 << 8) 321# define RADEON_PCIE_LC_RECONFIG_NOW (1 << 8)
321# define RADEON_PCIE_LC_RECONFIG_LATER (1 << 9) 322# define RADEON_PCIE_LC_RECONFIG_LATER (1 << 9)
322# define RADEON_PCIE_LC_SHORT_RECONFIG_EN (1 << 10) 323# define RADEON_PCIE_LC_SHORT_RECONFIG_EN (1 << 10)
324# define R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7)
325# define R600_PCIE_LC_RENEGOTIATION_SUPPORT (1 << 9)
326# define R600_PCIE_LC_RENEGOTIATE_EN (1 << 10)
327# define R600_PCIE_LC_SHORT_RECONFIG_EN (1 << 11)
328# define R600_PCIE_LC_UPCONFIGURE_SUPPORT (1 << 12)
329# define R600_PCIE_LC_UPCONFIGURE_DIS (1 << 13)
330
331#define R600_TARGET_AND_CURRENT_PROFILE_INDEX 0x70c
332#define R700_TARGET_AND_CURRENT_PROFILE_INDEX 0x66c
323 333
324#define RADEON_CACHE_CNTL 0x1724 334#define RADEON_CACHE_CNTL 0x1724
325#define RADEON_CACHE_LINE 0x0f0c /* PCI */ 335#define RADEON_CACHE_LINE 0x0f0c /* PCI */
@@ -422,6 +432,7 @@
422# define RADEON_CRTC_CSYNC_EN (1 << 4) 432# define RADEON_CRTC_CSYNC_EN (1 << 4)
423# define RADEON_CRTC_ICON_EN (1 << 15) 433# define RADEON_CRTC_ICON_EN (1 << 15)
424# define RADEON_CRTC_CUR_EN (1 << 16) 434# define RADEON_CRTC_CUR_EN (1 << 16)
435# define RADEON_CRTC_VSTAT_MODE_MASK (3 << 17)
425# define RADEON_CRTC_CUR_MODE_MASK (7 << 20) 436# define RADEON_CRTC_CUR_MODE_MASK (7 << 20)
426# define RADEON_CRTC_CUR_MODE_SHIFT 20 437# define RADEON_CRTC_CUR_MODE_SHIFT 20
427# define RADEON_CRTC_CUR_MODE_MONO 0 438# define RADEON_CRTC_CUR_MODE_MONO 0
@@ -509,6 +520,8 @@
509# define RADEON_CRTC_TILE_EN (1 << 15) 520# define RADEON_CRTC_TILE_EN (1 << 15)
510# define RADEON_CRTC_OFFSET_FLIP_CNTL (1 << 16) 521# define RADEON_CRTC_OFFSET_FLIP_CNTL (1 << 16)
511# define RADEON_CRTC_STEREO_OFFSET_EN (1 << 17) 522# define RADEON_CRTC_STEREO_OFFSET_EN (1 << 17)
523# define RADEON_CRTC_GUI_TRIG_OFFSET_LEFT_EN (1 << 28)
524# define RADEON_CRTC_GUI_TRIG_OFFSET_RIGHT_EN (1 << 29)
512 525
513#define R300_CRTC_TILE_X0_Y0 0x0350 526#define R300_CRTC_TILE_X0_Y0 0x0350
514#define R300_CRTC2_TILE_X0_Y0 0x0358 527#define R300_CRTC2_TILE_X0_Y0 0x0358
diff --git a/drivers/gpu/drm/radeon/radeon_trace.h b/drivers/gpu/drm/radeon/radeon_trace.h
new file mode 100644
index 000000000000..eafd8160a155
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_trace.h
@@ -0,0 +1,82 @@
1#if !defined(_RADEON_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
2#define _RADEON_TRACE_H_
3
4#include <linux/stringify.h>
5#include <linux/types.h>
6#include <linux/tracepoint.h>
7
8#include <drm/drmP.h>
9
10#undef TRACE_SYSTEM
11#define TRACE_SYSTEM radeon
12#define TRACE_SYSTEM_STRING __stringify(TRACE_SYSTEM)
13#define TRACE_INCLUDE_FILE radeon_trace
14
15TRACE_EVENT(radeon_bo_create,
16 TP_PROTO(struct radeon_bo *bo),
17 TP_ARGS(bo),
18 TP_STRUCT__entry(
19 __field(struct radeon_bo *, bo)
20 __field(u32, pages)
21 ),
22
23 TP_fast_assign(
24 __entry->bo = bo;
25 __entry->pages = bo->tbo.num_pages;
26 ),
27 TP_printk("bo=%p, pages=%u", __entry->bo, __entry->pages)
28);
29
30DECLARE_EVENT_CLASS(radeon_fence_request,
31
32 TP_PROTO(struct drm_device *dev, u32 seqno),
33
34 TP_ARGS(dev, seqno),
35
36 TP_STRUCT__entry(
37 __field(u32, dev)
38 __field(u32, seqno)
39 ),
40
41 TP_fast_assign(
42 __entry->dev = dev->primary->index;
43 __entry->seqno = seqno;
44 ),
45
46 TP_printk("dev=%u, seqno=%u", __entry->dev, __entry->seqno)
47);
48
49DEFINE_EVENT(radeon_fence_request, radeon_fence_emit,
50
51 TP_PROTO(struct drm_device *dev, u32 seqno),
52
53 TP_ARGS(dev, seqno)
54);
55
56DEFINE_EVENT(radeon_fence_request, radeon_fence_retire,
57
58 TP_PROTO(struct drm_device *dev, u32 seqno),
59
60 TP_ARGS(dev, seqno)
61);
62
63DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_begin,
64
65 TP_PROTO(struct drm_device *dev, u32 seqno),
66
67 TP_ARGS(dev, seqno)
68);
69
70DEFINE_EVENT(radeon_fence_request, radeon_fence_wait_end,
71
72 TP_PROTO(struct drm_device *dev, u32 seqno),
73
74 TP_ARGS(dev, seqno)
75);
76
77#endif
78
79/* This part must be outside protection */
80#undef TRACE_INCLUDE_PATH
81#define TRACE_INCLUDE_PATH .
82#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/radeon/radeon_trace_points.c b/drivers/gpu/drm/radeon/radeon_trace_points.c
new file mode 100644
index 000000000000..8175993df84d
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_trace_points.c
@@ -0,0 +1,9 @@
1/* Copyright Red Hat Inc 2010.
2 * Author : Dave Airlie <airlied@redhat.com>
3 */
4#include <drm/drmP.h>
5#include "radeon_drm.h"
6#include "radeon.h"
7
8#define CREATE_TRACE_POINTS
9#include "radeon_trace.h"
diff --git a/drivers/gpu/drm/radeon/reg_srcs/rv515 b/drivers/gpu/drm/radeon/reg_srcs/rv515
index b3f9f1d92005..ef422bbacfc1 100644
--- a/drivers/gpu/drm/radeon/reg_srcs/rv515
+++ b/drivers/gpu/drm/radeon/reg_srcs/rv515
@@ -304,6 +304,22 @@ rv515 0x6d40
3040x4630 US_CODE_ADDR 3040x4630 US_CODE_ADDR
3050x4634 US_CODE_RANGE 3050x4634 US_CODE_RANGE
3060x4638 US_CODE_OFFSET 3060x4638 US_CODE_OFFSET
3070x4640 US_FORMAT0_0
3080x4644 US_FORMAT0_1
3090x4648 US_FORMAT0_2
3100x464C US_FORMAT0_3
3110x4650 US_FORMAT0_4
3120x4654 US_FORMAT0_5
3130x4658 US_FORMAT0_6
3140x465C US_FORMAT0_7
3150x4660 US_FORMAT0_8
3160x4664 US_FORMAT0_9
3170x4668 US_FORMAT0_10
3180x466C US_FORMAT0_11
3190x4670 US_FORMAT0_12
3200x4674 US_FORMAT0_13
3210x4678 US_FORMAT0_14
3220x467C US_FORMAT0_15
3070x46A4 US_OUT_FMT_0 3230x46A4 US_OUT_FMT_0
3080x46A8 US_OUT_FMT_1 3240x46A8 US_OUT_FMT_1
3090x46AC US_OUT_FMT_2 3250x46AC US_OUT_FMT_2
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index f1c6e02c2e6b..b4192acaab5f 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -46,6 +46,56 @@
46void rs600_gpu_init(struct radeon_device *rdev); 46void rs600_gpu_init(struct radeon_device *rdev);
47int rs600_mc_wait_for_idle(struct radeon_device *rdev); 47int rs600_mc_wait_for_idle(struct radeon_device *rdev);
48 48
49void rs600_pre_page_flip(struct radeon_device *rdev, int crtc)
50{
51 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
52 u32 tmp;
53
54 /* make sure flip is at vb rather than hb */
55 tmp = RREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset);
56 tmp &= ~AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN;
57 WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, tmp);
58
59 /* set pageflip to happen anywhere in vblank interval */
60 WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
61
62 /* enable the pflip int */
63 radeon_irq_kms_pflip_irq_get(rdev, crtc);
64}
65
66void rs600_post_page_flip(struct radeon_device *rdev, int crtc)
67{
68 /* disable the pflip int */
69 radeon_irq_kms_pflip_irq_put(rdev, crtc);
70}
71
72u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
73{
74 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
75 u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
76
77 /* Lock the graphics update lock */
78 tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
79 WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
80
81 /* update the scanout addresses */
82 WREG32(AVIVO_D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
83 (u32)crtc_base);
84 WREG32(AVIVO_D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
85 (u32)crtc_base);
86
87 /* Wait for update_pending to go high. */
88 while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
89 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
90
91 /* Unlock the lock, so double-buffering can take place inside vblank */
92 tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
93 WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
94
95 /* Return current update_pending status: */
96 return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
97}
98
49void rs600_pm_misc(struct radeon_device *rdev) 99void rs600_pm_misc(struct radeon_device *rdev)
50{ 100{
51 int requested_index = rdev->pm.requested_power_state_index; 101 int requested_index = rdev->pm.requested_power_state_index;
@@ -515,10 +565,12 @@ int rs600_irq_set(struct radeon_device *rdev)
515 if (rdev->irq.gui_idle) { 565 if (rdev->irq.gui_idle) {
516 tmp |= S_000040_GUI_IDLE(1); 566 tmp |= S_000040_GUI_IDLE(1);
517 } 567 }
518 if (rdev->irq.crtc_vblank_int[0]) { 568 if (rdev->irq.crtc_vblank_int[0] ||
569 rdev->irq.pflip[0]) {
519 mode_int |= S_006540_D1MODE_VBLANK_INT_MASK(1); 570 mode_int |= S_006540_D1MODE_VBLANK_INT_MASK(1);
520 } 571 }
521 if (rdev->irq.crtc_vblank_int[1]) { 572 if (rdev->irq.crtc_vblank_int[1] ||
573 rdev->irq.pflip[1]) {
522 mode_int |= S_006540_D2MODE_VBLANK_INT_MASK(1); 574 mode_int |= S_006540_D2MODE_VBLANK_INT_MASK(1);
523 } 575 }
524 if (rdev->irq.hpd[0]) { 576 if (rdev->irq.hpd[0]) {
@@ -534,7 +586,7 @@ int rs600_irq_set(struct radeon_device *rdev)
534 return 0; 586 return 0;
535} 587}
536 588
537static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_int) 589static inline u32 rs600_irq_ack(struct radeon_device *rdev)
538{ 590{
539 uint32_t irqs = RREG32(R_000044_GEN_INT_STATUS); 591 uint32_t irqs = RREG32(R_000044_GEN_INT_STATUS);
540 uint32_t irq_mask = S_000044_SW_INT(1); 592 uint32_t irq_mask = S_000044_SW_INT(1);
@@ -547,27 +599,27 @@ static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_
547 } 599 }
548 600
549 if (G_000044_DISPLAY_INT_STAT(irqs)) { 601 if (G_000044_DISPLAY_INT_STAT(irqs)) {
550 *r500_disp_int = RREG32(R_007EDC_DISP_INTERRUPT_STATUS); 602 rdev->irq.stat_regs.r500.disp_int = RREG32(R_007EDC_DISP_INTERRUPT_STATUS);
551 if (G_007EDC_LB_D1_VBLANK_INTERRUPT(*r500_disp_int)) { 603 if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
552 WREG32(R_006534_D1MODE_VBLANK_STATUS, 604 WREG32(R_006534_D1MODE_VBLANK_STATUS,
553 S_006534_D1MODE_VBLANK_ACK(1)); 605 S_006534_D1MODE_VBLANK_ACK(1));
554 } 606 }
555 if (G_007EDC_LB_D2_VBLANK_INTERRUPT(*r500_disp_int)) { 607 if (G_007EDC_LB_D2_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
556 WREG32(R_006D34_D2MODE_VBLANK_STATUS, 608 WREG32(R_006D34_D2MODE_VBLANK_STATUS,
557 S_006D34_D2MODE_VBLANK_ACK(1)); 609 S_006D34_D2MODE_VBLANK_ACK(1));
558 } 610 }
559 if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(*r500_disp_int)) { 611 if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
560 tmp = RREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL); 612 tmp = RREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL);
561 tmp |= S_007D08_DC_HOT_PLUG_DETECT1_INT_ACK(1); 613 tmp |= S_007D08_DC_HOT_PLUG_DETECT1_INT_ACK(1);
562 WREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp); 614 WREG32(R_007D08_DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
563 } 615 }
564 if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(*r500_disp_int)) { 616 if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
565 tmp = RREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL); 617 tmp = RREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL);
566 tmp |= S_007D18_DC_HOT_PLUG_DETECT2_INT_ACK(1); 618 tmp |= S_007D18_DC_HOT_PLUG_DETECT2_INT_ACK(1);
567 WREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp); 619 WREG32(R_007D18_DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
568 } 620 }
569 } else { 621 } else {
570 *r500_disp_int = 0; 622 rdev->irq.stat_regs.r500.disp_int = 0;
571 } 623 }
572 624
573 if (irqs) { 625 if (irqs) {
@@ -578,32 +630,30 @@ static inline uint32_t rs600_irq_ack(struct radeon_device *rdev, u32 *r500_disp_
578 630
579void rs600_irq_disable(struct radeon_device *rdev) 631void rs600_irq_disable(struct radeon_device *rdev)
580{ 632{
581 u32 tmp;
582
583 WREG32(R_000040_GEN_INT_CNTL, 0); 633 WREG32(R_000040_GEN_INT_CNTL, 0);
584 WREG32(R_006540_DxMODE_INT_MASK, 0); 634 WREG32(R_006540_DxMODE_INT_MASK, 0);
585 /* Wait and acknowledge irq */ 635 /* Wait and acknowledge irq */
586 mdelay(1); 636 mdelay(1);
587 rs600_irq_ack(rdev, &tmp); 637 rs600_irq_ack(rdev);
588} 638}
589 639
590int rs600_irq_process(struct radeon_device *rdev) 640int rs600_irq_process(struct radeon_device *rdev)
591{ 641{
592 uint32_t status, msi_rearm; 642 u32 status, msi_rearm;
593 uint32_t r500_disp_int;
594 bool queue_hotplug = false; 643 bool queue_hotplug = false;
595 644
596 /* reset gui idle ack. the status bit is broken */ 645 /* reset gui idle ack. the status bit is broken */
597 rdev->irq.gui_idle_acked = false; 646 rdev->irq.gui_idle_acked = false;
598 647
599 status = rs600_irq_ack(rdev, &r500_disp_int); 648 status = rs600_irq_ack(rdev);
600 if (!status && !r500_disp_int) { 649 if (!status && !rdev->irq.stat_regs.r500.disp_int) {
601 return IRQ_NONE; 650 return IRQ_NONE;
602 } 651 }
603 while (status || r500_disp_int) { 652 while (status || rdev->irq.stat_regs.r500.disp_int) {
604 /* SW interrupt */ 653 /* SW interrupt */
605 if (G_000044_SW_INT(status)) 654 if (G_000044_SW_INT(status)) {
606 radeon_fence_process(rdev); 655 radeon_fence_process(rdev);
656 }
607 /* GUI idle */ 657 /* GUI idle */
608 if (G_000040_GUI_IDLE(status)) { 658 if (G_000040_GUI_IDLE(status)) {
609 rdev->irq.gui_idle_acked = true; 659 rdev->irq.gui_idle_acked = true;
@@ -611,30 +661,38 @@ int rs600_irq_process(struct radeon_device *rdev)
611 wake_up(&rdev->irq.idle_queue); 661 wake_up(&rdev->irq.idle_queue);
612 } 662 }
613 /* Vertical blank interrupts */ 663 /* Vertical blank interrupts */
614 if (G_007EDC_LB_D1_VBLANK_INTERRUPT(r500_disp_int)) { 664 if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
615 drm_handle_vblank(rdev->ddev, 0); 665 if (rdev->irq.crtc_vblank_int[0]) {
616 rdev->pm.vblank_sync = true; 666 drm_handle_vblank(rdev->ddev, 0);
617 wake_up(&rdev->irq.vblank_queue); 667 rdev->pm.vblank_sync = true;
668 wake_up(&rdev->irq.vblank_queue);
669 }
670 if (rdev->irq.pflip[0])
671 radeon_crtc_handle_flip(rdev, 0);
618 } 672 }
619 if (G_007EDC_LB_D2_VBLANK_INTERRUPT(r500_disp_int)) { 673 if (G_007EDC_LB_D2_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
620 drm_handle_vblank(rdev->ddev, 1); 674 if (rdev->irq.crtc_vblank_int[1]) {
621 rdev->pm.vblank_sync = true; 675 drm_handle_vblank(rdev->ddev, 1);
622 wake_up(&rdev->irq.vblank_queue); 676 rdev->pm.vblank_sync = true;
677 wake_up(&rdev->irq.vblank_queue);
678 }
679 if (rdev->irq.pflip[1])
680 radeon_crtc_handle_flip(rdev, 1);
623 } 681 }
624 if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(r500_disp_int)) { 682 if (G_007EDC_DC_HOT_PLUG_DETECT1_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
625 queue_hotplug = true; 683 queue_hotplug = true;
626 DRM_DEBUG("HPD1\n"); 684 DRM_DEBUG("HPD1\n");
627 } 685 }
628 if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(r500_disp_int)) { 686 if (G_007EDC_DC_HOT_PLUG_DETECT2_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
629 queue_hotplug = true; 687 queue_hotplug = true;
630 DRM_DEBUG("HPD2\n"); 688 DRM_DEBUG("HPD2\n");
631 } 689 }
632 status = rs600_irq_ack(rdev, &r500_disp_int); 690 status = rs600_irq_ack(rdev);
633 } 691 }
634 /* reset gui idle ack. the status bit is broken */ 692 /* reset gui idle ack. the status bit is broken */
635 rdev->irq.gui_idle_acked = false; 693 rdev->irq.gui_idle_acked = false;
636 if (queue_hotplug) 694 if (queue_hotplug)
637 queue_work(rdev->wq, &rdev->hotplug_work); 695 schedule_work(&rdev->hotplug_work);
638 if (rdev->msi_enabled) { 696 if (rdev->msi_enabled) {
639 switch (rdev->family) { 697 switch (rdev->family) {
640 case CHIP_RS600: 698 case CHIP_RS600:
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index 4dfead8cee33..3a264aa3a79a 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -41,6 +41,41 @@
41 41
42static void rv770_gpu_init(struct radeon_device *rdev); 42static void rv770_gpu_init(struct radeon_device *rdev);
43void rv770_fini(struct radeon_device *rdev); 43void rv770_fini(struct radeon_device *rdev);
44static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
45
46u32 rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
47{
48 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
49 u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
50
51 /* Lock the graphics update lock */
52 tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
53 WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
54
55 /* update the scanout addresses */
56 if (radeon_crtc->crtc_id) {
57 WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
58 WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
59 } else {
60 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
61 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
62 }
63 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
64 (u32)crtc_base);
65 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
66 (u32)crtc_base);
67
68 /* Wait for update_pending to go high. */
69 while (!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING));
70 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
71
72 /* Unlock the lock, so double-buffering can take place inside vblank */
73 tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
74 WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
75
76 /* Return current update_pending status: */
77 return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
78}
44 79
45/* get temperature in millidegrees */ 80/* get temperature in millidegrees */
46u32 rv770_get_temp(struct radeon_device *rdev) 81u32 rv770_get_temp(struct radeon_device *rdev)
@@ -489,6 +524,49 @@ static u32 r700_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
489 return backend_map; 524 return backend_map;
490} 525}
491 526
527static void rv770_program_channel_remap(struct radeon_device *rdev)
528{
529 u32 tcp_chan_steer, mc_shared_chremap, tmp;
530 bool force_no_swizzle;
531
532 switch (rdev->family) {
533 case CHIP_RV770:
534 case CHIP_RV730:
535 force_no_swizzle = false;
536 break;
537 case CHIP_RV710:
538 case CHIP_RV740:
539 default:
540 force_no_swizzle = true;
541 break;
542 }
543
544 tmp = RREG32(MC_SHARED_CHMAP);
545 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
546 case 0:
547 case 1:
548 default:
549 /* default mapping */
550 mc_shared_chremap = 0x00fac688;
551 break;
552 case 2:
553 case 3:
554 if (force_no_swizzle)
555 mc_shared_chremap = 0x00fac688;
556 else
557 mc_shared_chremap = 0x00bbc298;
558 break;
559 }
560
561 if (rdev->family == CHIP_RV740)
562 tcp_chan_steer = 0x00ef2a60;
563 else
564 tcp_chan_steer = 0x00fac688;
565
566 WREG32(TCP_CHAN_STEER, tcp_chan_steer);
567 WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
568}
569
492static void rv770_gpu_init(struct radeon_device *rdev) 570static void rv770_gpu_init(struct radeon_device *rdev)
493{ 571{
494 int i, j, num_qd_pipes; 572 int i, j, num_qd_pipes;
@@ -688,6 +766,8 @@ static void rv770_gpu_init(struct radeon_device *rdev)
688 WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff)); 766 WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
689 WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff)); 767 WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
690 768
769 rv770_program_channel_remap(rdev);
770
691 WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable); 771 WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
692 WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config); 772 WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
693 WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config); 773 WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
@@ -956,6 +1036,45 @@ static void rv770_vram_scratch_fini(struct radeon_device *rdev)
956 radeon_bo_unref(&rdev->vram_scratch.robj); 1036 radeon_bo_unref(&rdev->vram_scratch.robj);
957} 1037}
958 1038
1039void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1040{
1041 u64 size_bf, size_af;
1042
1043 if (mc->mc_vram_size > 0xE0000000) {
1044 /* leave room for at least 512M GTT */
1045 dev_warn(rdev->dev, "limiting VRAM\n");
1046 mc->real_vram_size = 0xE0000000;
1047 mc->mc_vram_size = 0xE0000000;
1048 }
1049 if (rdev->flags & RADEON_IS_AGP) {
1050 size_bf = mc->gtt_start;
1051 size_af = 0xFFFFFFFF - mc->gtt_end + 1;
1052 if (size_bf > size_af) {
1053 if (mc->mc_vram_size > size_bf) {
1054 dev_warn(rdev->dev, "limiting VRAM\n");
1055 mc->real_vram_size = size_bf;
1056 mc->mc_vram_size = size_bf;
1057 }
1058 mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1059 } else {
1060 if (mc->mc_vram_size > size_af) {
1061 dev_warn(rdev->dev, "limiting VRAM\n");
1062 mc->real_vram_size = size_af;
1063 mc->mc_vram_size = size_af;
1064 }
1065 mc->vram_start = mc->gtt_end;
1066 }
1067 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1068 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1069 mc->mc_vram_size >> 20, mc->vram_start,
1070 mc->vram_end, mc->real_vram_size >> 20);
1071 } else {
1072 radeon_vram_location(rdev, &rdev->mc, 0);
1073 rdev->mc.gtt_base_align = 0;
1074 radeon_gtt_location(rdev, mc);
1075 }
1076}
1077
959int rv770_mc_init(struct radeon_device *rdev) 1078int rv770_mc_init(struct radeon_device *rdev)
960{ 1079{
961 u32 tmp; 1080 u32 tmp;
@@ -996,7 +1115,7 @@ int rv770_mc_init(struct radeon_device *rdev)
996 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 1115 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
997 rdev->mc.visible_vram_size = rdev->mc.aper_size; 1116 rdev->mc.visible_vram_size = rdev->mc.aper_size;
998 rdev->mc.active_vram_size = rdev->mc.visible_vram_size; 1117 rdev->mc.active_vram_size = rdev->mc.visible_vram_size;
999 r600_vram_gtt_location(rdev, &rdev->mc); 1118 r700_vram_gtt_location(rdev, &rdev->mc);
1000 radeon_update_bandwidth_info(rdev); 1119 radeon_update_bandwidth_info(rdev);
1001 1120
1002 return 0; 1121 return 0;
@@ -1006,6 +1125,9 @@ static int rv770_startup(struct radeon_device *rdev)
1006{ 1125{
1007 int r; 1126 int r;
1008 1127
1128 /* enable pcie gen2 link */
1129 rv770_pcie_gen2_enable(rdev);
1130
1009 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 1131 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1010 r = r600_init_microcode(rdev); 1132 r = r600_init_microcode(rdev);
1011 if (r) { 1133 if (r) {
@@ -1244,3 +1366,75 @@ void rv770_fini(struct radeon_device *rdev)
1244 rdev->bios = NULL; 1366 rdev->bios = NULL;
1245 radeon_dummy_page_fini(rdev); 1367 radeon_dummy_page_fini(rdev);
1246} 1368}
1369
1370static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
1371{
1372 u32 link_width_cntl, lanes, speed_cntl, tmp;
1373 u16 link_cntl2;
1374
1375 if (rdev->flags & RADEON_IS_IGP)
1376 return;
1377
1378 if (!(rdev->flags & RADEON_IS_PCIE))
1379 return;
1380
1381 /* x2 cards have a special sequence */
1382 if (ASIC_IS_X2(rdev))
1383 return;
1384
1385 /* advertise upconfig capability */
1386 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1387 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1388 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1389 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1390 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
1391 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
1392 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
1393 LC_RECONFIG_ARC_MISSING_ESCAPE);
1394 link_width_cntl |= lanes | LC_RECONFIG_NOW |
1395 LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
1396 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1397 } else {
1398 link_width_cntl |= LC_UPCONFIGURE_DIS;
1399 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1400 }
1401
1402 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1403 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
1404 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
1405
1406 tmp = RREG32(0x541c);
1407 WREG32(0x541c, tmp | 0x8);
1408 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
1409 link_cntl2 = RREG16(0x4088);
1410 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
1411 link_cntl2 |= 0x2;
1412 WREG16(0x4088, link_cntl2);
1413 WREG32(MM_CFGREGS_CNTL, 0);
1414
1415 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1416 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
1417 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1418
1419 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1420 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
1421 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1422
1423 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1424 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
1425 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1426
1427 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
1428 speed_cntl |= LC_GEN2_EN_STRAP;
1429 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
1430
1431 } else {
1432 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
1433 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
1434 if (1)
1435 link_width_cntl |= LC_UPCONFIGURE_DIS;
1436 else
1437 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
1438 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1439 }
1440}
diff --git a/drivers/gpu/drm/radeon/rv770d.h b/drivers/gpu/drm/radeon/rv770d.h
index b7a5a20e81dc..abc8cf5a3672 100644
--- a/drivers/gpu/drm/radeon/rv770d.h
+++ b/drivers/gpu/drm/radeon/rv770d.h
@@ -138,6 +138,7 @@
138#define MC_SHARED_CHMAP 0x2004 138#define MC_SHARED_CHMAP 0x2004
139#define NOOFCHAN_SHIFT 12 139#define NOOFCHAN_SHIFT 12
140#define NOOFCHAN_MASK 0x00003000 140#define NOOFCHAN_MASK 0x00003000
141#define MC_SHARED_CHREMAP 0x2008
141 142
142#define MC_ARB_RAMCFG 0x2760 143#define MC_ARB_RAMCFG 0x2760
143#define NOOFBANK_SHIFT 0 144#define NOOFBANK_SHIFT 0
@@ -303,6 +304,7 @@
303#define BILINEAR_PRECISION_8_BIT (1 << 31) 304#define BILINEAR_PRECISION_8_BIT (1 << 31)
304 305
305#define TCP_CNTL 0x9610 306#define TCP_CNTL 0x9610
307#define TCP_CHAN_STEER 0x9614
306 308
307#define VGT_CACHE_INVALIDATION 0x88C4 309#define VGT_CACHE_INVALIDATION 0x88C4
308#define CACHE_INVALIDATION(x) ((x)<<0) 310#define CACHE_INVALIDATION(x) ((x)<<0)
@@ -351,4 +353,49 @@
351 353
352#define SRBM_STATUS 0x0E50 354#define SRBM_STATUS 0x0E50
353 355
356#define D1GRPH_PRIMARY_SURFACE_ADDRESS 0x6110
357#define D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x6914
358#define D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x6114
359#define D1GRPH_SECONDARY_SURFACE_ADDRESS 0x6118
360#define D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x691c
361#define D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x611c
362
363/* PCIE link stuff */
364#define PCIE_LC_TRAINING_CNTL 0xa1 /* PCIE_P */
365#define PCIE_LC_LINK_WIDTH_CNTL 0xa2 /* PCIE_P */
366# define LC_LINK_WIDTH_SHIFT 0
367# define LC_LINK_WIDTH_MASK 0x7
368# define LC_LINK_WIDTH_X0 0
369# define LC_LINK_WIDTH_X1 1
370# define LC_LINK_WIDTH_X2 2
371# define LC_LINK_WIDTH_X4 3
372# define LC_LINK_WIDTH_X8 4
373# define LC_LINK_WIDTH_X16 6
374# define LC_LINK_WIDTH_RD_SHIFT 4
375# define LC_LINK_WIDTH_RD_MASK 0x70
376# define LC_RECONFIG_ARC_MISSING_ESCAPE (1 << 7)
377# define LC_RECONFIG_NOW (1 << 8)
378# define LC_RENEGOTIATION_SUPPORT (1 << 9)
379# define LC_RENEGOTIATE_EN (1 << 10)
380# define LC_SHORT_RECONFIG_EN (1 << 11)
381# define LC_UPCONFIGURE_SUPPORT (1 << 12)
382# define LC_UPCONFIGURE_DIS (1 << 13)
383#define PCIE_LC_SPEED_CNTL 0xa4 /* PCIE_P */
384# define LC_GEN2_EN_STRAP (1 << 0)
385# define LC_TARGET_LINK_SPEED_OVERRIDE_EN (1 << 1)
386# define LC_FORCE_EN_HW_SPEED_CHANGE (1 << 5)
387# define LC_FORCE_DIS_HW_SPEED_CHANGE (1 << 6)
388# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK (0x3 << 8)
389# define LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT 3
390# define LC_CURRENT_DATA_RATE (1 << 11)
391# define LC_VOLTAGE_TIMER_SEL_MASK (0xf << 14)
392# define LC_CLR_FAILED_SPD_CHANGE_CNT (1 << 21)
393# define LC_OTHER_SIDE_EVER_SENT_GEN2 (1 << 23)
394# define LC_OTHER_SIDE_SUPPORTS_GEN2 (1 << 24)
395#define MM_CFGREGS_CNTL 0x544c
396# define MM_WR_TO_CFG_EN (1 << 3)
397#define LINK_CNTL2 0x88 /* F0 */
398# define TARGET_LINK_SPEED_MASK (0xf << 0)
399# define SELECTABLE_DEEMPHASIS (1 << 6)
400
354#endif 401#endif
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 934a96a78540..af61fc29e843 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -169,7 +169,7 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
169} 169}
170EXPORT_SYMBOL(ttm_bo_wait_unreserved); 170EXPORT_SYMBOL(ttm_bo_wait_unreserved);
171 171
172static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 172void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
173{ 173{
174 struct ttm_bo_device *bdev = bo->bdev; 174 struct ttm_bo_device *bdev = bo->bdev;
175 struct ttm_mem_type_manager *man; 175 struct ttm_mem_type_manager *man;
@@ -191,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
191 } 191 }
192} 192}
193 193
194/** 194int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
195 * Call with the lru_lock held.
196 */
197
198static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
199{ 195{
200 int put_count = 0; 196 int put_count = 0;
201 197
@@ -227,9 +223,18 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
227 /** 223 /**
228 * Deadlock avoidance for multi-bo reserving. 224 * Deadlock avoidance for multi-bo reserving.
229 */ 225 */
230 if (use_sequence && bo->seq_valid && 226 if (use_sequence && bo->seq_valid) {
231 (sequence - bo->val_seq < (1 << 31))) { 227 /**
232 return -EAGAIN; 228 * We've already reserved this one.
229 */
230 if (unlikely(sequence == bo->val_seq))
231 return -EDEADLK;
232 /**
233 * Already reserved by a thread that will not back
234 * off for us. We need to back off.
235 */
236 if (unlikely(sequence - bo->val_seq < (1 << 31)))
237 return -EAGAIN;
233 } 238 }
234 239
235 if (no_wait) 240 if (no_wait)
@@ -267,6 +272,13 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
267 BUG(); 272 BUG();
268} 273}
269 274
275void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
276 bool never_free)
277{
278 kref_sub(&bo->list_kref, count,
279 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
280}
281
270int ttm_bo_reserve(struct ttm_buffer_object *bo, 282int ttm_bo_reserve(struct ttm_buffer_object *bo,
271 bool interruptible, 283 bool interruptible,
272 bool no_wait, bool use_sequence, uint32_t sequence) 284 bool no_wait, bool use_sequence, uint32_t sequence)
@@ -282,20 +294,24 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
282 put_count = ttm_bo_del_from_lru(bo); 294 put_count = ttm_bo_del_from_lru(bo);
283 spin_unlock(&glob->lru_lock); 295 spin_unlock(&glob->lru_lock);
284 296
285 while (put_count--) 297 ttm_bo_list_ref_sub(bo, put_count, true);
286 kref_put(&bo->list_kref, ttm_bo_ref_bug);
287 298
288 return ret; 299 return ret;
289} 300}
290 301
302void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
303{
304 ttm_bo_add_to_lru(bo);
305 atomic_set(&bo->reserved, 0);
306 wake_up_all(&bo->event_queue);
307}
308
291void ttm_bo_unreserve(struct ttm_buffer_object *bo) 309void ttm_bo_unreserve(struct ttm_buffer_object *bo)
292{ 310{
293 struct ttm_bo_global *glob = bo->glob; 311 struct ttm_bo_global *glob = bo->glob;
294 312
295 spin_lock(&glob->lru_lock); 313 spin_lock(&glob->lru_lock);
296 ttm_bo_add_to_lru(bo); 314 ttm_bo_unreserve_locked(bo);
297 atomic_set(&bo->reserved, 0);
298 wake_up_all(&bo->event_queue);
299 spin_unlock(&glob->lru_lock); 315 spin_unlock(&glob->lru_lock);
300} 316}
301EXPORT_SYMBOL(ttm_bo_unreserve); 317EXPORT_SYMBOL(ttm_bo_unreserve);
@@ -362,8 +378,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
362 int ret = 0; 378 int ret = 0;
363 379
364 if (old_is_pci || new_is_pci || 380 if (old_is_pci || new_is_pci ||
365 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) 381 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
366 ttm_bo_unmap_virtual(bo); 382 ret = ttm_mem_io_lock(old_man, true);
383 if (unlikely(ret != 0))
384 goto out_err;
385 ttm_bo_unmap_virtual_locked(bo);
386 ttm_mem_io_unlock(old_man);
387 }
367 388
368 /* 389 /*
369 * Create and bind a ttm if required. 390 * Create and bind a ttm if required.
@@ -416,11 +437,9 @@ moved:
416 } 437 }
417 438
418 if (bo->mem.mm_node) { 439 if (bo->mem.mm_node) {
419 spin_lock(&bo->lock);
420 bo->offset = (bo->mem.start << PAGE_SHIFT) + 440 bo->offset = (bo->mem.start << PAGE_SHIFT) +
421 bdev->man[bo->mem.mem_type].gpu_offset; 441 bdev->man[bo->mem.mem_type].gpu_offset;
422 bo->cur_placement = bo->mem.placement; 442 bo->cur_placement = bo->mem.placement;
423 spin_unlock(&bo->lock);
424 } else 443 } else
425 bo->offset = 0; 444 bo->offset = 0;
426 445
@@ -452,7 +471,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
452 ttm_tt_destroy(bo->ttm); 471 ttm_tt_destroy(bo->ttm);
453 bo->ttm = NULL; 472 bo->ttm = NULL;
454 } 473 }
455
456 ttm_bo_mem_put(bo, &bo->mem); 474 ttm_bo_mem_put(bo, &bo->mem);
457 475
458 atomic_set(&bo->reserved, 0); 476 atomic_set(&bo->reserved, 0);
@@ -474,14 +492,14 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
474 int put_count; 492 int put_count;
475 int ret; 493 int ret;
476 494
477 spin_lock(&bo->lock); 495 spin_lock(&bdev->fence_lock);
478 (void) ttm_bo_wait(bo, false, false, true); 496 (void) ttm_bo_wait(bo, false, false, true);
479 if (!bo->sync_obj) { 497 if (!bo->sync_obj) {
480 498
481 spin_lock(&glob->lru_lock); 499 spin_lock(&glob->lru_lock);
482 500
483 /** 501 /**
484 * Lock inversion between bo::reserve and bo::lock here, 502 * Lock inversion between bo:reserve and bdev::fence_lock here,
485 * but that's OK, since we're only trylocking. 503 * but that's OK, since we're only trylocking.
486 */ 504 */
487 505
@@ -490,14 +508,13 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
490 if (unlikely(ret == -EBUSY)) 508 if (unlikely(ret == -EBUSY))
491 goto queue; 509 goto queue;
492 510
493 spin_unlock(&bo->lock); 511 spin_unlock(&bdev->fence_lock);
494 put_count = ttm_bo_del_from_lru(bo); 512 put_count = ttm_bo_del_from_lru(bo);
495 513
496 spin_unlock(&glob->lru_lock); 514 spin_unlock(&glob->lru_lock);
497 ttm_bo_cleanup_memtype_use(bo); 515 ttm_bo_cleanup_memtype_use(bo);
498 516
499 while (put_count--) 517 ttm_bo_list_ref_sub(bo, put_count, true);
500 kref_put(&bo->list_kref, ttm_bo_ref_bug);
501 518
502 return; 519 return;
503 } else { 520 } else {
@@ -512,7 +529,7 @@ queue:
512 kref_get(&bo->list_kref); 529 kref_get(&bo->list_kref);
513 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 530 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
514 spin_unlock(&glob->lru_lock); 531 spin_unlock(&glob->lru_lock);
515 spin_unlock(&bo->lock); 532 spin_unlock(&bdev->fence_lock);
516 533
517 if (sync_obj) { 534 if (sync_obj) {
518 driver->sync_obj_flush(sync_obj, sync_obj_arg); 535 driver->sync_obj_flush(sync_obj, sync_obj_arg);
@@ -537,14 +554,15 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
537 bool no_wait_reserve, 554 bool no_wait_reserve,
538 bool no_wait_gpu) 555 bool no_wait_gpu)
539{ 556{
557 struct ttm_bo_device *bdev = bo->bdev;
540 struct ttm_bo_global *glob = bo->glob; 558 struct ttm_bo_global *glob = bo->glob;
541 int put_count; 559 int put_count;
542 int ret = 0; 560 int ret = 0;
543 561
544retry: 562retry:
545 spin_lock(&bo->lock); 563 spin_lock(&bdev->fence_lock);
546 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 564 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
547 spin_unlock(&bo->lock); 565 spin_unlock(&bdev->fence_lock);
548 566
549 if (unlikely(ret != 0)) 567 if (unlikely(ret != 0))
550 return ret; 568 return ret;
@@ -580,8 +598,7 @@ retry:
580 spin_unlock(&glob->lru_lock); 598 spin_unlock(&glob->lru_lock);
581 ttm_bo_cleanup_memtype_use(bo); 599 ttm_bo_cleanup_memtype_use(bo);
582 600
583 while (put_count--) 601 ttm_bo_list_ref_sub(bo, put_count, true);
584 kref_put(&bo->list_kref, ttm_bo_ref_bug);
585 602
586 return 0; 603 return 0;
587} 604}
@@ -652,6 +669,7 @@ static void ttm_bo_release(struct kref *kref)
652 struct ttm_buffer_object *bo = 669 struct ttm_buffer_object *bo =
653 container_of(kref, struct ttm_buffer_object, kref); 670 container_of(kref, struct ttm_buffer_object, kref);
654 struct ttm_bo_device *bdev = bo->bdev; 671 struct ttm_bo_device *bdev = bo->bdev;
672 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
655 673
656 if (likely(bo->vm_node != NULL)) { 674 if (likely(bo->vm_node != NULL)) {
657 rb_erase(&bo->vm_rb, &bdev->addr_space_rb); 675 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
@@ -659,6 +677,9 @@ static void ttm_bo_release(struct kref *kref)
659 bo->vm_node = NULL; 677 bo->vm_node = NULL;
660 } 678 }
661 write_unlock(&bdev->vm_lock); 679 write_unlock(&bdev->vm_lock);
680 ttm_mem_io_lock(man, false);
681 ttm_mem_io_free_vm(bo);
682 ttm_mem_io_unlock(man);
662 ttm_bo_cleanup_refs_or_queue(bo); 683 ttm_bo_cleanup_refs_or_queue(bo);
663 kref_put(&bo->list_kref, ttm_bo_release_list); 684 kref_put(&bo->list_kref, ttm_bo_release_list);
664 write_lock(&bdev->vm_lock); 685 write_lock(&bdev->vm_lock);
@@ -698,9 +719,9 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
698 struct ttm_placement placement; 719 struct ttm_placement placement;
699 int ret = 0; 720 int ret = 0;
700 721
701 spin_lock(&bo->lock); 722 spin_lock(&bdev->fence_lock);
702 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 723 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
703 spin_unlock(&bo->lock); 724 spin_unlock(&bdev->fence_lock);
704 725
705 if (unlikely(ret != 0)) { 726 if (unlikely(ret != 0)) {
706 if (ret != -ERESTARTSYS) { 727 if (ret != -ERESTARTSYS) {
@@ -715,7 +736,8 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
715 736
716 evict_mem = bo->mem; 737 evict_mem = bo->mem;
717 evict_mem.mm_node = NULL; 738 evict_mem.mm_node = NULL;
718 evict_mem.bus.io_reserved = false; 739 evict_mem.bus.io_reserved_vm = false;
740 evict_mem.bus.io_reserved_count = 0;
719 741
720 placement.fpfn = 0; 742 placement.fpfn = 0;
721 placement.lpfn = 0; 743 placement.lpfn = 0;
@@ -802,8 +824,7 @@ retry:
802 824
803 BUG_ON(ret != 0); 825 BUG_ON(ret != 0);
804 826
805 while (put_count--) 827 ttm_bo_list_ref_sub(bo, put_count, true);
806 kref_put(&bo->list_kref, ttm_bo_ref_bug);
807 828
808 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); 829 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
809 ttm_bo_unreserve(bo); 830 ttm_bo_unreserve(bo);
@@ -1036,6 +1057,7 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1036{ 1057{
1037 int ret = 0; 1058 int ret = 0;
1038 struct ttm_mem_reg mem; 1059 struct ttm_mem_reg mem;
1060 struct ttm_bo_device *bdev = bo->bdev;
1039 1061
1040 BUG_ON(!atomic_read(&bo->reserved)); 1062 BUG_ON(!atomic_read(&bo->reserved));
1041 1063
@@ -1044,15 +1066,16 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1044 * Have the driver move function wait for idle when necessary, 1066 * Have the driver move function wait for idle when necessary,
1045 * instead of doing it here. 1067 * instead of doing it here.
1046 */ 1068 */
1047 spin_lock(&bo->lock); 1069 spin_lock(&bdev->fence_lock);
1048 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1070 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
1049 spin_unlock(&bo->lock); 1071 spin_unlock(&bdev->fence_lock);
1050 if (ret) 1072 if (ret)
1051 return ret; 1073 return ret;
1052 mem.num_pages = bo->num_pages; 1074 mem.num_pages = bo->num_pages;
1053 mem.size = mem.num_pages << PAGE_SHIFT; 1075 mem.size = mem.num_pages << PAGE_SHIFT;
1054 mem.page_alignment = bo->mem.page_alignment; 1076 mem.page_alignment = bo->mem.page_alignment;
1055 mem.bus.io_reserved = false; 1077 mem.bus.io_reserved_vm = false;
1078 mem.bus.io_reserved_count = 0;
1056 /* 1079 /*
1057 * Determine where to move the buffer. 1080 * Determine where to move the buffer.
1058 */ 1081 */
@@ -1163,7 +1186,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1163 } 1186 }
1164 bo->destroy = destroy; 1187 bo->destroy = destroy;
1165 1188
1166 spin_lock_init(&bo->lock);
1167 kref_init(&bo->kref); 1189 kref_init(&bo->kref);
1168 kref_init(&bo->list_kref); 1190 kref_init(&bo->list_kref);
1169 atomic_set(&bo->cpu_writers, 0); 1191 atomic_set(&bo->cpu_writers, 0);
@@ -1172,6 +1194,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1172 INIT_LIST_HEAD(&bo->lru); 1194 INIT_LIST_HEAD(&bo->lru);
1173 INIT_LIST_HEAD(&bo->ddestroy); 1195 INIT_LIST_HEAD(&bo->ddestroy);
1174 INIT_LIST_HEAD(&bo->swap); 1196 INIT_LIST_HEAD(&bo->swap);
1197 INIT_LIST_HEAD(&bo->io_reserve_lru);
1175 bo->bdev = bdev; 1198 bo->bdev = bdev;
1176 bo->glob = bdev->glob; 1199 bo->glob = bdev->glob;
1177 bo->type = type; 1200 bo->type = type;
@@ -1181,7 +1204,8 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1181 bo->mem.num_pages = bo->num_pages; 1204 bo->mem.num_pages = bo->num_pages;
1182 bo->mem.mm_node = NULL; 1205 bo->mem.mm_node = NULL;
1183 bo->mem.page_alignment = page_alignment; 1206 bo->mem.page_alignment = page_alignment;
1184 bo->mem.bus.io_reserved = false; 1207 bo->mem.bus.io_reserved_vm = false;
1208 bo->mem.bus.io_reserved_count = 0;
1185 bo->buffer_start = buffer_start & PAGE_MASK; 1209 bo->buffer_start = buffer_start & PAGE_MASK;
1186 bo->priv_flags = 0; 1210 bo->priv_flags = 0;
1187 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1211 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
@@ -1355,6 +1379,10 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1355 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1379 BUG_ON(type >= TTM_NUM_MEM_TYPES);
1356 man = &bdev->man[type]; 1380 man = &bdev->man[type];
1357 BUG_ON(man->has_type); 1381 BUG_ON(man->has_type);
1382 man->io_reserve_fastpath = true;
1383 man->use_io_reserve_lru = false;
1384 mutex_init(&man->io_reserve_mutex);
1385 INIT_LIST_HEAD(&man->io_reserve_lru);
1358 1386
1359 ret = bdev->driver->init_mem_type(bdev, type, man); 1387 ret = bdev->driver->init_mem_type(bdev, type, man);
1360 if (ret) 1388 if (ret)
@@ -1526,7 +1554,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
1526 bdev->dev_mapping = NULL; 1554 bdev->dev_mapping = NULL;
1527 bdev->glob = glob; 1555 bdev->glob = glob;
1528 bdev->need_dma32 = need_dma32; 1556 bdev->need_dma32 = need_dma32;
1529 1557 bdev->val_seq = 0;
1558 spin_lock_init(&bdev->fence_lock);
1530 mutex_lock(&glob->device_list_mutex); 1559 mutex_lock(&glob->device_list_mutex);
1531 list_add_tail(&bdev->device_list, &glob->device_list); 1560 list_add_tail(&bdev->device_list, &glob->device_list);
1532 mutex_unlock(&glob->device_list_mutex); 1561 mutex_unlock(&glob->device_list_mutex);
@@ -1560,7 +1589,7 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1560 return true; 1589 return true;
1561} 1590}
1562 1591
1563void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1592void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1564{ 1593{
1565 struct ttm_bo_device *bdev = bo->bdev; 1594 struct ttm_bo_device *bdev = bo->bdev;
1566 loff_t offset = (loff_t) bo->addr_space_offset; 1595 loff_t offset = (loff_t) bo->addr_space_offset;
@@ -1569,8 +1598,20 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1569 if (!bdev->dev_mapping) 1598 if (!bdev->dev_mapping)
1570 return; 1599 return;
1571 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1600 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
1572 ttm_mem_io_free(bdev, &bo->mem); 1601 ttm_mem_io_free_vm(bo);
1602}
1603
1604void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1605{
1606 struct ttm_bo_device *bdev = bo->bdev;
1607 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1608
1609 ttm_mem_io_lock(man, false);
1610 ttm_bo_unmap_virtual_locked(bo);
1611 ttm_mem_io_unlock(man);
1573} 1612}
1613
1614
1574EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1615EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1575 1616
1576static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1617static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
@@ -1650,6 +1691,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1650 bool lazy, bool interruptible, bool no_wait) 1691 bool lazy, bool interruptible, bool no_wait)
1651{ 1692{
1652 struct ttm_bo_driver *driver = bo->bdev->driver; 1693 struct ttm_bo_driver *driver = bo->bdev->driver;
1694 struct ttm_bo_device *bdev = bo->bdev;
1653 void *sync_obj; 1695 void *sync_obj;
1654 void *sync_obj_arg; 1696 void *sync_obj_arg;
1655 int ret = 0; 1697 int ret = 0;
@@ -1663,9 +1705,9 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1663 void *tmp_obj = bo->sync_obj; 1705 void *tmp_obj = bo->sync_obj;
1664 bo->sync_obj = NULL; 1706 bo->sync_obj = NULL;
1665 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1707 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1666 spin_unlock(&bo->lock); 1708 spin_unlock(&bdev->fence_lock);
1667 driver->sync_obj_unref(&tmp_obj); 1709 driver->sync_obj_unref(&tmp_obj);
1668 spin_lock(&bo->lock); 1710 spin_lock(&bdev->fence_lock);
1669 continue; 1711 continue;
1670 } 1712 }
1671 1713
@@ -1674,29 +1716,29 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1674 1716
1675 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1717 sync_obj = driver->sync_obj_ref(bo->sync_obj);
1676 sync_obj_arg = bo->sync_obj_arg; 1718 sync_obj_arg = bo->sync_obj_arg;
1677 spin_unlock(&bo->lock); 1719 spin_unlock(&bdev->fence_lock);
1678 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, 1720 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
1679 lazy, interruptible); 1721 lazy, interruptible);
1680 if (unlikely(ret != 0)) { 1722 if (unlikely(ret != 0)) {
1681 driver->sync_obj_unref(&sync_obj); 1723 driver->sync_obj_unref(&sync_obj);
1682 spin_lock(&bo->lock); 1724 spin_lock(&bdev->fence_lock);
1683 return ret; 1725 return ret;
1684 } 1726 }
1685 spin_lock(&bo->lock); 1727 spin_lock(&bdev->fence_lock);
1686 if (likely(bo->sync_obj == sync_obj && 1728 if (likely(bo->sync_obj == sync_obj &&
1687 bo->sync_obj_arg == sync_obj_arg)) { 1729 bo->sync_obj_arg == sync_obj_arg)) {
1688 void *tmp_obj = bo->sync_obj; 1730 void *tmp_obj = bo->sync_obj;
1689 bo->sync_obj = NULL; 1731 bo->sync_obj = NULL;
1690 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1732 clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1691 &bo->priv_flags); 1733 &bo->priv_flags);
1692 spin_unlock(&bo->lock); 1734 spin_unlock(&bdev->fence_lock);
1693 driver->sync_obj_unref(&sync_obj); 1735 driver->sync_obj_unref(&sync_obj);
1694 driver->sync_obj_unref(&tmp_obj); 1736 driver->sync_obj_unref(&tmp_obj);
1695 spin_lock(&bo->lock); 1737 spin_lock(&bdev->fence_lock);
1696 } else { 1738 } else {
1697 spin_unlock(&bo->lock); 1739 spin_unlock(&bdev->fence_lock);
1698 driver->sync_obj_unref(&sync_obj); 1740 driver->sync_obj_unref(&sync_obj);
1699 spin_lock(&bo->lock); 1741 spin_lock(&bdev->fence_lock);
1700 } 1742 }
1701 } 1743 }
1702 return 0; 1744 return 0;
@@ -1705,6 +1747,7 @@ EXPORT_SYMBOL(ttm_bo_wait);
1705 1747
1706int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1748int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1707{ 1749{
1750 struct ttm_bo_device *bdev = bo->bdev;
1708 int ret = 0; 1751 int ret = 0;
1709 1752
1710 /* 1753 /*
@@ -1714,9 +1757,9 @@ int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1714 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1757 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1715 if (unlikely(ret != 0)) 1758 if (unlikely(ret != 0))
1716 return ret; 1759 return ret;
1717 spin_lock(&bo->lock); 1760 spin_lock(&bdev->fence_lock);
1718 ret = ttm_bo_wait(bo, false, true, no_wait); 1761 ret = ttm_bo_wait(bo, false, true, no_wait);
1719 spin_unlock(&bo->lock); 1762 spin_unlock(&bdev->fence_lock);
1720 if (likely(ret == 0)) 1763 if (likely(ret == 0))
1721 atomic_inc(&bo->cpu_writers); 1764 atomic_inc(&bo->cpu_writers);
1722 ttm_bo_unreserve(bo); 1765 ttm_bo_unreserve(bo);
@@ -1782,16 +1825,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1782 put_count = ttm_bo_del_from_lru(bo); 1825 put_count = ttm_bo_del_from_lru(bo);
1783 spin_unlock(&glob->lru_lock); 1826 spin_unlock(&glob->lru_lock);
1784 1827
1785 while (put_count--) 1828 ttm_bo_list_ref_sub(bo, put_count, true);
1786 kref_put(&bo->list_kref, ttm_bo_ref_bug);
1787 1829
1788 /** 1830 /**
1789 * Wait for GPU, then move to system cached. 1831 * Wait for GPU, then move to system cached.
1790 */ 1832 */
1791 1833
1792 spin_lock(&bo->lock); 1834 spin_lock(&bo->bdev->fence_lock);
1793 ret = ttm_bo_wait(bo, false, false, false); 1835 ret = ttm_bo_wait(bo, false, false, false);
1794 spin_unlock(&bo->lock); 1836 spin_unlock(&bo->bdev->fence_lock);
1795 1837
1796 if (unlikely(ret != 0)) 1838 if (unlikely(ret != 0))
1797 goto out; 1839 goto out;
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 3106d5bcce32..77dbf408c0d0 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -75,37 +75,123 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
75} 75}
76EXPORT_SYMBOL(ttm_bo_move_ttm); 76EXPORT_SYMBOL(ttm_bo_move_ttm);
77 77
78int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 78int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
79{ 79{
80 int ret; 80 if (likely(man->io_reserve_fastpath))
81 return 0;
82
83 if (interruptible)
84 return mutex_lock_interruptible(&man->io_reserve_mutex);
85
86 mutex_lock(&man->io_reserve_mutex);
87 return 0;
88}
81 89
82 if (!mem->bus.io_reserved) { 90void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
83 mem->bus.io_reserved = true; 91{
92 if (likely(man->io_reserve_fastpath))
93 return;
94
95 mutex_unlock(&man->io_reserve_mutex);
96}
97
98static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
99{
100 struct ttm_buffer_object *bo;
101
102 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
103 return -EAGAIN;
104
105 bo = list_first_entry(&man->io_reserve_lru,
106 struct ttm_buffer_object,
107 io_reserve_lru);
108 list_del_init(&bo->io_reserve_lru);
109 ttm_bo_unmap_virtual_locked(bo);
110
111 return 0;
112}
113
114static int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
115 struct ttm_mem_reg *mem)
116{
117 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
118 int ret = 0;
119
120 if (!bdev->driver->io_mem_reserve)
121 return 0;
122 if (likely(man->io_reserve_fastpath))
123 return bdev->driver->io_mem_reserve(bdev, mem);
124
125 if (bdev->driver->io_mem_reserve &&
126 mem->bus.io_reserved_count++ == 0) {
127retry:
84 ret = bdev->driver->io_mem_reserve(bdev, mem); 128 ret = bdev->driver->io_mem_reserve(bdev, mem);
129 if (ret == -EAGAIN) {
130 ret = ttm_mem_io_evict(man);
131 if (ret == 0)
132 goto retry;
133 }
134 }
135 return ret;
136}
137
138static void ttm_mem_io_free(struct ttm_bo_device *bdev,
139 struct ttm_mem_reg *mem)
140{
141 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
142
143 if (likely(man->io_reserve_fastpath))
144 return;
145
146 if (bdev->driver->io_mem_reserve &&
147 --mem->bus.io_reserved_count == 0 &&
148 bdev->driver->io_mem_free)
149 bdev->driver->io_mem_free(bdev, mem);
150
151}
152
153int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
154{
155 struct ttm_mem_reg *mem = &bo->mem;
156 int ret;
157
158 if (!mem->bus.io_reserved_vm) {
159 struct ttm_mem_type_manager *man =
160 &bo->bdev->man[mem->mem_type];
161
162 ret = ttm_mem_io_reserve(bo->bdev, mem);
85 if (unlikely(ret != 0)) 163 if (unlikely(ret != 0))
86 return ret; 164 return ret;
165 mem->bus.io_reserved_vm = true;
166 if (man->use_io_reserve_lru)
167 list_add_tail(&bo->io_reserve_lru,
168 &man->io_reserve_lru);
87 } 169 }
88 return 0; 170 return 0;
89} 171}
90 172
91void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 173void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
92{ 174{
93 if (bdev->driver->io_mem_reserve) { 175 struct ttm_mem_reg *mem = &bo->mem;
94 if (mem->bus.io_reserved) { 176
95 mem->bus.io_reserved = false; 177 if (mem->bus.io_reserved_vm) {
96 bdev->driver->io_mem_free(bdev, mem); 178 mem->bus.io_reserved_vm = false;
97 } 179 list_del_init(&bo->io_reserve_lru);
180 ttm_mem_io_free(bo->bdev, mem);
98 } 181 }
99} 182}
100 183
101int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 184int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
102 void **virtual) 185 void **virtual)
103{ 186{
187 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
104 int ret; 188 int ret;
105 void *addr; 189 void *addr;
106 190
107 *virtual = NULL; 191 *virtual = NULL;
192 (void) ttm_mem_io_lock(man, false);
108 ret = ttm_mem_io_reserve(bdev, mem); 193 ret = ttm_mem_io_reserve(bdev, mem);
194 ttm_mem_io_unlock(man);
109 if (ret || !mem->bus.is_iomem) 195 if (ret || !mem->bus.is_iomem)
110 return ret; 196 return ret;
111 197
@@ -117,7 +203,9 @@ int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
117 else 203 else
118 addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); 204 addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
119 if (!addr) { 205 if (!addr) {
206 (void) ttm_mem_io_lock(man, false);
120 ttm_mem_io_free(bdev, mem); 207 ttm_mem_io_free(bdev, mem);
208 ttm_mem_io_unlock(man);
121 return -ENOMEM; 209 return -ENOMEM;
122 } 210 }
123 } 211 }
@@ -134,7 +222,9 @@ void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
134 222
135 if (virtual && mem->bus.addr == NULL) 223 if (virtual && mem->bus.addr == NULL)
136 iounmap(virtual); 224 iounmap(virtual);
225 (void) ttm_mem_io_lock(man, false);
137 ttm_mem_io_free(bdev, mem); 226 ttm_mem_io_free(bdev, mem);
227 ttm_mem_io_unlock(man);
138} 228}
139 229
140static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 230static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
@@ -231,7 +321,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
231 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 321 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
232 struct ttm_tt *ttm = bo->ttm; 322 struct ttm_tt *ttm = bo->ttm;
233 struct ttm_mem_reg *old_mem = &bo->mem; 323 struct ttm_mem_reg *old_mem = &bo->mem;
234 struct ttm_mem_reg old_copy = *old_mem; 324 struct ttm_mem_reg old_copy;
235 void *old_iomap; 325 void *old_iomap;
236 void *new_iomap; 326 void *new_iomap;
237 int ret; 327 int ret;
@@ -280,8 +370,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
280 } 370 }
281 mb(); 371 mb();
282out2: 372out2:
283 ttm_bo_free_old_node(bo); 373 old_copy = *old_mem;
284
285 *old_mem = *new_mem; 374 *old_mem = *new_mem;
286 new_mem->mm_node = NULL; 375 new_mem->mm_node = NULL;
287 376
@@ -292,9 +381,10 @@ out2:
292 } 381 }
293 382
294out1: 383out1:
295 ttm_mem_reg_iounmap(bdev, new_mem, new_iomap); 384 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
296out: 385out:
297 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 386 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
387 ttm_bo_mem_put(bo, &old_copy);
298 return ret; 388 return ret;
299} 389}
300EXPORT_SYMBOL(ttm_bo_move_memcpy); 390EXPORT_SYMBOL(ttm_bo_move_memcpy);
@@ -337,11 +427,11 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
337 * TODO: Explicit member copy would probably be better here. 427 * TODO: Explicit member copy would probably be better here.
338 */ 428 */
339 429
340 spin_lock_init(&fbo->lock);
341 init_waitqueue_head(&fbo->event_queue); 430 init_waitqueue_head(&fbo->event_queue);
342 INIT_LIST_HEAD(&fbo->ddestroy); 431 INIT_LIST_HEAD(&fbo->ddestroy);
343 INIT_LIST_HEAD(&fbo->lru); 432 INIT_LIST_HEAD(&fbo->lru);
344 INIT_LIST_HEAD(&fbo->swap); 433 INIT_LIST_HEAD(&fbo->swap);
434 INIT_LIST_HEAD(&fbo->io_reserve_lru);
345 fbo->vm_node = NULL; 435 fbo->vm_node = NULL;
346 atomic_set(&fbo->cpu_writers, 0); 436 atomic_set(&fbo->cpu_writers, 0);
347 437
@@ -453,6 +543,8 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
453 unsigned long start_page, unsigned long num_pages, 543 unsigned long start_page, unsigned long num_pages,
454 struct ttm_bo_kmap_obj *map) 544 struct ttm_bo_kmap_obj *map)
455{ 545{
546 struct ttm_mem_type_manager *man =
547 &bo->bdev->man[bo->mem.mem_type];
456 unsigned long offset, size; 548 unsigned long offset, size;
457 int ret; 549 int ret;
458 550
@@ -467,7 +559,9 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo,
467 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 559 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
468 return -EPERM; 560 return -EPERM;
469#endif 561#endif
562 (void) ttm_mem_io_lock(man, false);
470 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 563 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
564 ttm_mem_io_unlock(man);
471 if (ret) 565 if (ret)
472 return ret; 566 return ret;
473 if (!bo->mem.bus.is_iomem) { 567 if (!bo->mem.bus.is_iomem) {
@@ -482,12 +576,15 @@ EXPORT_SYMBOL(ttm_bo_kmap);
482 576
483void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 577void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
484{ 578{
579 struct ttm_buffer_object *bo = map->bo;
580 struct ttm_mem_type_manager *man =
581 &bo->bdev->man[bo->mem.mem_type];
582
485 if (!map->virtual) 583 if (!map->virtual)
486 return; 584 return;
487 switch (map->bo_kmap_type) { 585 switch (map->bo_kmap_type) {
488 case ttm_bo_map_iomap: 586 case ttm_bo_map_iomap:
489 iounmap(map->virtual); 587 iounmap(map->virtual);
490 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
491 break; 588 break;
492 case ttm_bo_map_vmap: 589 case ttm_bo_map_vmap:
493 vunmap(map->virtual); 590 vunmap(map->virtual);
@@ -500,6 +597,9 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
500 default: 597 default:
501 BUG(); 598 BUG();
502 } 599 }
600 (void) ttm_mem_io_lock(man, false);
601 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
602 ttm_mem_io_unlock(man);
503 map->virtual = NULL; 603 map->virtual = NULL;
504 map->page = NULL; 604 map->page = NULL;
505} 605}
@@ -520,7 +620,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
520 struct ttm_buffer_object *ghost_obj; 620 struct ttm_buffer_object *ghost_obj;
521 void *tmp_obj = NULL; 621 void *tmp_obj = NULL;
522 622
523 spin_lock(&bo->lock); 623 spin_lock(&bdev->fence_lock);
524 if (bo->sync_obj) { 624 if (bo->sync_obj) {
525 tmp_obj = bo->sync_obj; 625 tmp_obj = bo->sync_obj;
526 bo->sync_obj = NULL; 626 bo->sync_obj = NULL;
@@ -529,7 +629,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
529 bo->sync_obj_arg = sync_obj_arg; 629 bo->sync_obj_arg = sync_obj_arg;
530 if (evict) { 630 if (evict) {
531 ret = ttm_bo_wait(bo, false, false, false); 631 ret = ttm_bo_wait(bo, false, false, false);
532 spin_unlock(&bo->lock); 632 spin_unlock(&bdev->fence_lock);
533 if (tmp_obj) 633 if (tmp_obj)
534 driver->sync_obj_unref(&tmp_obj); 634 driver->sync_obj_unref(&tmp_obj);
535 if (ret) 635 if (ret)
@@ -552,7 +652,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
552 */ 652 */
553 653
554 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 654 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
555 spin_unlock(&bo->lock); 655 spin_unlock(&bdev->fence_lock);
556 if (tmp_obj) 656 if (tmp_obj)
557 driver->sync_obj_unref(&tmp_obj); 657 driver->sync_obj_unref(&tmp_obj);
558 658
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index fe6cb77899f4..221b924acebe 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -83,6 +83,8 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
83 int i; 83 int i;
84 unsigned long address = (unsigned long)vmf->virtual_address; 84 unsigned long address = (unsigned long)vmf->virtual_address;
85 int retval = VM_FAULT_NOPAGE; 85 int retval = VM_FAULT_NOPAGE;
86 struct ttm_mem_type_manager *man =
87 &bdev->man[bo->mem.mem_type];
86 88
87 /* 89 /*
88 * Work around locking order reversal in fault / nopfn 90 * Work around locking order reversal in fault / nopfn
@@ -118,24 +120,28 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
118 * move. 120 * move.
119 */ 121 */
120 122
121 spin_lock(&bo->lock); 123 spin_lock(&bdev->fence_lock);
122 if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { 124 if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
123 ret = ttm_bo_wait(bo, false, true, false); 125 ret = ttm_bo_wait(bo, false, true, false);
124 spin_unlock(&bo->lock); 126 spin_unlock(&bdev->fence_lock);
125 if (unlikely(ret != 0)) { 127 if (unlikely(ret != 0)) {
126 retval = (ret != -ERESTARTSYS) ? 128 retval = (ret != -ERESTARTSYS) ?
127 VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; 129 VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
128 goto out_unlock; 130 goto out_unlock;
129 } 131 }
130 } else 132 } else
131 spin_unlock(&bo->lock); 133 spin_unlock(&bdev->fence_lock);
132 134
133 135 ret = ttm_mem_io_lock(man, true);
134 ret = ttm_mem_io_reserve(bdev, &bo->mem); 136 if (unlikely(ret != 0)) {
135 if (ret) { 137 retval = VM_FAULT_NOPAGE;
136 retval = VM_FAULT_SIGBUS;
137 goto out_unlock; 138 goto out_unlock;
138 } 139 }
140 ret = ttm_mem_io_reserve_vm(bo);
141 if (unlikely(ret != 0)) {
142 retval = VM_FAULT_SIGBUS;
143 goto out_io_unlock;
144 }
139 145
140 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 146 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
141 bo->vm_node->start - vma->vm_pgoff; 147 bo->vm_node->start - vma->vm_pgoff;
@@ -144,7 +150,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
144 150
145 if (unlikely(page_offset >= bo->num_pages)) { 151 if (unlikely(page_offset >= bo->num_pages)) {
146 retval = VM_FAULT_SIGBUS; 152 retval = VM_FAULT_SIGBUS;
147 goto out_unlock; 153 goto out_io_unlock;
148 } 154 }
149 155
150 /* 156 /*
@@ -182,7 +188,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
182 page = ttm_tt_get_page(ttm, page_offset); 188 page = ttm_tt_get_page(ttm, page_offset);
183 if (unlikely(!page && i == 0)) { 189 if (unlikely(!page && i == 0)) {
184 retval = VM_FAULT_OOM; 190 retval = VM_FAULT_OOM;
185 goto out_unlock; 191 goto out_io_unlock;
186 } else if (unlikely(!page)) { 192 } else if (unlikely(!page)) {
187 break; 193 break;
188 } 194 }
@@ -200,14 +206,15 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
200 else if (unlikely(ret != 0)) { 206 else if (unlikely(ret != 0)) {
201 retval = 207 retval =
202 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 208 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
203 goto out_unlock; 209 goto out_io_unlock;
204 } 210 }
205 211
206 address += PAGE_SIZE; 212 address += PAGE_SIZE;
207 if (unlikely(++page_offset >= page_last)) 213 if (unlikely(++page_offset >= page_last))
208 break; 214 break;
209 } 215 }
210 216out_io_unlock:
217 ttm_mem_io_unlock(man);
211out_unlock: 218out_unlock:
212 ttm_bo_unreserve(bo); 219 ttm_bo_unreserve(bo);
213 return retval; 220 return retval;
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index c285c2902d15..3832fe10b4df 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -32,7 +32,7 @@
32#include <linux/sched.h> 32#include <linux/sched.h>
33#include <linux/module.h> 33#include <linux/module.h>
34 34
35void ttm_eu_backoff_reservation(struct list_head *list) 35static void ttm_eu_backoff_reservation_locked(struct list_head *list)
36{ 36{
37 struct ttm_validate_buffer *entry; 37 struct ttm_validate_buffer *entry;
38 38
@@ -41,10 +41,77 @@ void ttm_eu_backoff_reservation(struct list_head *list)
41 if (!entry->reserved) 41 if (!entry->reserved)
42 continue; 42 continue;
43 43
44 if (entry->removed) {
45 ttm_bo_add_to_lru(bo);
46 entry->removed = false;
47
48 }
44 entry->reserved = false; 49 entry->reserved = false;
45 ttm_bo_unreserve(bo); 50 atomic_set(&bo->reserved, 0);
51 wake_up_all(&bo->event_queue);
52 }
53}
54
55static void ttm_eu_del_from_lru_locked(struct list_head *list)
56{
57 struct ttm_validate_buffer *entry;
58
59 list_for_each_entry(entry, list, head) {
60 struct ttm_buffer_object *bo = entry->bo;
61 if (!entry->reserved)
62 continue;
63
64 if (!entry->removed) {
65 entry->put_count = ttm_bo_del_from_lru(bo);
66 entry->removed = true;
67 }
46 } 68 }
47} 69}
70
71static void ttm_eu_list_ref_sub(struct list_head *list)
72{
73 struct ttm_validate_buffer *entry;
74
75 list_for_each_entry(entry, list, head) {
76 struct ttm_buffer_object *bo = entry->bo;
77
78 if (entry->put_count) {
79 ttm_bo_list_ref_sub(bo, entry->put_count, true);
80 entry->put_count = 0;
81 }
82 }
83}
84
85static int ttm_eu_wait_unreserved_locked(struct list_head *list,
86 struct ttm_buffer_object *bo)
87{
88 struct ttm_bo_global *glob = bo->glob;
89 int ret;
90
91 ttm_eu_del_from_lru_locked(list);
92 spin_unlock(&glob->lru_lock);
93 ret = ttm_bo_wait_unreserved(bo, true);
94 spin_lock(&glob->lru_lock);
95 if (unlikely(ret != 0))
96 ttm_eu_backoff_reservation_locked(list);
97 return ret;
98}
99
100
101void ttm_eu_backoff_reservation(struct list_head *list)
102{
103 struct ttm_validate_buffer *entry;
104 struct ttm_bo_global *glob;
105
106 if (list_empty(list))
107 return;
108
109 entry = list_first_entry(list, struct ttm_validate_buffer, head);
110 glob = entry->bo->glob;
111 spin_lock(&glob->lru_lock);
112 ttm_eu_backoff_reservation_locked(list);
113 spin_unlock(&glob->lru_lock);
114}
48EXPORT_SYMBOL(ttm_eu_backoff_reservation); 115EXPORT_SYMBOL(ttm_eu_backoff_reservation);
49 116
50/* 117/*
@@ -59,37 +126,76 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation);
59 * buffers in different orders. 126 * buffers in different orders.
60 */ 127 */
61 128
62int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq) 129int ttm_eu_reserve_buffers(struct list_head *list)
63{ 130{
131 struct ttm_bo_global *glob;
64 struct ttm_validate_buffer *entry; 132 struct ttm_validate_buffer *entry;
65 int ret; 133 int ret;
134 uint32_t val_seq;
135
136 if (list_empty(list))
137 return 0;
138
139 list_for_each_entry(entry, list, head) {
140 entry->reserved = false;
141 entry->put_count = 0;
142 entry->removed = false;
143 }
144
145 entry = list_first_entry(list, struct ttm_validate_buffer, head);
146 glob = entry->bo->glob;
66 147
67retry: 148retry:
149 spin_lock(&glob->lru_lock);
150 val_seq = entry->bo->bdev->val_seq++;
151
68 list_for_each_entry(entry, list, head) { 152 list_for_each_entry(entry, list, head) {
69 struct ttm_buffer_object *bo = entry->bo; 153 struct ttm_buffer_object *bo = entry->bo;
70 154
71 entry->reserved = false; 155retry_this_bo:
72 ret = ttm_bo_reserve(bo, true, false, true, val_seq); 156 ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
73 if (ret != 0) { 157 switch (ret) {
74 ttm_eu_backoff_reservation(list); 158 case 0:
75 if (ret == -EAGAIN) { 159 break;
76 ret = ttm_bo_wait_unreserved(bo, true); 160 case -EBUSY:
77 if (unlikely(ret != 0)) 161 ret = ttm_eu_wait_unreserved_locked(list, bo);
78 return ret; 162 if (unlikely(ret != 0)) {
79 goto retry; 163 spin_unlock(&glob->lru_lock);
80 } else 164 ttm_eu_list_ref_sub(list);
81 return ret; 165 return ret;
166 }
167 goto retry_this_bo;
168 case -EAGAIN:
169 ttm_eu_backoff_reservation_locked(list);
170 spin_unlock(&glob->lru_lock);
171 ttm_eu_list_ref_sub(list);
172 ret = ttm_bo_wait_unreserved(bo, true);
173 if (unlikely(ret != 0))
174 return ret;
175 goto retry;
176 default:
177 ttm_eu_backoff_reservation_locked(list);
178 spin_unlock(&glob->lru_lock);
179 ttm_eu_list_ref_sub(list);
180 return ret;
82 } 181 }
83 182
84 entry->reserved = true; 183 entry->reserved = true;
85 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { 184 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
86 ttm_eu_backoff_reservation(list); 185 ttm_eu_backoff_reservation_locked(list);
186 spin_unlock(&glob->lru_lock);
187 ttm_eu_list_ref_sub(list);
87 ret = ttm_bo_wait_cpu(bo, false); 188 ret = ttm_bo_wait_cpu(bo, false);
88 if (ret) 189 if (ret)
89 return ret; 190 return ret;
90 goto retry; 191 goto retry;
91 } 192 }
92 } 193 }
194
195 ttm_eu_del_from_lru_locked(list);
196 spin_unlock(&glob->lru_lock);
197 ttm_eu_list_ref_sub(list);
198
93 return 0; 199 return 0;
94} 200}
95EXPORT_SYMBOL(ttm_eu_reserve_buffers); 201EXPORT_SYMBOL(ttm_eu_reserve_buffers);
@@ -97,21 +203,36 @@ EXPORT_SYMBOL(ttm_eu_reserve_buffers);
97void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) 203void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj)
98{ 204{
99 struct ttm_validate_buffer *entry; 205 struct ttm_validate_buffer *entry;
206 struct ttm_buffer_object *bo;
207 struct ttm_bo_global *glob;
208 struct ttm_bo_device *bdev;
209 struct ttm_bo_driver *driver;
100 210
101 list_for_each_entry(entry, list, head) { 211 if (list_empty(list))
102 struct ttm_buffer_object *bo = entry->bo; 212 return;
103 struct ttm_bo_driver *driver = bo->bdev->driver; 213
104 void *old_sync_obj; 214 bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo;
215 bdev = bo->bdev;
216 driver = bdev->driver;
217 glob = bo->glob;
105 218
106 spin_lock(&bo->lock); 219 spin_lock(&bdev->fence_lock);
107 old_sync_obj = bo->sync_obj; 220 spin_lock(&glob->lru_lock);
221
222 list_for_each_entry(entry, list, head) {
223 bo = entry->bo;
224 entry->old_sync_obj = bo->sync_obj;
108 bo->sync_obj = driver->sync_obj_ref(sync_obj); 225 bo->sync_obj = driver->sync_obj_ref(sync_obj);
109 bo->sync_obj_arg = entry->new_sync_obj_arg; 226 bo->sync_obj_arg = entry->new_sync_obj_arg;
110 spin_unlock(&bo->lock); 227 ttm_bo_unreserve_locked(bo);
111 ttm_bo_unreserve(bo);
112 entry->reserved = false; 228 entry->reserved = false;
113 if (old_sync_obj) 229 }
114 driver->sync_obj_unref(&old_sync_obj); 230 spin_unlock(&glob->lru_lock);
231 spin_unlock(&bdev->fence_lock);
232
233 list_for_each_entry(entry, list, head) {
234 if (entry->old_sync_obj)
235 driver->sync_obj_unref(&entry->old_sync_obj);
115 } 236 }
116} 237}
117EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); 238EXPORT_SYMBOL(ttm_eu_fence_buffer_objects);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index e7a58d055041..10fc01f69c40 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -264,7 +264,6 @@ struct vmw_private {
264 */ 264 */
265 265
266 struct vmw_sw_context ctx; 266 struct vmw_sw_context ctx;
267 uint32_t val_seq;
268 struct mutex cmdbuf_mutex; 267 struct mutex cmdbuf_mutex;
269 268
270 /** 269 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 76954e3528c1..41b95ed6dbcd 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -653,8 +653,7 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
653 ret = vmw_cmd_check_all(dev_priv, sw_context, cmd, arg->command_size); 653 ret = vmw_cmd_check_all(dev_priv, sw_context, cmd, arg->command_size);
654 if (unlikely(ret != 0)) 654 if (unlikely(ret != 0))
655 goto out_err; 655 goto out_err;
656 ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes, 656 ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes);
657 dev_priv->val_seq++);
658 if (unlikely(ret != 0)) 657 if (unlikely(ret != 0))
659 goto out_err; 658 goto out_err;
660 659
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index fe096a7cc0d7..bfab60c938ac 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -480,9 +480,6 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
480 info->fix.smem_start = 0; 480 info->fix.smem_start = 0;
481 info->fix.smem_len = fb_size; 481 info->fix.smem_len = fb_size;
482 482
483 info->fix.mmio_start = 0;
484 info->fix.mmio_len = 0;
485
486 info->pseudo_palette = par->pseudo_palette; 483 info->pseudo_palette = par->pseudo_palette;
487 info->screen_base = par->vmalloc; 484 info->screen_base = par->vmalloc;
488 info->screen_size = fb_size; 485 info->screen_size = fb_size;
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index c8768f38511e..e01cacba685f 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -33,6 +33,7 @@ struct vga_switcheroo_client {
33 struct fb_info *fb_info; 33 struct fb_info *fb_info;
34 int pwr_state; 34 int pwr_state;
35 void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state); 35 void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state);
36 void (*reprobe)(struct pci_dev *pdev);
36 bool (*can_switch)(struct pci_dev *pdev); 37 bool (*can_switch)(struct pci_dev *pdev);
37 int id; 38 int id;
38 bool active; 39 bool active;
@@ -103,6 +104,7 @@ static void vga_switcheroo_enable(void)
103 104
104int vga_switcheroo_register_client(struct pci_dev *pdev, 105int vga_switcheroo_register_client(struct pci_dev *pdev,
105 void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state), 106 void (*set_gpu_state)(struct pci_dev *pdev, enum vga_switcheroo_state),
107 void (*reprobe)(struct pci_dev *pdev),
106 bool (*can_switch)(struct pci_dev *pdev)) 108 bool (*can_switch)(struct pci_dev *pdev))
107{ 109{
108 int index; 110 int index;
@@ -117,6 +119,7 @@ int vga_switcheroo_register_client(struct pci_dev *pdev,
117 vgasr_priv.clients[index].pwr_state = VGA_SWITCHEROO_ON; 119 vgasr_priv.clients[index].pwr_state = VGA_SWITCHEROO_ON;
118 vgasr_priv.clients[index].pdev = pdev; 120 vgasr_priv.clients[index].pdev = pdev;
119 vgasr_priv.clients[index].set_gpu_state = set_gpu_state; 121 vgasr_priv.clients[index].set_gpu_state = set_gpu_state;
122 vgasr_priv.clients[index].reprobe = reprobe;
120 vgasr_priv.clients[index].can_switch = can_switch; 123 vgasr_priv.clients[index].can_switch = can_switch;
121 vgasr_priv.clients[index].id = -1; 124 vgasr_priv.clients[index].id = -1;
122 if (pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW) 125 if (pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW)
@@ -174,7 +177,8 @@ static int vga_switcheroo_show(struct seq_file *m, void *v)
174 int i; 177 int i;
175 mutex_lock(&vgasr_mutex); 178 mutex_lock(&vgasr_mutex);
176 for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) { 179 for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
177 seq_printf(m, "%d:%c:%s:%s\n", i, 180 seq_printf(m, "%d:%s:%c:%s:%s\n", i,
181 vgasr_priv.clients[i].id == VGA_SWITCHEROO_DIS ? "DIS" : "IGD",
178 vgasr_priv.clients[i].active ? '+' : ' ', 182 vgasr_priv.clients[i].active ? '+' : ' ',
179 vgasr_priv.clients[i].pwr_state ? "Pwr" : "Off", 183 vgasr_priv.clients[i].pwr_state ? "Pwr" : "Off",
180 pci_name(vgasr_priv.clients[i].pdev)); 184 pci_name(vgasr_priv.clients[i].pdev));
@@ -190,9 +194,8 @@ static int vga_switcheroo_debugfs_open(struct inode *inode, struct file *file)
190 194
191static int vga_switchon(struct vga_switcheroo_client *client) 195static int vga_switchon(struct vga_switcheroo_client *client)
192{ 196{
193 int ret; 197 if (vgasr_priv.handler->power_state)
194 198 vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
195 ret = vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_ON);
196 /* call the driver callback to turn on device */ 199 /* call the driver callback to turn on device */
197 client->set_gpu_state(client->pdev, VGA_SWITCHEROO_ON); 200 client->set_gpu_state(client->pdev, VGA_SWITCHEROO_ON);
198 client->pwr_state = VGA_SWITCHEROO_ON; 201 client->pwr_state = VGA_SWITCHEROO_ON;
@@ -203,12 +206,14 @@ static int vga_switchoff(struct vga_switcheroo_client *client)
203{ 206{
204 /* call the driver callback to turn off device */ 207 /* call the driver callback to turn off device */
205 client->set_gpu_state(client->pdev, VGA_SWITCHEROO_OFF); 208 client->set_gpu_state(client->pdev, VGA_SWITCHEROO_OFF);
206 vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF); 209 if (vgasr_priv.handler->power_state)
210 vgasr_priv.handler->power_state(client->id, VGA_SWITCHEROO_OFF);
207 client->pwr_state = VGA_SWITCHEROO_OFF; 211 client->pwr_state = VGA_SWITCHEROO_OFF;
208 return 0; 212 return 0;
209} 213}
210 214
211static int vga_switchto(struct vga_switcheroo_client *new_client) 215/* stage one happens before delay */
216static int vga_switchto_stage1(struct vga_switcheroo_client *new_client)
212{ 217{
213 int ret; 218 int ret;
214 int i; 219 int i;
@@ -235,10 +240,28 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
235 vga_switchon(new_client); 240 vga_switchon(new_client);
236 241
237 /* swap shadow resource to denote boot VGA device has changed so X starts on new device */ 242 /* swap shadow resource to denote boot VGA device has changed so X starts on new device */
238 active->active = false;
239
240 active->pdev->resource[PCI_ROM_RESOURCE].flags &= ~IORESOURCE_ROM_SHADOW; 243 active->pdev->resource[PCI_ROM_RESOURCE].flags &= ~IORESOURCE_ROM_SHADOW;
241 new_client->pdev->resource[PCI_ROM_RESOURCE].flags |= IORESOURCE_ROM_SHADOW; 244 new_client->pdev->resource[PCI_ROM_RESOURCE].flags |= IORESOURCE_ROM_SHADOW;
245 return 0;
246}
247
248/* post delay */
249static int vga_switchto_stage2(struct vga_switcheroo_client *new_client)
250{
251 int ret;
252 int i;
253 struct vga_switcheroo_client *active = NULL;
254
255 for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
256 if (vgasr_priv.clients[i].active == true) {
257 active = &vgasr_priv.clients[i];
258 break;
259 }
260 }
261 if (!active)
262 return 0;
263
264 active->active = false;
242 265
243 if (new_client->fb_info) { 266 if (new_client->fb_info) {
244 struct fb_event event; 267 struct fb_event event;
@@ -250,6 +273,9 @@ static int vga_switchto(struct vga_switcheroo_client *new_client)
250 if (ret) 273 if (ret)
251 return ret; 274 return ret;
252 275
276 if (new_client->reprobe)
277 new_client->reprobe(new_client->pdev);
278
253 if (active->pwr_state == VGA_SWITCHEROO_ON) 279 if (active->pwr_state == VGA_SWITCHEROO_ON)
254 vga_switchoff(active); 280 vga_switchoff(active);
255 281
@@ -265,6 +291,7 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
265 const char *pdev_name; 291 const char *pdev_name;
266 int i, ret; 292 int i, ret;
267 bool delay = false, can_switch; 293 bool delay = false, can_switch;
294 bool just_mux = false;
268 int client_id = -1; 295 int client_id = -1;
269 struct vga_switcheroo_client *client = NULL; 296 struct vga_switcheroo_client *client = NULL;
270 297
@@ -319,6 +346,15 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
319 if (strncmp(usercmd, "DIS", 3) == 0) 346 if (strncmp(usercmd, "DIS", 3) == 0)
320 client_id = VGA_SWITCHEROO_DIS; 347 client_id = VGA_SWITCHEROO_DIS;
321 348
349 if (strncmp(usercmd, "MIGD", 4) == 0) {
350 just_mux = true;
351 client_id = VGA_SWITCHEROO_IGD;
352 }
353 if (strncmp(usercmd, "MDIS", 4) == 0) {
354 just_mux = true;
355 client_id = VGA_SWITCHEROO_DIS;
356 }
357
322 if (client_id == -1) 358 if (client_id == -1)
323 goto out; 359 goto out;
324 360
@@ -330,6 +366,12 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
330 } 366 }
331 367
332 vgasr_priv.delayed_switch_active = false; 368 vgasr_priv.delayed_switch_active = false;
369
370 if (just_mux) {
371 ret = vgasr_priv.handler->switchto(client_id);
372 goto out;
373 }
374
333 /* okay we want a switch - test if devices are willing to switch */ 375 /* okay we want a switch - test if devices are willing to switch */
334 can_switch = true; 376 can_switch = true;
335 for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) { 377 for (i = 0; i < VGA_SWITCHEROO_MAX_CLIENTS; i++) {
@@ -345,18 +387,22 @@ vga_switcheroo_debugfs_write(struct file *filp, const char __user *ubuf,
345 387
346 if (can_switch == true) { 388 if (can_switch == true) {
347 pdev_name = pci_name(client->pdev); 389 pdev_name = pci_name(client->pdev);
348 ret = vga_switchto(client); 390 ret = vga_switchto_stage1(client);
349 if (ret) 391 if (ret)
350 printk(KERN_ERR "vga_switcheroo: switching failed %d\n", ret); 392 printk(KERN_ERR "vga_switcheroo: switching failed stage 1 %d\n", ret);
393
394 ret = vga_switchto_stage2(client);
395 if (ret)
396 printk(KERN_ERR "vga_switcheroo: switching failed stage 2 %d\n", ret);
397
351 } else { 398 } else {
352 printk(KERN_INFO "vga_switcheroo: setting delayed switch to client %d\n", client->id); 399 printk(KERN_INFO "vga_switcheroo: setting delayed switch to client %d\n", client->id);
353 vgasr_priv.delayed_switch_active = true; 400 vgasr_priv.delayed_switch_active = true;
354 vgasr_priv.delayed_client_id = client_id; 401 vgasr_priv.delayed_client_id = client_id;
355 402
356 /* we should at least power up the card to 403 ret = vga_switchto_stage1(client);
357 make the switch faster */ 404 if (ret)
358 if (client->pwr_state == VGA_SWITCHEROO_OFF) 405 printk(KERN_ERR "vga_switcheroo: delayed switching stage 1 failed %d\n", ret);
359 vga_switchon(client);
360 } 406 }
361 407
362out: 408out:
@@ -438,9 +484,9 @@ int vga_switcheroo_process_delayed_switch(void)
438 goto err; 484 goto err;
439 485
440 pdev_name = pci_name(client->pdev); 486 pdev_name = pci_name(client->pdev);
441 ret = vga_switchto(client); 487 ret = vga_switchto_stage2(client);
442 if (ret) 488 if (ret)
443 printk(KERN_ERR "vga_switcheroo: delayed switching failed %d\n", ret); 489 printk(KERN_ERR "vga_switcheroo: delayed switching failed stage 2 %d\n", ret);
444 490
445 vgasr_priv.delayed_switch_active = false; 491 vgasr_priv.delayed_switch_active = false;
446 err = 0; 492 err = 0;
diff --git a/include/drm/drmP.h b/include/drm/drmP.h
index 274eaaa15c36..a4694c610330 100644
--- a/include/drm/drmP.h
+++ b/include/drm/drmP.h
@@ -683,6 +683,21 @@ struct drm_master {
683 void *driver_priv; /**< Private structure for driver to use */ 683 void *driver_priv; /**< Private structure for driver to use */
684}; 684};
685 685
686/* Size of ringbuffer for vblank timestamps. Just double-buffer
687 * in initial implementation.
688 */
689#define DRM_VBLANKTIME_RBSIZE 2
690
691/* Flags and return codes for get_vblank_timestamp() driver function. */
692#define DRM_CALLED_FROM_VBLIRQ 1
693#define DRM_VBLANKTIME_SCANOUTPOS_METHOD (1 << 0)
694#define DRM_VBLANKTIME_INVBL (1 << 1)
695
696/* get_scanout_position() return flags */
697#define DRM_SCANOUTPOS_VALID (1 << 0)
698#define DRM_SCANOUTPOS_INVBL (1 << 1)
699#define DRM_SCANOUTPOS_ACCURATE (1 << 2)
700
686/** 701/**
687 * DRM driver structure. This structure represent the common code for 702 * DRM driver structure. This structure represent the common code for
688 * a family of cards. There will one drm_device for each card present 703 * a family of cards. There will one drm_device for each card present
@@ -760,6 +775,68 @@ struct drm_driver {
760 */ 775 */
761 int (*device_is_agp) (struct drm_device *dev); 776 int (*device_is_agp) (struct drm_device *dev);
762 777
778 /**
779 * Called by vblank timestamping code.
780 *
781 * Return the current display scanout position from a crtc.
782 *
783 * \param dev DRM device.
784 * \param crtc Id of the crtc to query.
785 * \param *vpos Target location for current vertical scanout position.
786 * \param *hpos Target location for current horizontal scanout position.
787 *
788 * Returns vpos as a positive number while in active scanout area.
789 * Returns vpos as a negative number inside vblank, counting the number
790 * of scanlines to go until end of vblank, e.g., -1 means "one scanline
791 * until start of active scanout / end of vblank."
792 *
793 * \return Flags, or'ed together as follows:
794 *
795 * DRM_SCANOUTPOS_VALID = Query successfull.
796 * DRM_SCANOUTPOS_INVBL = Inside vblank.
797 * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of
798 * this flag means that returned position may be offset by a constant
799 * but unknown small number of scanlines wrt. real scanout position.
800 *
801 */
802 int (*get_scanout_position) (struct drm_device *dev, int crtc,
803 int *vpos, int *hpos);
804
805 /**
806 * Called by \c drm_get_last_vbltimestamp. Should return a precise
807 * timestamp when the most recent VBLANK interval ended or will end.
808 *
809 * Specifically, the timestamp in @vblank_time should correspond as
810 * closely as possible to the time when the first video scanline of
811 * the video frame after the end of VBLANK will start scanning out,
812 * the time immmediately after end of the VBLANK interval. If the
813 * @crtc is currently inside VBLANK, this will be a time in the future.
814 * If the @crtc is currently scanning out a frame, this will be the
815 * past start time of the current scanout. This is meant to adhere
816 * to the OpenML OML_sync_control extension specification.
817 *
818 * \param dev dev DRM device handle.
819 * \param crtc crtc for which timestamp should be returned.
820 * \param *max_error Maximum allowable timestamp error in nanoseconds.
821 * Implementation should strive to provide timestamp
822 * with an error of at most *max_error nanoseconds.
823 * Returns true upper bound on error for timestamp.
824 * \param *vblank_time Target location for returned vblank timestamp.
825 * \param flags 0 = Defaults, no special treatment needed.
826 * \param DRM_CALLED_FROM_VBLIRQ = Function is called from vblank
827 * irq handler. Some drivers need to apply some workarounds
828 * for gpu-specific vblank irq quirks if flag is set.
829 *
830 * \returns
831 * Zero if timestamping isn't supported in current display mode or a
832 * negative number on failure. A positive status code on success,
833 * which describes how the vblank_time timestamp was computed.
834 */
835 int (*get_vblank_timestamp) (struct drm_device *dev, int crtc,
836 int *max_error,
837 struct timeval *vblank_time,
838 unsigned flags);
839
763 /* these have to be filled in */ 840 /* these have to be filled in */
764 841
765 irqreturn_t(*irq_handler) (DRM_IRQ_ARGS); 842 irqreturn_t(*irq_handler) (DRM_IRQ_ARGS);
@@ -983,6 +1060,8 @@ struct drm_device {
983 1060
984 wait_queue_head_t *vbl_queue; /**< VBLANK wait queue */ 1061 wait_queue_head_t *vbl_queue; /**< VBLANK wait queue */
985 atomic_t *_vblank_count; /**< number of VBLANK interrupts (driver must alloc the right number of counters) */ 1062 atomic_t *_vblank_count; /**< number of VBLANK interrupts (driver must alloc the right number of counters) */
1063 struct timeval *_vblank_time; /**< timestamp of current vblank_count (drivers must alloc right number of fields) */
1064 spinlock_t vblank_time_lock; /**< Protects vblank count and time updates during vblank enable/disable */
986 spinlock_t vbl_lock; 1065 spinlock_t vbl_lock;
987 atomic_t *vblank_refcount; /* number of users of vblank interruptsper crtc */ 1066 atomic_t *vblank_refcount; /* number of users of vblank interruptsper crtc */
988 u32 *last_vblank; /* protected by dev->vbl_lock, used */ 1067 u32 *last_vblank; /* protected by dev->vbl_lock, used */
@@ -1041,12 +1120,14 @@ struct drm_device {
1041 /*@{ */ 1120 /*@{ */
1042 spinlock_t object_name_lock; 1121 spinlock_t object_name_lock;
1043 struct idr object_name_idr; 1122 struct idr object_name_idr;
1044 uint32_t invalidate_domains; /* domains pending invalidation */
1045 uint32_t flush_domains; /* domains pending flush */
1046 /*@} */ 1123 /*@} */
1047 1124 int switch_power_state;
1048}; 1125};
1049 1126
1127#define DRM_SWITCH_POWER_ON 0
1128#define DRM_SWITCH_POWER_OFF 1
1129#define DRM_SWITCH_POWER_CHANGING 2
1130
1050static __inline__ int drm_core_check_feature(struct drm_device *dev, 1131static __inline__ int drm_core_check_feature(struct drm_device *dev,
1051 int feature) 1132 int feature)
1052{ 1133{
@@ -1284,11 +1365,22 @@ extern int drm_wait_vblank(struct drm_device *dev, void *data,
1284 struct drm_file *filp); 1365 struct drm_file *filp);
1285extern int drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq); 1366extern int drm_vblank_wait(struct drm_device *dev, unsigned int *vbl_seq);
1286extern u32 drm_vblank_count(struct drm_device *dev, int crtc); 1367extern u32 drm_vblank_count(struct drm_device *dev, int crtc);
1368extern u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
1369 struct timeval *vblanktime);
1287extern void drm_handle_vblank(struct drm_device *dev, int crtc); 1370extern void drm_handle_vblank(struct drm_device *dev, int crtc);
1288extern int drm_vblank_get(struct drm_device *dev, int crtc); 1371extern int drm_vblank_get(struct drm_device *dev, int crtc);
1289extern void drm_vblank_put(struct drm_device *dev, int crtc); 1372extern void drm_vblank_put(struct drm_device *dev, int crtc);
1290extern void drm_vblank_off(struct drm_device *dev, int crtc); 1373extern void drm_vblank_off(struct drm_device *dev, int crtc);
1291extern void drm_vblank_cleanup(struct drm_device *dev); 1374extern void drm_vblank_cleanup(struct drm_device *dev);
1375extern u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
1376 struct timeval *tvblank, unsigned flags);
1377extern int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
1378 int crtc, int *max_error,
1379 struct timeval *vblank_time,
1380 unsigned flags,
1381 struct drm_crtc *refcrtc);
1382extern void drm_calc_timestamping_constants(struct drm_crtc *crtc);
1383
1292/* Modesetting support */ 1384/* Modesetting support */
1293extern void drm_vblank_pre_modeset(struct drm_device *dev, int crtc); 1385extern void drm_vblank_pre_modeset(struct drm_device *dev, int crtc);
1294extern void drm_vblank_post_modeset(struct drm_device *dev, int crtc); 1386extern void drm_vblank_post_modeset(struct drm_device *dev, int crtc);
@@ -1321,7 +1413,6 @@ extern int drm_agp_unbind_ioctl(struct drm_device *dev, void *data,
1321extern int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request); 1413extern int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request);
1322extern int drm_agp_bind_ioctl(struct drm_device *dev, void *data, 1414extern int drm_agp_bind_ioctl(struct drm_device *dev, void *data,
1323 struct drm_file *file_priv); 1415 struct drm_file *file_priv);
1324extern void drm_agp_chipset_flush(struct drm_device *dev);
1325 1416
1326 /* Stub support (drm_stub.h) */ 1417 /* Stub support (drm_stub.h) */
1327extern int drm_setmaster_ioctl(struct drm_device *dev, void *data, 1418extern int drm_setmaster_ioctl(struct drm_device *dev, void *data,
@@ -1340,6 +1431,9 @@ extern void drm_put_dev(struct drm_device *dev);
1340extern int drm_put_minor(struct drm_minor **minor); 1431extern int drm_put_minor(struct drm_minor **minor);
1341extern unsigned int drm_debug; 1432extern unsigned int drm_debug;
1342 1433
1434extern unsigned int drm_vblank_offdelay;
1435extern unsigned int drm_timestamp_precision;
1436
1343extern struct class *drm_class; 1437extern struct class *drm_class;
1344extern struct proc_dir_entry *drm_proc_root; 1438extern struct proc_dir_entry *drm_proc_root;
1345extern struct dentry *drm_debugfs_root; 1439extern struct dentry *drm_debugfs_root;
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 029aa688e787..acd7fade160d 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -351,8 +351,14 @@ struct drm_crtc {
351 351
352 bool enabled; 352 bool enabled;
353 353
354 /* Requested mode from modesetting. */
354 struct drm_display_mode mode; 355 struct drm_display_mode mode;
355 356
357 /* Programmed mode in hw, after adjustments for encoders,
358 * crtc, panel scaling etc. Needed for timestamping etc.
359 */
360 struct drm_display_mode hwmode;
361
356 int x, y; 362 int x, y;
357 const struct drm_crtc_funcs *funcs; 363 const struct drm_crtc_funcs *funcs;
358 364
@@ -360,6 +366,9 @@ struct drm_crtc {
360 uint32_t gamma_size; 366 uint32_t gamma_size;
361 uint16_t *gamma_store; 367 uint16_t *gamma_store;
362 368
369 /* Constants needed for precise vblank and swap timestamping. */
370 s64 framedur_ns, linedur_ns, pixeldur_ns;
371
363 /* if you are using the helper */ 372 /* if you are using the helper */
364 void *helper_private; 373 void *helper_private;
365}; 374};
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index f22e7fe4b6db..aac27bd56e89 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -121,9 +121,6 @@ int drm_fb_helper_setcolreg(unsigned regno,
121void drm_fb_helper_restore(void); 121void drm_fb_helper_restore(void);
122void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper, 122void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper,
123 uint32_t fb_width, uint32_t fb_height); 123 uint32_t fb_width, uint32_t fb_height);
124void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
125 uint32_t depth);
126
127int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info); 124int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info);
128 125
129bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper); 126bool drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper);
diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h
index bf01531193d5..e39177778601 100644
--- a/include/drm/drm_mm.h
+++ b/include/drm/drm_mm.h
@@ -62,11 +62,14 @@ struct drm_mm {
62 struct list_head unused_nodes; 62 struct list_head unused_nodes;
63 int num_unused; 63 int num_unused;
64 spinlock_t unused_lock; 64 spinlock_t unused_lock;
65 unsigned int scan_check_range : 1;
65 unsigned scan_alignment; 66 unsigned scan_alignment;
66 unsigned long scan_size; 67 unsigned long scan_size;
67 unsigned long scan_hit_start; 68 unsigned long scan_hit_start;
68 unsigned scan_hit_size; 69 unsigned scan_hit_size;
69 unsigned scanned_blocks; 70 unsigned scanned_blocks;
71 unsigned long scan_start;
72 unsigned long scan_end;
70}; 73};
71 74
72/* 75/*
@@ -145,6 +148,10 @@ static inline struct drm_mm *drm_get_mm(struct drm_mm_node *block)
145 148
146void drm_mm_init_scan(struct drm_mm *mm, unsigned long size, 149void drm_mm_init_scan(struct drm_mm *mm, unsigned long size,
147 unsigned alignment); 150 unsigned alignment);
151void drm_mm_init_scan_with_range(struct drm_mm *mm, unsigned long size,
152 unsigned alignment,
153 unsigned long start,
154 unsigned long end);
148int drm_mm_scan_add_block(struct drm_mm_node *node); 155int drm_mm_scan_add_block(struct drm_mm_node *node);
149int drm_mm_scan_remove_block(struct drm_mm_node *node); 156int drm_mm_scan_remove_block(struct drm_mm_node *node);
150 157
diff --git a/include/drm/drm_pciids.h b/include/drm/drm_pciids.h
index 883c1d439899..fe29ae328bd9 100644
--- a/include/drm/drm_pciids.h
+++ b/include/drm/drm_pciids.h
@@ -142,6 +142,42 @@
142 {0x1002, 0x5e4c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ 142 {0x1002, 0x5e4c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \
143 {0x1002, 0x5e4d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ 143 {0x1002, 0x5e4d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \
144 {0x1002, 0x5e4f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \ 144 {0x1002, 0x5e4f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RV410|RADEON_NEW_MEMMAP}, \
145 {0x1002, 0x6720, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
146 {0x1002, 0x6721, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
147 {0x1002, 0x6722, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
148 {0x1002, 0x6723, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
149 {0x1002, 0x6724, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
150 {0x1002, 0x6725, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
151 {0x1002, 0x6726, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
152 {0x1002, 0x6727, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
153 {0x1002, 0x6728, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
154 {0x1002, 0x6729, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
155 {0x1002, 0x6738, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
156 {0x1002, 0x6739, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BARTS|RADEON_NEW_MEMMAP}, \
157 {0x1002, 0x6740, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
158 {0x1002, 0x6741, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
159 {0x1002, 0x6742, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
160 {0x1002, 0x6743, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
161 {0x1002, 0x6744, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
162 {0x1002, 0x6745, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
163 {0x1002, 0x6746, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
164 {0x1002, 0x6747, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
165 {0x1002, 0x6748, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
166 {0x1002, 0x6749, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
167 {0x1002, 0x6750, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
168 {0x1002, 0x6758, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
169 {0x1002, 0x6759, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TURKS|RADEON_NEW_MEMMAP}, \
170 {0x1002, 0x6760, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
171 {0x1002, 0x6761, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
172 {0x1002, 0x6762, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
173 {0x1002, 0x6763, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
174 {0x1002, 0x6764, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
175 {0x1002, 0x6765, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
176 {0x1002, 0x6766, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
177 {0x1002, 0x6767, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
178 {0x1002, 0x6768, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
179 {0x1002, 0x6770, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
180 {0x1002, 0x6779, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CAICOS|RADEON_NEW_MEMMAP}, \
145 {0x1002, 0x6880, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \ 181 {0x1002, 0x6880, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP}, \
146 {0x1002, 0x6888, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_NEW_MEMMAP}, \ 182 {0x1002, 0x6888, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_NEW_MEMMAP}, \
147 {0x1002, 0x6889, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_NEW_MEMMAP}, \ 183 {0x1002, 0x6889, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CYPRESS|RADEON_NEW_MEMMAP}, \
@@ -419,6 +455,10 @@
419 {0x1002, 0x9713, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ 455 {0x1002, 0x9713, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
420 {0x1002, 0x9714, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ 456 {0x1002, 0x9714, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
421 {0x1002, 0x9715, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \ 457 {0x1002, 0x9715, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RS880|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
458 {0x1002, 0x9802, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
459 {0x1002, 0x9803, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
460 {0x1002, 0x9804, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
461 {0x1002, 0x9805, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_PALM|RADEON_NEW_MEMMAP|RADEON_IS_IGP}, \
422 {0, 0, 0} 462 {0, 0, 0}
423 463
424#define r128_PCI_IDS \ 464#define r128_PCI_IDS \
diff --git a/include/drm/i915_drm.h b/include/drm/i915_drm.h
index a2776e2807a4..0039f1f97ad8 100644
--- a/include/drm/i915_drm.h
+++ b/include/drm/i915_drm.h
@@ -289,6 +289,7 @@ typedef struct drm_i915_irq_wait {
289#define I915_PARAM_HAS_BLT 11 289#define I915_PARAM_HAS_BLT 11
290#define I915_PARAM_HAS_RELAXED_FENCING 12 290#define I915_PARAM_HAS_RELAXED_FENCING 12
291#define I915_PARAM_HAS_COHERENT_RINGS 13 291#define I915_PARAM_HAS_COHERENT_RINGS 13
292#define I915_PARAM_HAS_EXEC_CONSTANTS 14
292 293
293typedef struct drm_i915_getparam { 294typedef struct drm_i915_getparam {
294 int param; 295 int param;
@@ -635,6 +636,17 @@ struct drm_i915_gem_execbuffer2 {
635#define I915_EXEC_RENDER (1<<0) 636#define I915_EXEC_RENDER (1<<0)
636#define I915_EXEC_BSD (2<<0) 637#define I915_EXEC_BSD (2<<0)
637#define I915_EXEC_BLT (3<<0) 638#define I915_EXEC_BLT (3<<0)
639
640/* Used for switching the constants addressing mode on gen4+ RENDER ring.
641 * Gen6+ only supports relative addressing to dynamic state (default) and
642 * absolute addressing.
643 *
644 * These flags are ignored for the BSD and BLT rings.
645 */
646#define I915_EXEC_CONSTANTS_MASK (3<<6)
647#define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */
648#define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6)
649#define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */
638 __u64 flags; 650 __u64 flags;
639 __u64 rsvd1; 651 __u64 rsvd1;
640 __u64 rsvd2; 652 __u64 rsvd2;
diff --git a/include/drm/intel-gtt.h b/include/drm/intel-gtt.h
index d3c81946f613..9e343c0998b4 100644
--- a/include/drm/intel-gtt.h
+++ b/include/drm/intel-gtt.h
@@ -2,17 +2,40 @@
2 2
3#ifndef _DRM_INTEL_GTT_H 3#ifndef _DRM_INTEL_GTT_H
4#define _DRM_INTEL_GTT_H 4#define _DRM_INTEL_GTT_H
5struct intel_gtt { 5
6 /* Number of stolen gtt entries at the beginning. */ 6const struct intel_gtt {
7 unsigned int gtt_stolen_entries; 7 /* Size of memory reserved for graphics by the BIOS */
8 unsigned int stolen_size;
8 /* Total number of gtt entries. */ 9 /* Total number of gtt entries. */
9 unsigned int gtt_total_entries; 10 unsigned int gtt_total_entries;
10 /* Part of the gtt that is mappable by the cpu, for those chips where 11 /* Part of the gtt that is mappable by the cpu, for those chips where
11 * this is not the full gtt. */ 12 * this is not the full gtt. */
12 unsigned int gtt_mappable_entries; 13 unsigned int gtt_mappable_entries;
13}; 14 /* Whether i915 needs to use the dmar apis or not. */
15 unsigned int needs_dmar : 1;
16} *intel_gtt_get(void);
14 17
15struct intel_gtt *intel_gtt_get(void); 18void intel_gtt_chipset_flush(void);
19void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg);
20void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries);
21int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
22 struct scatterlist **sg_list, int *num_sg);
23void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
24 unsigned int sg_len,
25 unsigned int pg_start,
26 unsigned int flags);
27void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
28 struct page **pages, unsigned int flags);
16 29
17#endif 30/* Special gtt memory types */
31#define AGP_DCACHE_MEMORY 1
32#define AGP_PHYS_MEMORY 2
33
34/* New caching attributes for gen6/sandybridge */
35#define AGP_USER_CACHED_MEMORY_LLC_MLC (AGP_USER_TYPES + 2)
36#define AGP_USER_UNCACHED_MEMORY (AGP_USER_TYPES + 4)
18 37
38/* flag for GFDT type */
39#define AGP_USER_CACHED_MEMORY_GFDT (1 << 3)
40
41#endif
diff --git a/include/drm/nouveau_drm.h b/include/drm/nouveau_drm.h
index bc5590b1a1ac..e2cfe80f6fca 100644
--- a/include/drm/nouveau_drm.h
+++ b/include/drm/nouveau_drm.h
@@ -71,16 +71,14 @@ struct drm_nouveau_gpuobj_free {
71#define NOUVEAU_GETPARAM_PCI_VENDOR 3 71#define NOUVEAU_GETPARAM_PCI_VENDOR 3
72#define NOUVEAU_GETPARAM_PCI_DEVICE 4 72#define NOUVEAU_GETPARAM_PCI_DEVICE 4
73#define NOUVEAU_GETPARAM_BUS_TYPE 5 73#define NOUVEAU_GETPARAM_BUS_TYPE 5
74#define NOUVEAU_GETPARAM_FB_PHYSICAL 6
75#define NOUVEAU_GETPARAM_AGP_PHYSICAL 7
76#define NOUVEAU_GETPARAM_FB_SIZE 8 74#define NOUVEAU_GETPARAM_FB_SIZE 8
77#define NOUVEAU_GETPARAM_AGP_SIZE 9 75#define NOUVEAU_GETPARAM_AGP_SIZE 9
78#define NOUVEAU_GETPARAM_PCI_PHYSICAL 10
79#define NOUVEAU_GETPARAM_CHIPSET_ID 11 76#define NOUVEAU_GETPARAM_CHIPSET_ID 11
80#define NOUVEAU_GETPARAM_VM_VRAM_BASE 12 77#define NOUVEAU_GETPARAM_VM_VRAM_BASE 12
81#define NOUVEAU_GETPARAM_GRAPH_UNITS 13 78#define NOUVEAU_GETPARAM_GRAPH_UNITS 13
82#define NOUVEAU_GETPARAM_PTIMER_TIME 14 79#define NOUVEAU_GETPARAM_PTIMER_TIME 14
83#define NOUVEAU_GETPARAM_HAS_BO_USAGE 15 80#define NOUVEAU_GETPARAM_HAS_BO_USAGE 15
81#define NOUVEAU_GETPARAM_HAS_PAGEFLIP 16
84struct drm_nouveau_getparam { 82struct drm_nouveau_getparam {
85 uint64_t param; 83 uint64_t param;
86 uint64_t value; 84 uint64_t value;
@@ -171,7 +169,6 @@ struct drm_nouveau_gem_pushbuf {
171}; 169};
172 170
173#define NOUVEAU_GEM_CPU_PREP_NOWAIT 0x00000001 171#define NOUVEAU_GEM_CPU_PREP_NOWAIT 0x00000001
174#define NOUVEAU_GEM_CPU_PREP_NOBLOCK 0x00000002
175#define NOUVEAU_GEM_CPU_PREP_WRITE 0x00000004 172#define NOUVEAU_GEM_CPU_PREP_WRITE 0x00000004
176struct drm_nouveau_gem_cpu_prep { 173struct drm_nouveau_gem_cpu_prep {
177 uint32_t handle; 174 uint32_t handle;
diff --git a/include/drm/radeon_drm.h b/include/drm/radeon_drm.h
index 10f8b53bdd40..e95a86b8b689 100644
--- a/include/drm/radeon_drm.h
+++ b/include/drm/radeon_drm.h
@@ -906,6 +906,7 @@ struct drm_radeon_cs {
906#define RADEON_INFO_ACCEL_WORKING2 0x05 906#define RADEON_INFO_ACCEL_WORKING2 0x05
907#define RADEON_INFO_TILING_CONFIG 0x06 907#define RADEON_INFO_TILING_CONFIG 0x06
908#define RADEON_INFO_WANT_HYPERZ 0x07 908#define RADEON_INFO_WANT_HYPERZ 0x07
909#define RADEON_INFO_WANT_CMASK 0x08 /* get access to CMASK on r300 */
909 910
910struct drm_radeon_info { 911struct drm_radeon_info {
911 uint32_t request; 912 uint32_t request;
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index beafc156a535..50852aad260a 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -74,6 +74,8 @@ struct ttm_placement {
74 * @is_iomem: is this io memory ? 74 * @is_iomem: is this io memory ?
75 * @size: size in byte 75 * @size: size in byte
76 * @offset: offset from the base address 76 * @offset: offset from the base address
77 * @io_reserved_vm: The VM system has a refcount in @io_reserved_count
78 * @io_reserved_count: Refcounting the numbers of callers to ttm_mem_io_reserve
77 * 79 *
78 * Structure indicating the bus placement of an object. 80 * Structure indicating the bus placement of an object.
79 */ 81 */
@@ -83,7 +85,8 @@ struct ttm_bus_placement {
83 unsigned long size; 85 unsigned long size;
84 unsigned long offset; 86 unsigned long offset;
85 bool is_iomem; 87 bool is_iomem;
86 bool io_reserved; 88 bool io_reserved_vm;
89 uint64_t io_reserved_count;
87}; 90};
88 91
89 92
@@ -154,7 +157,6 @@ struct ttm_tt;
154 * keeps one refcount. When this refcount reaches zero, 157 * keeps one refcount. When this refcount reaches zero,
155 * the object is destroyed. 158 * the object is destroyed.
156 * @event_queue: Queue for processes waiting on buffer object status change. 159 * @event_queue: Queue for processes waiting on buffer object status change.
157 * @lock: spinlock protecting mostly synchronization members.
158 * @mem: structure describing current placement. 160 * @mem: structure describing current placement.
159 * @persistant_swap_storage: Usually the swap storage is deleted for buffers 161 * @persistant_swap_storage: Usually the swap storage is deleted for buffers
160 * pinned in physical memory. If this behaviour is not desired, this member 162 * pinned in physical memory. If this behaviour is not desired, this member
@@ -213,7 +215,6 @@ struct ttm_buffer_object {
213 struct kref kref; 215 struct kref kref;
214 struct kref list_kref; 216 struct kref list_kref;
215 wait_queue_head_t event_queue; 217 wait_queue_head_t event_queue;
216 spinlock_t lock;
217 218
218 /** 219 /**
219 * Members protected by the bo::reserved lock. 220 * Members protected by the bo::reserved lock.
@@ -237,6 +238,7 @@ struct ttm_buffer_object {
237 struct list_head lru; 238 struct list_head lru;
238 struct list_head ddestroy; 239 struct list_head ddestroy;
239 struct list_head swap; 240 struct list_head swap;
241 struct list_head io_reserve_lru;
240 uint32_t val_seq; 242 uint32_t val_seq;
241 bool seq_valid; 243 bool seq_valid;
242 244
@@ -248,10 +250,10 @@ struct ttm_buffer_object {
248 atomic_t reserved; 250 atomic_t reserved;
249 251
250 /** 252 /**
251 * Members protected by the bo::lock 253 * Members protected by struct buffer_object_device::fence_lock
252 * In addition, setting sync_obj to anything else 254 * In addition, setting sync_obj to anything else
253 * than NULL requires bo::reserved to be held. This allows for 255 * than NULL requires bo::reserved to be held. This allows for
254 * checking NULL while reserved but not holding bo::lock. 256 * checking NULL while reserved but not holding the mentioned lock.
255 */ 257 */
256 258
257 void *sync_obj_arg; 259 void *sync_obj_arg;
@@ -364,6 +366,44 @@ extern int ttm_bo_validate(struct ttm_buffer_object *bo,
364 */ 366 */
365extern void ttm_bo_unref(struct ttm_buffer_object **bo); 367extern void ttm_bo_unref(struct ttm_buffer_object **bo);
366 368
369
370/**
371 * ttm_bo_list_ref_sub
372 *
373 * @bo: The buffer object.
374 * @count: The number of references with which to decrease @bo::list_kref;
375 * @never_free: The refcount should not reach zero with this operation.
376 *
377 * Release @count lru list references to this buffer object.
378 */
379extern void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
380 bool never_free);
381
382/**
383 * ttm_bo_add_to_lru
384 *
385 * @bo: The buffer object.
386 *
387 * Add this bo to the relevant mem type lru and, if it's backed by
388 * system pages (ttms) to the swap list.
389 * This function must be called with struct ttm_bo_global::lru_lock held, and
390 * is typically called immediately prior to unreserving a bo.
391 */
392extern void ttm_bo_add_to_lru(struct ttm_buffer_object *bo);
393
394/**
395 * ttm_bo_del_from_lru
396 *
397 * @bo: The buffer object.
398 *
399 * Remove this bo from all lru lists used to lookup and reserve an object.
400 * This function must be called with struct ttm_bo_global::lru_lock held,
401 * and is usually called just immediately after the bo has been reserved to
402 * avoid recursive reservation from lru lists.
403 */
404extern int ttm_bo_del_from_lru(struct ttm_buffer_object *bo);
405
406
367/** 407/**
368 * ttm_bo_lock_delayed_workqueue 408 * ttm_bo_lock_delayed_workqueue
369 * 409 *
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 8e0c848326b6..1da8af6ac884 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -179,30 +179,6 @@ struct ttm_tt {
179#define TTM_MEMTYPE_FLAG_MAPPABLE (1 << 1) /* Memory mappable */ 179#define TTM_MEMTYPE_FLAG_MAPPABLE (1 << 1) /* Memory mappable */
180#define TTM_MEMTYPE_FLAG_CMA (1 << 3) /* Can't map aperture */ 180#define TTM_MEMTYPE_FLAG_CMA (1 << 3) /* Can't map aperture */
181 181
182/**
183 * struct ttm_mem_type_manager
184 *
185 * @has_type: The memory type has been initialized.
186 * @use_type: The memory type is enabled.
187 * @flags: TTM_MEMTYPE_XX flags identifying the traits of the memory
188 * managed by this memory type.
189 * @gpu_offset: If used, the GPU offset of the first managed page of
190 * fixed memory or the first managed location in an aperture.
191 * @size: Size of the managed region.
192 * @available_caching: A mask of available caching types, TTM_PL_FLAG_XX,
193 * as defined in ttm_placement_common.h
194 * @default_caching: The default caching policy used for a buffer object
195 * placed in this memory type if the user doesn't provide one.
196 * @manager: The range manager used for this memory type. FIXME: If the aperture
197 * has a page size different from the underlying system, the granularity
198 * of this manager should take care of this. But the range allocating code
199 * in ttm_bo.c needs to be modified for this.
200 * @lru: The lru list for this memory type.
201 *
202 * This structure is used to identify and manage memory types for a device.
203 * It's set up by the ttm_bo_driver::init_mem_type method.
204 */
205
206struct ttm_mem_type_manager; 182struct ttm_mem_type_manager;
207 183
208struct ttm_mem_type_manager_func { 184struct ttm_mem_type_manager_func {
@@ -287,6 +263,36 @@ struct ttm_mem_type_manager_func {
287 void (*debug)(struct ttm_mem_type_manager *man, const char *prefix); 263 void (*debug)(struct ttm_mem_type_manager *man, const char *prefix);
288}; 264};
289 265
266/**
267 * struct ttm_mem_type_manager
268 *
269 * @has_type: The memory type has been initialized.
270 * @use_type: The memory type is enabled.
271 * @flags: TTM_MEMTYPE_XX flags identifying the traits of the memory
272 * managed by this memory type.
273 * @gpu_offset: If used, the GPU offset of the first managed page of
274 * fixed memory or the first managed location in an aperture.
275 * @size: Size of the managed region.
276 * @available_caching: A mask of available caching types, TTM_PL_FLAG_XX,
277 * as defined in ttm_placement_common.h
278 * @default_caching: The default caching policy used for a buffer object
279 * placed in this memory type if the user doesn't provide one.
280 * @func: structure pointer implementing the range manager. See above
281 * @priv: Driver private closure for @func.
282 * @io_reserve_mutex: Mutex optionally protecting shared io_reserve structures
283 * @use_io_reserve_lru: Use an lru list to try to unreserve io_mem_regions
284 * reserved by the TTM vm system.
285 * @io_reserve_lru: Optional lru list for unreserving io mem regions.
286 * @io_reserve_fastpath: Only use bdev::driver::io_mem_reserve to obtain
287 * static information. bdev::driver::io_mem_free is never used.
288 * @lru: The lru list for this memory type.
289 *
290 * This structure is used to identify and manage memory types for a device.
291 * It's set up by the ttm_bo_driver::init_mem_type method.
292 */
293
294
295
290struct ttm_mem_type_manager { 296struct ttm_mem_type_manager {
291 struct ttm_bo_device *bdev; 297 struct ttm_bo_device *bdev;
292 298
@@ -303,6 +309,15 @@ struct ttm_mem_type_manager {
303 uint32_t default_caching; 309 uint32_t default_caching;
304 const struct ttm_mem_type_manager_func *func; 310 const struct ttm_mem_type_manager_func *func;
305 void *priv; 311 void *priv;
312 struct mutex io_reserve_mutex;
313 bool use_io_reserve_lru;
314 bool io_reserve_fastpath;
315
316 /*
317 * Protected by @io_reserve_mutex:
318 */
319
320 struct list_head io_reserve_lru;
306 321
307 /* 322 /*
308 * Protected by the global->lru_lock. 323 * Protected by the global->lru_lock.
@@ -510,9 +525,12 @@ struct ttm_bo_global {
510 * 525 *
511 * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver. 526 * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver.
512 * @man: An array of mem_type_managers. 527 * @man: An array of mem_type_managers.
528 * @fence_lock: Protects the synchronizing members on *all* bos belonging
529 * to this device.
513 * @addr_space_mm: Range manager for the device address space. 530 * @addr_space_mm: Range manager for the device address space.
514 * lru_lock: Spinlock that protects the buffer+device lru lists and 531 * lru_lock: Spinlock that protects the buffer+device lru lists and
515 * ddestroy lists. 532 * ddestroy lists.
533 * @val_seq: Current validation sequence.
516 * @nice_mode: Try nicely to wait for buffer idle when cleaning a manager. 534 * @nice_mode: Try nicely to wait for buffer idle when cleaning a manager.
517 * If a GPU lockup has been detected, this is forced to 0. 535 * If a GPU lockup has been detected, this is forced to 0.
518 * @dev_mapping: A pointer to the struct address_space representing the 536 * @dev_mapping: A pointer to the struct address_space representing the
@@ -531,6 +549,7 @@ struct ttm_bo_device {
531 struct ttm_bo_driver *driver; 549 struct ttm_bo_driver *driver;
532 rwlock_t vm_lock; 550 rwlock_t vm_lock;
533 struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; 551 struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES];
552 spinlock_t fence_lock;
534 /* 553 /*
535 * Protected by the vm lock. 554 * Protected by the vm lock.
536 */ 555 */
@@ -541,6 +560,7 @@ struct ttm_bo_device {
541 * Protected by the global:lru lock. 560 * Protected by the global:lru lock.
542 */ 561 */
543 struct list_head ddestroy; 562 struct list_head ddestroy;
563 uint32_t val_seq;
544 564
545 /* 565 /*
546 * Protected by load / firstopen / lastclose /unload sync. 566 * Protected by load / firstopen / lastclose /unload sync.
@@ -753,31 +773,6 @@ extern void ttm_bo_mem_put_locked(struct ttm_buffer_object *bo,
753 773
754extern int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait); 774extern int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait);
755 775
756/**
757 * ttm_bo_pci_offset - Get the PCI offset for the buffer object memory.
758 *
759 * @bo Pointer to a struct ttm_buffer_object.
760 * @bus_base On return the base of the PCI region
761 * @bus_offset On return the byte offset into the PCI region
762 * @bus_size On return the byte size of the buffer object or zero if
763 * the buffer object memory is not accessible through a PCI region.
764 *
765 * Returns:
766 * -EINVAL if the buffer object is currently not mappable.
767 * 0 otherwise.
768 */
769
770extern int ttm_bo_pci_offset(struct ttm_bo_device *bdev,
771 struct ttm_mem_reg *mem,
772 unsigned long *bus_base,
773 unsigned long *bus_offset,
774 unsigned long *bus_size);
775
776extern int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
777 struct ttm_mem_reg *mem);
778extern void ttm_mem_io_free(struct ttm_bo_device *bdev,
779 struct ttm_mem_reg *mem);
780
781extern void ttm_bo_global_release(struct drm_global_reference *ref); 776extern void ttm_bo_global_release(struct drm_global_reference *ref);
782extern int ttm_bo_global_init(struct drm_global_reference *ref); 777extern int ttm_bo_global_init(struct drm_global_reference *ref);
783 778
@@ -810,6 +805,22 @@ extern int ttm_bo_device_init(struct ttm_bo_device *bdev,
810extern void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo); 805extern void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo);
811 806
812/** 807/**
808 * ttm_bo_unmap_virtual
809 *
810 * @bo: tear down the virtual mappings for this BO
811 *
812 * The caller must take ttm_mem_io_lock before calling this function.
813 */
814extern void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo);
815
816extern int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo);
817extern void ttm_mem_io_free_vm(struct ttm_buffer_object *bo);
818extern int ttm_mem_io_lock(struct ttm_mem_type_manager *man,
819 bool interruptible);
820extern void ttm_mem_io_unlock(struct ttm_mem_type_manager *man);
821
822
823/**
813 * ttm_bo_reserve: 824 * ttm_bo_reserve:
814 * 825 *
815 * @bo: A pointer to a struct ttm_buffer_object. 826 * @bo: A pointer to a struct ttm_buffer_object.
@@ -859,11 +870,44 @@ extern void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo);
859 * try again. (only if use_sequence == 1). 870 * try again. (only if use_sequence == 1).
860 * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by 871 * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
861 * a signal. Release all buffer reservations and return to user-space. 872 * a signal. Release all buffer reservations and return to user-space.
873 * -EBUSY: The function needed to sleep, but @no_wait was true
874 * -EDEADLK: Bo already reserved using @sequence. This error code will only
875 * be returned if @use_sequence is set to true.
862 */ 876 */
863extern int ttm_bo_reserve(struct ttm_buffer_object *bo, 877extern int ttm_bo_reserve(struct ttm_buffer_object *bo,
864 bool interruptible, 878 bool interruptible,
865 bool no_wait, bool use_sequence, uint32_t sequence); 879 bool no_wait, bool use_sequence, uint32_t sequence);
866 880
881
882/**
883 * ttm_bo_reserve_locked:
884 *
885 * @bo: A pointer to a struct ttm_buffer_object.
886 * @interruptible: Sleep interruptible if waiting.
887 * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY.
888 * @use_sequence: If @bo is already reserved, Only sleep waiting for
889 * it to become unreserved if @sequence < (@bo)->sequence.
890 *
891 * Must be called with struct ttm_bo_global::lru_lock held,
892 * and will not remove reserved buffers from the lru lists.
893 * The function may release the LRU spinlock if it needs to sleep.
894 * Otherwise identical to ttm_bo_reserve.
895 *
896 * Returns:
897 * -EAGAIN: The reservation may cause a deadlock.
898 * Release all buffer reservations, wait for @bo to become unreserved and
899 * try again. (only if use_sequence == 1).
900 * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
901 * a signal. Release all buffer reservations and return to user-space.
902 * -EBUSY: The function needed to sleep, but @no_wait was true
903 * -EDEADLK: Bo already reserved using @sequence. This error code will only
904 * be returned if @use_sequence is set to true.
905 */
906extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
907 bool interruptible,
908 bool no_wait, bool use_sequence,
909 uint32_t sequence);
910
867/** 911/**
868 * ttm_bo_unreserve 912 * ttm_bo_unreserve
869 * 913 *
@@ -874,6 +918,16 @@ extern int ttm_bo_reserve(struct ttm_buffer_object *bo,
874extern void ttm_bo_unreserve(struct ttm_buffer_object *bo); 918extern void ttm_bo_unreserve(struct ttm_buffer_object *bo);
875 919
876/** 920/**
921 * ttm_bo_unreserve_locked
922 *
923 * @bo: A pointer to a struct ttm_buffer_object.
924 *
925 * Unreserve a previous reservation of @bo.
926 * Needs to be called with struct ttm_bo_global::lru_lock held.
927 */
928extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo);
929
930/**
877 * ttm_bo_wait_unreserved 931 * ttm_bo_wait_unreserved
878 * 932 *
879 * @bo: A pointer to a struct ttm_buffer_object. 933 * @bo: A pointer to a struct ttm_buffer_object.
diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h
index cd2c475da9ea..26cc7f9ffa41 100644
--- a/include/drm/ttm/ttm_execbuf_util.h
+++ b/include/drm/ttm/ttm_execbuf_util.h
@@ -41,7 +41,10 @@
41 * @bo: refcounted buffer object pointer. 41 * @bo: refcounted buffer object pointer.
42 * @new_sync_obj_arg: New sync_obj_arg for @bo, to be used once 42 * @new_sync_obj_arg: New sync_obj_arg for @bo, to be used once
43 * adding a new sync object. 43 * adding a new sync object.
44 * @reservied: Indicates whether @bo has been reserved for validation. 44 * @reserved: Indicates whether @bo has been reserved for validation.
45 * @removed: Indicates whether @bo has been removed from lru lists.
46 * @put_count: Number of outstanding references on bo::list_kref.
47 * @old_sync_obj: Pointer to a sync object about to be unreferenced
45 */ 48 */
46 49
47struct ttm_validate_buffer { 50struct ttm_validate_buffer {
@@ -49,6 +52,9 @@ struct ttm_validate_buffer {
49 struct ttm_buffer_object *bo; 52 struct ttm_buffer_object *bo;
50 void *new_sync_obj_arg; 53 void *new_sync_obj_arg;
51 bool reserved; 54 bool reserved;
55 bool removed;
56 int put_count;
57 void *old_sync_obj;
52}; 58};
53 59
54/** 60/**
@@ -66,7 +72,6 @@ extern void ttm_eu_backoff_reservation(struct list_head *list);
66 * function ttm_eu_reserve_buffers 72 * function ttm_eu_reserve_buffers
67 * 73 *
68 * @list: thread private list of ttm_validate_buffer structs. 74 * @list: thread private list of ttm_validate_buffer structs.
69 * @val_seq: A unique sequence number.
70 * 75 *
71 * Tries to reserve bos pointed to by the list entries for validation. 76 * Tries to reserve bos pointed to by the list entries for validation.
72 * If the function returns 0, all buffers are marked as "unfenced", 77 * If the function returns 0, all buffers are marked as "unfenced",
@@ -88,7 +93,7 @@ extern void ttm_eu_backoff_reservation(struct list_head *list);
88 * has failed. 93 * has failed.
89 */ 94 */
90 95
91extern int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq); 96extern int ttm_eu_reserve_buffers(struct list_head *list);
92 97
93/** 98/**
94 * function ttm_eu_fence_buffer_objects. 99 * function ttm_eu_fence_buffer_objects.
diff --git a/include/linux/agp_backend.h b/include/linux/agp_backend.h
index 09ea4a1e9505..eaf6cd75a1b1 100644
--- a/include/linux/agp_backend.h
+++ b/include/linux/agp_backend.h
@@ -102,10 +102,8 @@ extern struct agp_memory *agp_allocate_memory(struct agp_bridge_data *, size_t,
102extern int agp_copy_info(struct agp_bridge_data *, struct agp_kern_info *); 102extern int agp_copy_info(struct agp_bridge_data *, struct agp_kern_info *);
103extern int agp_bind_memory(struct agp_memory *, off_t); 103extern int agp_bind_memory(struct agp_memory *, off_t);
104extern int agp_unbind_memory(struct agp_memory *); 104extern int agp_unbind_memory(struct agp_memory *);
105extern int agp_rebind_memory(void);
106extern void agp_enable(struct agp_bridge_data *, u32); 105extern void agp_enable(struct agp_bridge_data *, u32);
107extern struct agp_bridge_data *agp_backend_acquire(struct pci_dev *); 106extern struct agp_bridge_data *agp_backend_acquire(struct pci_dev *);
108extern void agp_backend_release(struct agp_bridge_data *); 107extern void agp_backend_release(struct agp_bridge_data *);
109extern void agp_flush_chipset(struct agp_bridge_data *);
110 108
111#endif /* _AGP_BACKEND_H */ 109#endif /* _AGP_BACKEND_H */
diff --git a/include/linux/intel-gtt.h b/include/linux/intel-gtt.h
deleted file mode 100644
index 1d19ab2afa39..000000000000
--- a/include/linux/intel-gtt.h
+++ /dev/null
@@ -1,20 +0,0 @@
1/*
2 * Common Intel AGPGART and GTT definitions.
3 */
4#ifndef _INTEL_GTT_H
5#define _INTEL_GTT_H
6
7#include <linux/agp_backend.h>
8
9/* This is for Intel only GTT controls.
10 *
11 * Sandybridge: AGP_USER_CACHED_MEMORY default to LLC only
12 */
13
14#define AGP_USER_CACHED_MEMORY_LLC_MLC (AGP_USER_TYPES + 2)
15#define AGP_USER_UNCACHED_MEMORY (AGP_USER_TYPES + 4)
16
17/* flag for GFDT type */
18#define AGP_USER_CACHED_MEMORY_GFDT (1 << 3)
19
20#endif
diff --git a/include/linux/kref.h b/include/linux/kref.h
index 6cc38fc07ab7..d4a62ab2ee5e 100644
--- a/include/linux/kref.h
+++ b/include/linux/kref.h
@@ -24,5 +24,7 @@ struct kref {
24void kref_init(struct kref *kref); 24void kref_init(struct kref *kref);
25void kref_get(struct kref *kref); 25void kref_get(struct kref *kref);
26int kref_put(struct kref *kref, void (*release) (struct kref *kref)); 26int kref_put(struct kref *kref, void (*release) (struct kref *kref));
27int kref_sub(struct kref *kref, unsigned int count,
28 void (*release) (struct kref *kref));
27 29
28#endif /* _KREF_H_ */ 30#endif /* _KREF_H_ */
diff --git a/include/linux/vga_switcheroo.h b/include/linux/vga_switcheroo.h
index ae9ab13b963d..4b9a7f596f92 100644
--- a/include/linux/vga_switcheroo.h
+++ b/include/linux/vga_switcheroo.h
@@ -33,6 +33,7 @@ struct vga_switcheroo_handler {
33void vga_switcheroo_unregister_client(struct pci_dev *dev); 33void vga_switcheroo_unregister_client(struct pci_dev *dev);
34int vga_switcheroo_register_client(struct pci_dev *dev, 34int vga_switcheroo_register_client(struct pci_dev *dev,
35 void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state), 35 void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state),
36 void (*reprobe)(struct pci_dev *dev),
36 bool (*can_switch)(struct pci_dev *dev)); 37 bool (*can_switch)(struct pci_dev *dev));
37 38
38void vga_switcheroo_client_fb_set(struct pci_dev *dev, 39void vga_switcheroo_client_fb_set(struct pci_dev *dev,
@@ -48,6 +49,7 @@ int vga_switcheroo_process_delayed_switch(void);
48static inline void vga_switcheroo_unregister_client(struct pci_dev *dev) {} 49static inline void vga_switcheroo_unregister_client(struct pci_dev *dev) {}
49static inline int vga_switcheroo_register_client(struct pci_dev *dev, 50static inline int vga_switcheroo_register_client(struct pci_dev *dev,
50 void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state), 51 void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state),
52 void (*reprobe)(struct pci_dev *dev),
51 bool (*can_switch)(struct pci_dev *dev)) { return 0; } 53 bool (*can_switch)(struct pci_dev *dev)) { return 0; }
52static inline void vga_switcheroo_client_fb_set(struct pci_dev *dev, struct fb_info *info) {} 54static inline void vga_switcheroo_client_fb_set(struct pci_dev *dev, struct fb_info *info) {}
53static inline int vga_switcheroo_register_handler(struct vga_switcheroo_handler *handler) { return 0; } 55static inline int vga_switcheroo_register_handler(struct vga_switcheroo_handler *handler) { return 0; }
diff --git a/lib/kref.c b/lib/kref.c
index d3d227a08a4b..3efb882b11db 100644
--- a/lib/kref.c
+++ b/lib/kref.c
@@ -62,6 +62,36 @@ int kref_put(struct kref *kref, void (*release)(struct kref *kref))
62 return 0; 62 return 0;
63} 63}
64 64
65
66/**
67 * kref_sub - subtract a number of refcounts for object.
68 * @kref: object.
69 * @count: Number of recounts to subtract.
70 * @release: pointer to the function that will clean up the object when the
71 * last reference to the object is released.
72 * This pointer is required, and it is not acceptable to pass kfree
73 * in as this function.
74 *
75 * Subtract @count from the refcount, and if 0, call release().
76 * Return 1 if the object was removed, otherwise return 0. Beware, if this
77 * function returns 0, you still can not count on the kref from remaining in
78 * memory. Only use the return value if you want to see if the kref is now
79 * gone, not present.
80 */
81int kref_sub(struct kref *kref, unsigned int count,
82 void (*release)(struct kref *kref))
83{
84 WARN_ON(release == NULL);
85 WARN_ON(release == (void (*)(struct kref *))kfree);
86
87 if (atomic_sub_and_test((int) count, &kref->refcount)) {
88 release(kref);
89 return 1;
90 }
91 return 0;
92}
93
65EXPORT_SYMBOL(kref_init); 94EXPORT_SYMBOL(kref_init);
66EXPORT_SYMBOL(kref_get); 95EXPORT_SYMBOL(kref_get);
67EXPORT_SYMBOL(kref_put); 96EXPORT_SYMBOL(kref_put);
97EXPORT_SYMBOL(kref_sub);