aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu')
-rw-r--r--drivers/gpu/drm/drm_drv.c12
-rw-r--r--drivers/gpu/drm/drm_irq.c84
-rw-r--r--drivers/gpu/drm/drm_lock.c9
-rw-r--r--drivers/gpu/drm/drm_stub.c1
-rw-r--r--drivers/gpu/drm/i915/Makefile3
-rw-r--r--drivers/gpu/drm/i915/i915_dma.c29
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h57
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c897
-rw-r--r--drivers/gpu/drm/i915/i915_gem_proc.c5
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c7
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c676
-rw-r--r--drivers/gpu/drm/i915/i915_opregion.c18
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h4
-rw-r--r--drivers/gpu/drm/i915/i915_suspend.c15
-rw-r--r--drivers/gpu/drm/mga/mga_dma.c8
-rw-r--r--drivers/gpu/drm/mga/mga_irq.c5
-rw-r--r--drivers/gpu/drm/r128/r128_drv.c6
-rw-r--r--drivers/gpu/drm/r128/r128_drv.h1
-rw-r--r--drivers/gpu/drm/r128/r128_irq.c2
-rw-r--r--drivers/gpu/drm/radeon/radeon_cp.c36
-rw-r--r--drivers/gpu/drm/radeon/radeon_drv.h15
-rw-r--r--drivers/gpu/drm/radeon/radeon_irq.c13
-rw-r--r--drivers/gpu/drm/via/via_irq.c1
-rw-r--r--drivers/gpu/drm/via/via_map.c11
24 files changed, 872 insertions, 1043 deletions
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 96f416afc3f6..996097acb5e7 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -266,11 +266,19 @@ int drm_init(struct drm_driver *driver)
266 for (i = 0; driver->pci_driver.id_table[i].vendor != 0; i++) { 266 for (i = 0; driver->pci_driver.id_table[i].vendor != 0; i++) {
267 pid = (struct pci_device_id *)&driver->pci_driver.id_table[i]; 267 pid = (struct pci_device_id *)&driver->pci_driver.id_table[i];
268 268
269 /* Loop around setting up a DRM device for each PCI device
270 * matching our ID and device class. If we had the internal
271 * function that pci_get_subsys and pci_get_class used, we'd
272 * be able to just pass pid in instead of doing a two-stage
273 * thing.
274 */
269 pdev = NULL; 275 pdev = NULL;
270 /* pass back in pdev to account for multiple identical cards */
271 while ((pdev = 276 while ((pdev =
272 pci_get_subsys(pid->vendor, pid->device, pid->subvendor, 277 pci_get_subsys(pid->vendor, pid->device, pid->subvendor,
273 pid->subdevice, pdev)) != NULL) { 278 pid->subdevice, pdev)) != NULL) {
279 if ((pdev->class & pid->class_mask) != pid->class)
280 continue;
281
274 /* stealth mode requires a manual probe */ 282 /* stealth mode requires a manual probe */
275 pci_dev_get(pdev); 283 pci_dev_get(pdev);
276 drm_get_dev(pdev, pid, driver); 284 drm_get_dev(pdev, pid, driver);
@@ -297,6 +305,8 @@ static void drm_cleanup(struct drm_device * dev)
297 return; 305 return;
298 } 306 }
299 307
308 drm_vblank_cleanup(dev);
309
300 drm_lastclose(dev); 310 drm_lastclose(dev);
301 311
302 if (drm_core_has_MTRR(dev) && drm_core_has_AGP(dev) && 312 if (drm_core_has_MTRR(dev) && drm_core_has_AGP(dev) &&
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 212a94f715b2..1e787f894b3c 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -94,7 +94,7 @@ static void vblank_disable_fn(unsigned long arg)
94 } 94 }
95} 95}
96 96
97static void drm_vblank_cleanup(struct drm_device *dev) 97void drm_vblank_cleanup(struct drm_device *dev)
98{ 98{
99 /* Bail if the driver didn't call drm_vblank_init() */ 99 /* Bail if the driver didn't call drm_vblank_init() */
100 if (dev->num_crtcs == 0) 100 if (dev->num_crtcs == 0)
@@ -278,10 +278,6 @@ int drm_irq_uninstall(struct drm_device * dev)
278 278
279 free_irq(dev->pdev->irq, dev); 279 free_irq(dev->pdev->irq, dev);
280 280
281 drm_vblank_cleanup(dev);
282
283 dev->locked_tasklet_func = NULL;
284
285 return 0; 281 return 0;
286} 282}
287EXPORT_SYMBOL(drm_irq_uninstall); 283EXPORT_SYMBOL(drm_irq_uninstall);
@@ -699,81 +695,3 @@ void drm_handle_vblank(struct drm_device *dev, int crtc)
699 drm_vbl_send_signals(dev, crtc); 695 drm_vbl_send_signals(dev, crtc);
700} 696}
701EXPORT_SYMBOL(drm_handle_vblank); 697EXPORT_SYMBOL(drm_handle_vblank);
702
703/**
704 * Tasklet wrapper function.
705 *
706 * \param data DRM device in disguise.
707 *
708 * Attempts to grab the HW lock and calls the driver callback on success. On
709 * failure, leave the lock marked as contended so the callback can be called
710 * from drm_unlock().
711 */
712static void drm_locked_tasklet_func(unsigned long data)
713{
714 struct drm_device *dev = (struct drm_device *)data;
715 unsigned long irqflags;
716 void (*tasklet_func)(struct drm_device *);
717
718 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
719 tasklet_func = dev->locked_tasklet_func;
720 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
721
722 if (!tasklet_func ||
723 !drm_lock_take(&dev->lock,
724 DRM_KERNEL_CONTEXT)) {
725 return;
726 }
727
728 dev->lock.lock_time = jiffies;
729 atomic_inc(&dev->counts[_DRM_STAT_LOCKS]);
730
731 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
732 tasklet_func = dev->locked_tasklet_func;
733 dev->locked_tasklet_func = NULL;
734 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
735
736 if (tasklet_func != NULL)
737 tasklet_func(dev);
738
739 drm_lock_free(&dev->lock,
740 DRM_KERNEL_CONTEXT);
741}
742
743/**
744 * Schedule a tasklet to call back a driver hook with the HW lock held.
745 *
746 * \param dev DRM device.
747 * \param func Driver callback.
748 *
749 * This is intended for triggering actions that require the HW lock from an
750 * interrupt handler. The lock will be grabbed ASAP after the interrupt handler
751 * completes. Note that the callback may be called from interrupt or process
752 * context, it must not make any assumptions about this. Also, the HW lock will
753 * be held with the kernel context or any client context.
754 */
755void drm_locked_tasklet(struct drm_device *dev, void (*func)(struct drm_device *))
756{
757 unsigned long irqflags;
758 static DECLARE_TASKLET(drm_tasklet, drm_locked_tasklet_func, 0);
759
760 if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ) ||
761 test_bit(TASKLET_STATE_SCHED, &drm_tasklet.state))
762 return;
763
764 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
765
766 if (dev->locked_tasklet_func) {
767 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
768 return;
769 }
770
771 dev->locked_tasklet_func = func;
772
773 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
774
775 drm_tasklet.data = (unsigned long)dev;
776
777 tasklet_hi_schedule(&drm_tasklet);
778}
779EXPORT_SYMBOL(drm_locked_tasklet);
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index 888159e03d26..1cfa72031f8f 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -154,8 +154,6 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv)
154int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv) 154int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv)
155{ 155{
156 struct drm_lock *lock = data; 156 struct drm_lock *lock = data;
157 unsigned long irqflags;
158 void (*tasklet_func)(struct drm_device *);
159 157
160 if (lock->context == DRM_KERNEL_CONTEXT) { 158 if (lock->context == DRM_KERNEL_CONTEXT) {
161 DRM_ERROR("Process %d using kernel context %d\n", 159 DRM_ERROR("Process %d using kernel context %d\n",
@@ -163,13 +161,6 @@ int drm_unlock(struct drm_device *dev, void *data, struct drm_file *file_priv)
163 return -EINVAL; 161 return -EINVAL;
164 } 162 }
165 163
166 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
167 tasklet_func = dev->locked_tasklet_func;
168 dev->locked_tasklet_func = NULL;
169 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
170 if (tasklet_func != NULL)
171 tasklet_func(dev);
172
173 atomic_inc(&dev->counts[_DRM_STAT_UNLOCKS]); 164 atomic_inc(&dev->counts[_DRM_STAT_UNLOCKS]);
174 165
175 /* kernel_context_switch isn't used by any of the x86 drm 166 /* kernel_context_switch isn't used by any of the x86 drm
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index 141e33004a76..66c96ec66672 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -92,7 +92,6 @@ static int drm_fill_in_dev(struct drm_device * dev, struct pci_dev *pdev,
92 92
93 spin_lock_init(&dev->count_lock); 93 spin_lock_init(&dev->count_lock);
94 spin_lock_init(&dev->drw_lock); 94 spin_lock_init(&dev->drw_lock);
95 spin_lock_init(&dev->tasklet_lock);
96 spin_lock_init(&dev->lock.spinlock); 95 spin_lock_init(&dev->lock.spinlock);
97 init_timer(&dev->timer); 96 init_timer(&dev->timer);
98 mutex_init(&dev->struct_mutex); 97 mutex_init(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 5ba78e4fd2b5..d8fb5d8ee7ea 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,13 +3,14 @@
3# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. 3# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
4 4
5ccflags-y := -Iinclude/drm 5ccflags-y := -Iinclude/drm
6i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_opregion.o \ 6i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \
7 i915_suspend.o \ 7 i915_suspend.o \
8 i915_gem.o \ 8 i915_gem.o \
9 i915_gem_debug.o \ 9 i915_gem_debug.o \
10 i915_gem_proc.o \ 10 i915_gem_proc.o \
11 i915_gem_tiling.o 11 i915_gem_tiling.o
12 12
13i915-$(CONFIG_ACPI) += i915_opregion.o
13i915-$(CONFIG_COMPAT) += i915_ioc32.o 14i915-$(CONFIG_COMPAT) += i915_ioc32.o
14 15
15obj-$(CONFIG_DRM_I915) += i915.o 16obj-$(CONFIG_DRM_I915) += i915.o
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 01de536e0211..553dd4bc3075 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -154,6 +154,9 @@ static int i915_dma_cleanup(struct drm_device * dev)
154 if (I915_NEED_GFX_HWS(dev)) 154 if (I915_NEED_GFX_HWS(dev))
155 i915_free_hws(dev); 155 i915_free_hws(dev);
156 156
157 dev_priv->sarea = NULL;
158 dev_priv->sarea_priv = NULL;
159
157 return 0; 160 return 0;
158} 161}
159 162
@@ -442,7 +445,7 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
442 445
443 BEGIN_LP_RING(4); 446 BEGIN_LP_RING(4);
444 OUT_RING(MI_STORE_DWORD_INDEX); 447 OUT_RING(MI_STORE_DWORD_INDEX);
445 OUT_RING(5 << MI_STORE_DWORD_INDEX_SHIFT); 448 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
446 OUT_RING(dev_priv->counter); 449 OUT_RING(dev_priv->counter);
447 OUT_RING(0); 450 OUT_RING(0);
448 ADVANCE_LP_RING(); 451 ADVANCE_LP_RING();
@@ -573,7 +576,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
573 576
574 BEGIN_LP_RING(4); 577 BEGIN_LP_RING(4);
575 OUT_RING(MI_STORE_DWORD_INDEX); 578 OUT_RING(MI_STORE_DWORD_INDEX);
576 OUT_RING(5 << MI_STORE_DWORD_INDEX_SHIFT); 579 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
577 OUT_RING(dev_priv->counter); 580 OUT_RING(dev_priv->counter);
578 OUT_RING(0); 581 OUT_RING(0);
579 ADVANCE_LP_RING(); 582 ADVANCE_LP_RING();
@@ -608,7 +611,6 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
608 struct drm_file *file_priv) 611 struct drm_file *file_priv)
609{ 612{
610 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 613 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
611 u32 *hw_status = dev_priv->hw_status_page;
612 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) 614 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
613 dev_priv->sarea_priv; 615 dev_priv->sarea_priv;
614 drm_i915_batchbuffer_t *batch = data; 616 drm_i915_batchbuffer_t *batch = data;
@@ -634,7 +636,7 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
634 mutex_unlock(&dev->struct_mutex); 636 mutex_unlock(&dev->struct_mutex);
635 637
636 if (sarea_priv) 638 if (sarea_priv)
637 sarea_priv->last_dispatch = (int)hw_status[5]; 639 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
638 return ret; 640 return ret;
639} 641}
640 642
@@ -642,7 +644,6 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
642 struct drm_file *file_priv) 644 struct drm_file *file_priv)
643{ 645{
644 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 646 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
645 u32 *hw_status = dev_priv->hw_status_page;
646 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *) 647 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
647 dev_priv->sarea_priv; 648 dev_priv->sarea_priv;
648 drm_i915_cmdbuffer_t *cmdbuf = data; 649 drm_i915_cmdbuffer_t *cmdbuf = data;
@@ -670,7 +671,7 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
670 } 671 }
671 672
672 if (sarea_priv) 673 if (sarea_priv)
673 sarea_priv->last_dispatch = (int)hw_status[5]; 674 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
674 return 0; 675 return 0;
675} 676}
676 677
@@ -846,16 +847,23 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
846 * and the registers being closely associated. 847 * and the registers being closely associated.
847 * 848 *
848 * According to chipset errata, on the 965GM, MSI interrupts may 849 * According to chipset errata, on the 965GM, MSI interrupts may
849 * be lost or delayed 850 * be lost or delayed, but we use them anyways to avoid
851 * stuck interrupts on some machines.
850 */ 852 */
851 if (!IS_I945G(dev) && !IS_I945GM(dev) && !IS_I965GM(dev)) 853 if (!IS_I945G(dev) && !IS_I945GM(dev))
852 if (pci_enable_msi(dev->pdev)) 854 pci_enable_msi(dev->pdev);
853 DRM_ERROR("failed to enable MSI\n");
854 855
855 intel_opregion_init(dev); 856 intel_opregion_init(dev);
856 857
857 spin_lock_init(&dev_priv->user_irq_lock); 858 spin_lock_init(&dev_priv->user_irq_lock);
858 859
860 ret = drm_vblank_init(dev, I915_NUM_PIPE);
861
862 if (ret) {
863 (void) i915_driver_unload(dev);
864 return ret;
865 }
866
859 return ret; 867 return ret;
860} 868}
861 869
@@ -960,6 +968,7 @@ struct drm_ioctl_desc i915_ioctls[] = {
960 DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0), 968 DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0),
961 DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0), 969 DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0),
962 DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0), 970 DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0),
971 DRM_IOCTL_DEF(DRM_I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, 0),
963}; 972};
964 973
965int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls); 974int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index f20ffe17df71..adc972cc6bfc 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -31,6 +31,7 @@
31#define _I915_DRV_H_ 31#define _I915_DRV_H_
32 32
33#include "i915_reg.h" 33#include "i915_reg.h"
34#include <linux/io-mapping.h>
34 35
35/* General customization: 36/* General customization:
36 */ 37 */
@@ -46,6 +47,8 @@ enum pipe {
46 PIPE_B, 47 PIPE_B,
47}; 48};
48 49
50#define I915_NUM_PIPE 2
51
49/* Interface history: 52/* Interface history:
50 * 53 *
51 * 1.1: Original. 54 * 1.1: Original.
@@ -87,13 +90,6 @@ struct mem_block {
87 struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */ 90 struct drm_file *file_priv; /* NULL: free, -1: heap, other: real files */
88}; 91};
89 92
90typedef struct _drm_i915_vbl_swap {
91 struct list_head head;
92 drm_drawable_t drw_id;
93 unsigned int pipe;
94 unsigned int sequence;
95} drm_i915_vbl_swap_t;
96
97struct opregion_header; 93struct opregion_header;
98struct opregion_acpi; 94struct opregion_acpi;
99struct opregion_swsci; 95struct opregion_swsci;
@@ -138,6 +134,7 @@ typedef struct drm_i915_private {
138 int user_irq_refcount; 134 int user_irq_refcount;
139 /** Cached value of IMR to avoid reads in updating the bitfield */ 135 /** Cached value of IMR to avoid reads in updating the bitfield */
140 u32 irq_mask_reg; 136 u32 irq_mask_reg;
137 u32 pipestat[2];
141 138
142 int tex_lru_log_granularity; 139 int tex_lru_log_granularity;
143 int allow_batchbuffer; 140 int allow_batchbuffer;
@@ -145,10 +142,6 @@ typedef struct drm_i915_private {
145 unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds; 142 unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
146 int vblank_pipe; 143 int vblank_pipe;
147 144
148 spinlock_t swaps_lock;
149 drm_i915_vbl_swap_t vbl_swaps;
150 unsigned int swaps_pending;
151
152 struct intel_opregion opregion; 145 struct intel_opregion opregion;
153 146
154 /* Register state */ 147 /* Register state */
@@ -156,6 +149,8 @@ typedef struct drm_i915_private {
156 u32 saveDSPACNTR; 149 u32 saveDSPACNTR;
157 u32 saveDSPBCNTR; 150 u32 saveDSPBCNTR;
158 u32 saveDSPARB; 151 u32 saveDSPARB;
152 u32 saveRENDERSTANDBY;
153 u32 saveHWS;
159 u32 savePIPEACONF; 154 u32 savePIPEACONF;
160 u32 savePIPEBCONF; 155 u32 savePIPEBCONF;
161 u32 savePIPEASRC; 156 u32 savePIPEASRC;
@@ -240,16 +235,19 @@ typedef struct drm_i915_private {
240 u8 saveDACDATA[256*3]; /* 256 3-byte colors */ 235 u8 saveDACDATA[256*3]; /* 256 3-byte colors */
241 u8 saveCR[37]; 236 u8 saveCR[37];
242 237
243 /** Work task for vblank-related ring access */
244 struct work_struct vblank_work;
245
246 struct { 238 struct {
247 struct drm_mm gtt_space; 239 struct drm_mm gtt_space;
248 240
241 struct io_mapping *gtt_mapping;
242
249 /** 243 /**
250 * List of objects currently involved in rendering from the 244 * List of objects currently involved in rendering from the
251 * ringbuffer. 245 * ringbuffer.
252 * 246 *
247 * Includes buffers having the contents of their GPU caches
248 * flushed, not necessarily primitives. last_rendering_seqno
249 * represents when the rendering involved will be completed.
250 *
253 * A reference is held on the buffer while on this list. 251 * A reference is held on the buffer while on this list.
254 */ 252 */
255 struct list_head active_list; 253 struct list_head active_list;
@@ -259,6 +257,8 @@ typedef struct drm_i915_private {
259 * still have a write_domain which needs to be flushed before 257 * still have a write_domain which needs to be flushed before
260 * unbinding. 258 * unbinding.
261 * 259 *
260 * last_rendering_seqno is 0 while an object is in this list.
261 *
262 * A reference is held on the buffer while on this list. 262 * A reference is held on the buffer while on this list.
263 */ 263 */
264 struct list_head flushing_list; 264 struct list_head flushing_list;
@@ -267,6 +267,8 @@ typedef struct drm_i915_private {
267 * LRU list of objects which are not in the ringbuffer and 267 * LRU list of objects which are not in the ringbuffer and
268 * are ready to unbind, but are still in the GTT. 268 * are ready to unbind, but are still in the GTT.
269 * 269 *
270 * last_rendering_seqno is 0 while an object is in this list.
271 *
270 * A reference is not held on the buffer while on this list, 272 * A reference is not held on the buffer while on this list,
271 * as merely being GTT-bound shouldn't prevent its being 273 * as merely being GTT-bound shouldn't prevent its being
272 * freed, and we'll pull it off the list in the free path. 274 * freed, and we'll pull it off the list in the free path.
@@ -377,8 +379,8 @@ struct drm_i915_gem_object {
377 uint32_t agp_type; 379 uint32_t agp_type;
378 380
379 /** 381 /**
380 * Flagging of which individual pages are valid in GEM_DOMAIN_CPU when 382 * If present, while GEM_DOMAIN_CPU is in the read domain this array
381 * GEM_DOMAIN_CPU is not in the object's read domain. 383 * flags which individual pages are valid.
382 */ 384 */
383 uint8_t *page_cpu_valid; 385 uint8_t *page_cpu_valid;
384}; 386};
@@ -400,9 +402,6 @@ struct drm_i915_gem_request {
400 /** Time at which this request was emitted, in jiffies. */ 402 /** Time at which this request was emitted, in jiffies. */
401 unsigned long emitted_jiffies; 403 unsigned long emitted_jiffies;
402 404
403 /** Cache domains that were flushed at the start of the request. */
404 uint32_t flush_domains;
405
406 struct list_head list; 405 struct list_head list;
407}; 406};
408 407
@@ -441,7 +440,6 @@ extern int i915_irq_wait(struct drm_device *dev, void *data,
441void i915_user_irq_get(struct drm_device *dev); 440void i915_user_irq_get(struct drm_device *dev);
442void i915_user_irq_put(struct drm_device *dev); 441void i915_user_irq_put(struct drm_device *dev);
443 442
444extern void i915_vblank_work_handler(struct work_struct *work);
445extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS); 443extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS);
446extern void i915_driver_irq_preinstall(struct drm_device * dev); 444extern void i915_driver_irq_preinstall(struct drm_device * dev);
447extern int i915_driver_irq_postinstall(struct drm_device *dev); 445extern int i915_driver_irq_postinstall(struct drm_device *dev);
@@ -457,6 +455,13 @@ extern int i915_vblank_swap(struct drm_device *dev, void *data,
457 struct drm_file *file_priv); 455 struct drm_file *file_priv);
458extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask); 456extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask);
459 457
458void
459i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
460
461void
462i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
463
464
460/* i915_mem.c */ 465/* i915_mem.c */
461extern int i915_mem_alloc(struct drm_device *dev, void *data, 466extern int i915_mem_alloc(struct drm_device *dev, void *data,
462 struct drm_file *file_priv); 467 struct drm_file *file_priv);
@@ -502,6 +507,8 @@ int i915_gem_set_tiling(struct drm_device *dev, void *data,
502 struct drm_file *file_priv); 507 struct drm_file *file_priv);
503int i915_gem_get_tiling(struct drm_device *dev, void *data, 508int i915_gem_get_tiling(struct drm_device *dev, void *data,
504 struct drm_file *file_priv); 509 struct drm_file *file_priv);
510int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
511 struct drm_file *file_priv);
505void i915_gem_load(struct drm_device *dev); 512void i915_gem_load(struct drm_device *dev);
506int i915_gem_proc_init(struct drm_minor *minor); 513int i915_gem_proc_init(struct drm_minor *minor);
507void i915_gem_proc_cleanup(struct drm_minor *minor); 514void i915_gem_proc_cleanup(struct drm_minor *minor);
@@ -539,11 +546,18 @@ extern int i915_restore_state(struct drm_device *dev);
539extern int i915_save_state(struct drm_device *dev); 546extern int i915_save_state(struct drm_device *dev);
540extern int i915_restore_state(struct drm_device *dev); 547extern int i915_restore_state(struct drm_device *dev);
541 548
549#ifdef CONFIG_ACPI
542/* i915_opregion.c */ 550/* i915_opregion.c */
543extern int intel_opregion_init(struct drm_device *dev); 551extern int intel_opregion_init(struct drm_device *dev);
544extern void intel_opregion_free(struct drm_device *dev); 552extern void intel_opregion_free(struct drm_device *dev);
545extern void opregion_asle_intr(struct drm_device *dev); 553extern void opregion_asle_intr(struct drm_device *dev);
546extern void opregion_enable_asle(struct drm_device *dev); 554extern void opregion_enable_asle(struct drm_device *dev);
555#else
556static inline int intel_opregion_init(struct drm_device *dev) { return 0; }
557static inline void intel_opregion_free(struct drm_device *dev) { return; }
558static inline void opregion_asle_intr(struct drm_device *dev) { return; }
559static inline void opregion_enable_asle(struct drm_device *dev) { return; }
560#endif
547 561
548/** 562/**
549 * Lock test for when it's just for synchronization of ring access. 563 * Lock test for when it's just for synchronization of ring access.
@@ -610,8 +624,9 @@ extern void opregion_enable_asle(struct drm_device *dev);
610 * The area from dword 0x20 to 0x3ff is available for driver usage. 624 * The area from dword 0x20 to 0x3ff is available for driver usage.
611 */ 625 */
612#define READ_HWSP(dev_priv, reg) (((volatile u32*)(dev_priv->hw_status_page))[reg]) 626#define READ_HWSP(dev_priv, reg) (((volatile u32*)(dev_priv->hw_status_page))[reg])
613#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, 5) 627#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
614#define I915_GEM_HWS_INDEX 0x20 628#define I915_GEM_HWS_INDEX 0x20
629#define I915_BREADCRUMB_INDEX 0x21
615 630
616extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller); 631extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
617 632
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 17ae330ff269..ad672d854828 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -31,21 +31,23 @@
31#include "i915_drv.h" 31#include "i915_drv.h"
32#include <linux/swap.h> 32#include <linux/swap.h>
33 33
34static int 34#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
35i915_gem_object_set_domain(struct drm_gem_object *obj, 35
36 uint32_t read_domains, 36static void
37 uint32_t write_domain); 37i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
38static int 38 uint32_t read_domains,
39i915_gem_object_set_domain_range(struct drm_gem_object *obj, 39 uint32_t write_domain);
40 uint64_t offset, 40static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
41 uint64_t size, 41static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
42 uint32_t read_domains, 42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
43 uint32_t write_domain); 43static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
44static int 44 int write);
45i915_gem_set_domain(struct drm_gem_object *obj, 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 struct drm_file *file_priv, 46 int write);
47 uint32_t read_domains, 47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint32_t write_domain); 48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
49static int i915_gem_object_get_page_list(struct drm_gem_object *obj); 51static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
50static void i915_gem_object_free_page_list(struct drm_gem_object *obj); 52static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); 53static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
@@ -79,6 +81,22 @@ i915_gem_init_ioctl(struct drm_device *dev, void *data,
79 return 0; 81 return 0;
80} 82}
81 83
84int
85i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
86 struct drm_file *file_priv)
87{
88 struct drm_i915_gem_get_aperture *args = data;
89
90 if (!(dev->driver->driver_features & DRIVER_GEM))
91 return -ENODEV;
92
93 args->aper_size = dev->gtt_total;
94 args->aper_available_size = (args->aper_size -
95 atomic_read(&dev->pin_memory));
96
97 return 0;
98}
99
82 100
83/** 101/**
84 * Creates a new mm object and returns a handle to it. 102 * Creates a new mm object and returns a handle to it.
@@ -144,8 +162,8 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
144 162
145 mutex_lock(&dev->struct_mutex); 163 mutex_lock(&dev->struct_mutex);
146 164
147 ret = i915_gem_object_set_domain_range(obj, args->offset, args->size, 165 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
148 I915_GEM_DOMAIN_CPU, 0); 166 args->size);
149 if (ret != 0) { 167 if (ret != 0) {
150 drm_gem_object_unreference(obj); 168 drm_gem_object_unreference(obj);
151 mutex_unlock(&dev->struct_mutex); 169 mutex_unlock(&dev->struct_mutex);
@@ -171,35 +189,50 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
171 return 0; 189 return 0;
172} 190}
173 191
174/* 192/* This is the fast write path which cannot handle
175 * Try to write quickly with an atomic kmap. Return true on success. 193 * page faults in the source data
176 *
177 * If this fails (which includes a partial write), we'll redo the whole
178 * thing with the slow version.
179 *
180 * This is a workaround for the low performance of iounmap (approximate
181 * 10% cpu cost on normal 3D workloads). kmap_atomic on HIGHMEM kernels
182 * happens to let us map card memory without taking IPIs. When the vmap
183 * rework lands we should be able to dump this hack.
184 */ 194 */
185static inline int fast_user_write(unsigned long pfn, char __user *user_data, 195
186 int l, int o) 196static inline int
197fast_user_write(struct io_mapping *mapping,
198 loff_t page_base, int page_offset,
199 char __user *user_data,
200 int length)
187{ 201{
188#ifdef CONFIG_HIGHMEM
189 unsigned long unwritten;
190 char *vaddr_atomic; 202 char *vaddr_atomic;
203 unsigned long unwritten;
191 204
192 vaddr_atomic = kmap_atomic_pfn(pfn, KM_USER0); 205 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
193#if WATCH_PWRITE 206 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
194 DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n", 207 user_data, length);
195 i, o, l, pfn, vaddr_atomic); 208 io_mapping_unmap_atomic(vaddr_atomic);
196#endif 209 if (unwritten)
197 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + o, user_data, l); 210 return -EFAULT;
198 kunmap_atomic(vaddr_atomic, KM_USER0); 211 return 0;
199 return !unwritten; 212}
200#else 213
214/* Here's the write path which can sleep for
215 * page faults
216 */
217
218static inline int
219slow_user_write(struct io_mapping *mapping,
220 loff_t page_base, int page_offset,
221 char __user *user_data,
222 int length)
223{
224 char __iomem *vaddr;
225 unsigned long unwritten;
226
227 vaddr = io_mapping_map_wc(mapping, page_base);
228 if (vaddr == NULL)
229 return -EFAULT;
230 unwritten = __copy_from_user(vaddr + page_offset,
231 user_data, length);
232 io_mapping_unmap(vaddr);
233 if (unwritten)
234 return -EFAULT;
201 return 0; 235 return 0;
202#endif
203} 236}
204 237
205static int 238static int
@@ -208,10 +241,12 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
208 struct drm_file *file_priv) 241 struct drm_file *file_priv)
209{ 242{
210 struct drm_i915_gem_object *obj_priv = obj->driver_private; 243 struct drm_i915_gem_object *obj_priv = obj->driver_private;
244 drm_i915_private_t *dev_priv = dev->dev_private;
211 ssize_t remain; 245 ssize_t remain;
212 loff_t offset; 246 loff_t offset, page_base;
213 char __user *user_data; 247 char __user *user_data;
214 int ret = 0; 248 int page_offset, page_length;
249 int ret;
215 250
216 user_data = (char __user *) (uintptr_t) args->data_ptr; 251 user_data = (char __user *) (uintptr_t) args->data_ptr;
217 remain = args->size; 252 remain = args->size;
@@ -225,8 +260,7 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
225 mutex_unlock(&dev->struct_mutex); 260 mutex_unlock(&dev->struct_mutex);
226 return ret; 261 return ret;
227 } 262 }
228 ret = i915_gem_set_domain(obj, file_priv, 263 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
229 I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
230 if (ret) 264 if (ret)
231 goto fail; 265 goto fail;
232 266
@@ -235,57 +269,37 @@ i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
235 obj_priv->dirty = 1; 269 obj_priv->dirty = 1;
236 270
237 while (remain > 0) { 271 while (remain > 0) {
238 unsigned long pfn;
239 int i, o, l;
240
241 /* Operation in this page 272 /* Operation in this page
242 * 273 *
243 * i = page number 274 * page_base = page offset within aperture
244 * o = offset within page 275 * page_offset = offset within page
245 * l = bytes to copy 276 * page_length = bytes to copy for this page
246 */ 277 */
247 i = offset >> PAGE_SHIFT; 278 page_base = (offset & ~(PAGE_SIZE-1));
248 o = offset & (PAGE_SIZE-1); 279 page_offset = offset & (PAGE_SIZE-1);
249 l = remain; 280 page_length = remain;
250 if ((o + l) > PAGE_SIZE) 281 if ((page_offset + remain) > PAGE_SIZE)
251 l = PAGE_SIZE - o; 282 page_length = PAGE_SIZE - page_offset;
252 283
253 pfn = (dev->agp->base >> PAGE_SHIFT) + i; 284 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
254 285 page_offset, user_data, page_length);
255 if (!fast_user_write(pfn, user_data, l, o)) { 286
256 unsigned long unwritten; 287 /* If we get a fault while copying data, then (presumably) our
257 char __iomem *vaddr; 288 * source page isn't available. In this case, use the
258 289 * non-atomic function
259 vaddr = ioremap_wc(pfn << PAGE_SHIFT, PAGE_SIZE); 290 */
260#if WATCH_PWRITE 291 if (ret) {
261 DRM_INFO("pwrite slow i %d o %d l %d " 292 ret = slow_user_write (dev_priv->mm.gtt_mapping,
262 "pfn %ld vaddr %p\n", 293 page_base, page_offset,
263 i, o, l, pfn, vaddr); 294 user_data, page_length);
264#endif 295 if (ret)
265 if (vaddr == NULL) {
266 ret = -EFAULT;
267 goto fail;
268 }
269 unwritten = __copy_from_user(vaddr + o, user_data, l);
270#if WATCH_PWRITE
271 DRM_INFO("unwritten %ld\n", unwritten);
272#endif
273 iounmap(vaddr);
274 if (unwritten) {
275 ret = -EFAULT;
276 goto fail; 296 goto fail;
277 }
278 } 297 }
279 298
280 remain -= l; 299 remain -= page_length;
281 user_data += l; 300 user_data += page_length;
282 offset += l; 301 offset += page_length;
283 } 302 }
284#if WATCH_PWRITE && 1
285 i915_gem_clflush_object(obj);
286 i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
287 i915_gem_clflush_object(obj);
288#endif
289 303
290fail: 304fail:
291 i915_gem_object_unpin(obj); 305 i915_gem_object_unpin(obj);
@@ -305,8 +319,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
305 319
306 mutex_lock(&dev->struct_mutex); 320 mutex_lock(&dev->struct_mutex);
307 321
308 ret = i915_gem_set_domain(obj, file_priv, 322 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
309 I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
310 if (ret) { 323 if (ret) {
311 mutex_unlock(&dev->struct_mutex); 324 mutex_unlock(&dev->struct_mutex);
312 return ret; 325 return ret;
@@ -382,7 +395,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
382} 395}
383 396
384/** 397/**
385 * Called when user space prepares to use an object 398 * Called when user space prepares to use an object with the CPU, either
399 * through the mmap ioctl's mapping or a GTT mapping.
386 */ 400 */
387int 401int
388i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 402i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
@@ -390,11 +404,26 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
390{ 404{
391 struct drm_i915_gem_set_domain *args = data; 405 struct drm_i915_gem_set_domain *args = data;
392 struct drm_gem_object *obj; 406 struct drm_gem_object *obj;
407 uint32_t read_domains = args->read_domains;
408 uint32_t write_domain = args->write_domain;
393 int ret; 409 int ret;
394 410
395 if (!(dev->driver->driver_features & DRIVER_GEM)) 411 if (!(dev->driver->driver_features & DRIVER_GEM))
396 return -ENODEV; 412 return -ENODEV;
397 413
414 /* Only handle setting domains to types used by the CPU. */
415 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
416 return -EINVAL;
417
418 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
419 return -EINVAL;
420
421 /* Having something in the write domain implies it's in the read
422 * domain, and only that read domain. Enforce that in the request.
423 */
424 if (write_domain != 0 && read_domains != write_domain)
425 return -EINVAL;
426
398 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 427 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
399 if (obj == NULL) 428 if (obj == NULL)
400 return -EBADF; 429 return -EBADF;
@@ -402,10 +431,21 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
402 mutex_lock(&dev->struct_mutex); 431 mutex_lock(&dev->struct_mutex);
403#if WATCH_BUF 432#if WATCH_BUF
404 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n", 433 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
405 obj, obj->size, args->read_domains, args->write_domain); 434 obj, obj->size, read_domains, write_domain);
406#endif 435#endif
407 ret = i915_gem_set_domain(obj, file_priv, 436 if (read_domains & I915_GEM_DOMAIN_GTT) {
408 args->read_domains, args->write_domain); 437 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
438
439 /* Silently promote "you're not bound, there was nothing to do"
440 * to success, since the client was just asking us to
441 * make sure everything was done.
442 */
443 if (ret == -EINVAL)
444 ret = 0;
445 } else {
446 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
447 }
448
409 drm_gem_object_unreference(obj); 449 drm_gem_object_unreference(obj);
410 mutex_unlock(&dev->struct_mutex); 450 mutex_unlock(&dev->struct_mutex);
411 return ret; 451 return ret;
@@ -440,10 +480,9 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
440 obj_priv = obj->driver_private; 480 obj_priv = obj->driver_private;
441 481
442 /* Pinned buffers may be scanout, so flush the cache */ 482 /* Pinned buffers may be scanout, so flush the cache */
443 if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) { 483 if (obj_priv->pin_count)
444 i915_gem_clflush_object(obj); 484 i915_gem_object_flush_cpu_write_domain(obj);
445 drm_agp_chipset_flush(dev); 485
446 }
447 drm_gem_object_unreference(obj); 486 drm_gem_object_unreference(obj);
448 mutex_unlock(&dev->struct_mutex); 487 mutex_unlock(&dev->struct_mutex);
449 return ret; 488 return ret;
@@ -517,7 +556,7 @@ i915_gem_object_free_page_list(struct drm_gem_object *obj)
517} 556}
518 557
519static void 558static void
520i915_gem_object_move_to_active(struct drm_gem_object *obj) 559i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
521{ 560{
522 struct drm_device *dev = obj->dev; 561 struct drm_device *dev = obj->dev;
523 drm_i915_private_t *dev_priv = dev->dev_private; 562 drm_i915_private_t *dev_priv = dev->dev_private;
@@ -531,8 +570,20 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj)
531 /* Move from whatever list we were on to the tail of execution. */ 570 /* Move from whatever list we were on to the tail of execution. */
532 list_move_tail(&obj_priv->list, 571 list_move_tail(&obj_priv->list,
533 &dev_priv->mm.active_list); 572 &dev_priv->mm.active_list);
573 obj_priv->last_rendering_seqno = seqno;
534} 574}
535 575
576static void
577i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
578{
579 struct drm_device *dev = obj->dev;
580 drm_i915_private_t *dev_priv = dev->dev_private;
581 struct drm_i915_gem_object *obj_priv = obj->driver_private;
582
583 BUG_ON(!obj_priv->active);
584 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
585 obj_priv->last_rendering_seqno = 0;
586}
536 587
537static void 588static void
538i915_gem_object_move_to_inactive(struct drm_gem_object *obj) 589i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
@@ -547,6 +598,7 @@ i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
547 else 598 else
548 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 599 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
549 600
601 obj_priv->last_rendering_seqno = 0;
550 if (obj_priv->active) { 602 if (obj_priv->active) {
551 obj_priv->active = 0; 603 obj_priv->active = 0;
552 drm_gem_object_unreference(obj); 604 drm_gem_object_unreference(obj);
@@ -595,10 +647,28 @@ i915_add_request(struct drm_device *dev, uint32_t flush_domains)
595 647
596 request->seqno = seqno; 648 request->seqno = seqno;
597 request->emitted_jiffies = jiffies; 649 request->emitted_jiffies = jiffies;
598 request->flush_domains = flush_domains;
599 was_empty = list_empty(&dev_priv->mm.request_list); 650 was_empty = list_empty(&dev_priv->mm.request_list);
600 list_add_tail(&request->list, &dev_priv->mm.request_list); 651 list_add_tail(&request->list, &dev_priv->mm.request_list);
601 652
653 /* Associate any objects on the flushing list matching the write
654 * domain we're flushing with our flush.
655 */
656 if (flush_domains != 0) {
657 struct drm_i915_gem_object *obj_priv, *next;
658
659 list_for_each_entry_safe(obj_priv, next,
660 &dev_priv->mm.flushing_list, list) {
661 struct drm_gem_object *obj = obj_priv->obj;
662
663 if ((obj->write_domain & flush_domains) ==
664 obj->write_domain) {
665 obj->write_domain = 0;
666 i915_gem_object_move_to_active(obj, seqno);
667 }
668 }
669
670 }
671
602 if (was_empty && !dev_priv->mm.suspended) 672 if (was_empty && !dev_priv->mm.suspended)
603 schedule_delayed_work(&dev_priv->mm.retire_work, HZ); 673 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
604 return seqno; 674 return seqno;
@@ -661,30 +731,10 @@ i915_gem_retire_request(struct drm_device *dev,
661 __func__, request->seqno, obj); 731 __func__, request->seqno, obj);
662#endif 732#endif
663 733
664 if (obj->write_domain != 0) { 734 if (obj->write_domain != 0)
665 list_move_tail(&obj_priv->list, 735 i915_gem_object_move_to_flushing(obj);
666 &dev_priv->mm.flushing_list); 736 else
667 } else {
668 i915_gem_object_move_to_inactive(obj); 737 i915_gem_object_move_to_inactive(obj);
669 }
670 }
671
672 if (request->flush_domains != 0) {
673 struct drm_i915_gem_object *obj_priv, *next;
674
675 /* Clear the write domain and activity from any buffers
676 * that are just waiting for a flush matching the one retired.
677 */
678 list_for_each_entry_safe(obj_priv, next,
679 &dev_priv->mm.flushing_list, list) {
680 struct drm_gem_object *obj = obj_priv->obj;
681
682 if (obj->write_domain & request->flush_domains) {
683 obj->write_domain = 0;
684 i915_gem_object_move_to_inactive(obj);
685 }
686 }
687
688 } 738 }
689} 739}
690 740
@@ -877,25 +927,10 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj)
877 struct drm_i915_gem_object *obj_priv = obj->driver_private; 927 struct drm_i915_gem_object *obj_priv = obj->driver_private;
878 int ret; 928 int ret;
879 929
880 /* If there are writes queued to the buffer, flush and 930 /* This function only exists to support waiting for existing rendering,
881 * create a new seqno to wait for. 931 * not for emitting required flushes.
882 */ 932 */
883 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) { 933 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
884 uint32_t write_domain = obj->write_domain;
885#if WATCH_BUF
886 DRM_INFO("%s: flushing object %p from write domain %08x\n",
887 __func__, obj, write_domain);
888#endif
889 i915_gem_flush(dev, 0, write_domain);
890
891 i915_gem_object_move_to_active(obj);
892 obj_priv->last_rendering_seqno = i915_add_request(dev,
893 write_domain);
894 BUG_ON(obj_priv->last_rendering_seqno == 0);
895#if WATCH_LRU
896 DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
897#endif
898 }
899 934
900 /* If there is rendering queued on the buffer being evicted, wait for 935 /* If there is rendering queued on the buffer being evicted, wait for
901 * it. 936 * it.
@@ -935,24 +970,16 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
935 return -EINVAL; 970 return -EINVAL;
936 } 971 }
937 972
938 /* Wait for any rendering to complete
939 */
940 ret = i915_gem_object_wait_rendering(obj);
941 if (ret) {
942 DRM_ERROR("wait_rendering failed: %d\n", ret);
943 return ret;
944 }
945
946 /* Move the object to the CPU domain to ensure that 973 /* Move the object to the CPU domain to ensure that
947 * any possible CPU writes while it's not in the GTT 974 * any possible CPU writes while it's not in the GTT
948 * are flushed when we go to remap it. This will 975 * are flushed when we go to remap it. This will
949 * also ensure that all pending GPU writes are finished 976 * also ensure that all pending GPU writes are finished
950 * before we unbind. 977 * before we unbind.
951 */ 978 */
952 ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU, 979 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
953 I915_GEM_DOMAIN_CPU);
954 if (ret) { 980 if (ret) {
955 DRM_ERROR("set_domain failed: %d\n", ret); 981 if (ret != -ERESTARTSYS)
982 DRM_ERROR("set_domain failed: %d\n", ret);
956 return ret; 983 return ret;
957 } 984 }
958 985
@@ -1068,6 +1095,21 @@ i915_gem_evict_something(struct drm_device *dev)
1068} 1095}
1069 1096
1070static int 1097static int
1098i915_gem_evict_everything(struct drm_device *dev)
1099{
1100 int ret;
1101
1102 for (;;) {
1103 ret = i915_gem_evict_something(dev);
1104 if (ret != 0)
1105 break;
1106 }
1107 if (ret == -ENOMEM)
1108 return 0;
1109 return ret;
1110}
1111
1112static int
1071i915_gem_object_get_page_list(struct drm_gem_object *obj) 1113i915_gem_object_get_page_list(struct drm_gem_object *obj)
1072{ 1114{
1073 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1115 struct drm_i915_gem_object *obj_priv = obj->driver_private;
@@ -1153,7 +1195,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1153 1195
1154 ret = i915_gem_evict_something(dev); 1196 ret = i915_gem_evict_something(dev);
1155 if (ret != 0) { 1197 if (ret != 0) {
1156 DRM_ERROR("Failed to evict a buffer %d\n", ret); 1198 if (ret != -ERESTARTSYS)
1199 DRM_ERROR("Failed to evict a buffer %d\n", ret);
1157 return ret; 1200 return ret;
1158 } 1201 }
1159 goto search_free; 1202 goto search_free;
@@ -1213,6 +1256,143 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
1213 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE); 1256 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1214} 1257}
1215 1258
1259/** Flushes any GPU write domain for the object if it's dirty. */
1260static void
1261i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
1262{
1263 struct drm_device *dev = obj->dev;
1264 uint32_t seqno;
1265
1266 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
1267 return;
1268
1269 /* Queue the GPU write cache flushing we need. */
1270 i915_gem_flush(dev, 0, obj->write_domain);
1271 seqno = i915_add_request(dev, obj->write_domain);
1272 obj->write_domain = 0;
1273 i915_gem_object_move_to_active(obj, seqno);
1274}
1275
1276/** Flushes the GTT write domain for the object if it's dirty. */
1277static void
1278i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
1279{
1280 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
1281 return;
1282
1283 /* No actual flushing is required for the GTT write domain. Writes
1284 * to it immediately go to main memory as far as we know, so there's
1285 * no chipset flush. It also doesn't land in render cache.
1286 */
1287 obj->write_domain = 0;
1288}
1289
1290/** Flushes the CPU write domain for the object if it's dirty. */
1291static void
1292i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
1293{
1294 struct drm_device *dev = obj->dev;
1295
1296 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
1297 return;
1298
1299 i915_gem_clflush_object(obj);
1300 drm_agp_chipset_flush(dev);
1301 obj->write_domain = 0;
1302}
1303
1304/**
1305 * Moves a single object to the GTT read, and possibly write domain.
1306 *
1307 * This function returns when the move is complete, including waiting on
1308 * flushes to occur.
1309 */
1310static int
1311i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
1312{
1313 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1314 int ret;
1315
1316 /* Not valid to be called on unbound objects. */
1317 if (obj_priv->gtt_space == NULL)
1318 return -EINVAL;
1319
1320 i915_gem_object_flush_gpu_write_domain(obj);
1321 /* Wait on any GPU rendering and flushing to occur. */
1322 ret = i915_gem_object_wait_rendering(obj);
1323 if (ret != 0)
1324 return ret;
1325
1326 /* If we're writing through the GTT domain, then CPU and GPU caches
1327 * will need to be invalidated at next use.
1328 */
1329 if (write)
1330 obj->read_domains &= I915_GEM_DOMAIN_GTT;
1331
1332 i915_gem_object_flush_cpu_write_domain(obj);
1333
1334 /* It should now be out of any other write domains, and we can update
1335 * the domain values for our changes.
1336 */
1337 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
1338 obj->read_domains |= I915_GEM_DOMAIN_GTT;
1339 if (write) {
1340 obj->write_domain = I915_GEM_DOMAIN_GTT;
1341 obj_priv->dirty = 1;
1342 }
1343
1344 return 0;
1345}
1346
1347/**
1348 * Moves a single object to the CPU read, and possibly write domain.
1349 *
1350 * This function returns when the move is complete, including waiting on
1351 * flushes to occur.
1352 */
1353static int
1354i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
1355{
1356 struct drm_device *dev = obj->dev;
1357 int ret;
1358
1359 i915_gem_object_flush_gpu_write_domain(obj);
1360 /* Wait on any GPU rendering and flushing to occur. */
1361 ret = i915_gem_object_wait_rendering(obj);
1362 if (ret != 0)
1363 return ret;
1364
1365 i915_gem_object_flush_gtt_write_domain(obj);
1366
1367 /* If we have a partially-valid cache of the object in the CPU,
1368 * finish invalidating it and free the per-page flags.
1369 */
1370 i915_gem_object_set_to_full_cpu_read_domain(obj);
1371
1372 /* Flush the CPU cache if it's still invalid. */
1373 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1374 i915_gem_clflush_object(obj);
1375 drm_agp_chipset_flush(dev);
1376
1377 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1378 }
1379
1380 /* It should now be out of any other write domains, and we can update
1381 * the domain values for our changes.
1382 */
1383 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1384
1385 /* If we're writing through the CPU, then the GPU read domains will
1386 * need to be invalidated at next use.
1387 */
1388 if (write) {
1389 obj->read_domains &= I915_GEM_DOMAIN_CPU;
1390 obj->write_domain = I915_GEM_DOMAIN_CPU;
1391 }
1392
1393 return 0;
1394}
1395
1216/* 1396/*
1217 * Set the next domain for the specified object. This 1397 * Set the next domain for the specified object. This
1218 * may not actually perform the necessary flushing/invaliding though, 1398 * may not actually perform the necessary flushing/invaliding though,
@@ -1324,16 +1504,18 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
1324 * MI_FLUSH 1504 * MI_FLUSH
1325 * drm_agp_chipset_flush 1505 * drm_agp_chipset_flush
1326 */ 1506 */
1327static int 1507static void
1328i915_gem_object_set_domain(struct drm_gem_object *obj, 1508i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
1329 uint32_t read_domains, 1509 uint32_t read_domains,
1330 uint32_t write_domain) 1510 uint32_t write_domain)
1331{ 1511{
1332 struct drm_device *dev = obj->dev; 1512 struct drm_device *dev = obj->dev;
1333 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1513 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1334 uint32_t invalidate_domains = 0; 1514 uint32_t invalidate_domains = 0;
1335 uint32_t flush_domains = 0; 1515 uint32_t flush_domains = 0;
1336 int ret; 1516
1517 BUG_ON(read_domains & I915_GEM_DOMAIN_CPU);
1518 BUG_ON(write_domain == I915_GEM_DOMAIN_CPU);
1337 1519
1338#if WATCH_BUF 1520#if WATCH_BUF
1339 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n", 1521 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
@@ -1370,34 +1552,11 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
1370 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n", 1552 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1371 __func__, flush_domains, invalidate_domains); 1553 __func__, flush_domains, invalidate_domains);
1372#endif 1554#endif
1373 /*
1374 * If we're invaliding the CPU cache and flushing a GPU cache,
1375 * then pause for rendering so that the GPU caches will be
1376 * flushed before the cpu cache is invalidated
1377 */
1378 if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
1379 (flush_domains & ~(I915_GEM_DOMAIN_CPU |
1380 I915_GEM_DOMAIN_GTT))) {
1381 ret = i915_gem_object_wait_rendering(obj);
1382 if (ret)
1383 return ret;
1384 }
1385 i915_gem_clflush_object(obj); 1555 i915_gem_clflush_object(obj);
1386 } 1556 }
1387 1557
1388 if ((write_domain | flush_domains) != 0) 1558 if ((write_domain | flush_domains) != 0)
1389 obj->write_domain = write_domain; 1559 obj->write_domain = write_domain;
1390
1391 /* If we're invalidating the CPU domain, clear the per-page CPU
1392 * domain list as well.
1393 */
1394 if (obj_priv->page_cpu_valid != NULL &&
1395 (write_domain != 0 ||
1396 read_domains & I915_GEM_DOMAIN_CPU)) {
1397 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1398 DRM_MEM_DRIVER);
1399 obj_priv->page_cpu_valid = NULL;
1400 }
1401 obj->read_domains = read_domains; 1560 obj->read_domains = read_domains;
1402 1561
1403 dev->invalidate_domains |= invalidate_domains; 1562 dev->invalidate_domains |= invalidate_domains;
@@ -1408,49 +1567,94 @@ i915_gem_object_set_domain(struct drm_gem_object *obj,
1408 obj->read_domains, obj->write_domain, 1567 obj->read_domains, obj->write_domain,
1409 dev->invalidate_domains, dev->flush_domains); 1568 dev->invalidate_domains, dev->flush_domains);
1410#endif 1569#endif
1411 return 0;
1412} 1570}
1413 1571
1414/** 1572/**
1415 * Set the read/write domain on a range of the object. 1573 * Moves the object from a partially CPU read to a full one.
1416 * 1574 *
1417 * Currently only implemented for CPU reads, otherwise drops to normal 1575 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
1418 * i915_gem_object_set_domain(). 1576 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
1419 */ 1577 */
1420static int 1578static void
1421i915_gem_object_set_domain_range(struct drm_gem_object *obj, 1579i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
1422 uint64_t offset,
1423 uint64_t size,
1424 uint32_t read_domains,
1425 uint32_t write_domain)
1426{ 1580{
1581 struct drm_device *dev = obj->dev;
1427 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1582 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1428 int ret, i;
1429 1583
1430 if (obj->read_domains & I915_GEM_DOMAIN_CPU) 1584 if (!obj_priv->page_cpu_valid)
1431 return 0; 1585 return;
1432 1586
1433 if (read_domains != I915_GEM_DOMAIN_CPU || 1587 /* If we're partially in the CPU read domain, finish moving it in.
1434 write_domain != 0) 1588 */
1435 return i915_gem_object_set_domain(obj, 1589 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
1436 read_domains, write_domain); 1590 int i;
1437 1591
1438 /* Wait on any GPU rendering to the object to be flushed. */ 1592 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
1439 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) { 1593 if (obj_priv->page_cpu_valid[i])
1440 ret = i915_gem_object_wait_rendering(obj); 1594 continue;
1441 if (ret) 1595 drm_clflush_pages(obj_priv->page_list + i, 1);
1442 return ret; 1596 }
1597 drm_agp_chipset_flush(dev);
1443 } 1598 }
1444 1599
1600 /* Free the page_cpu_valid mappings which are now stale, whether
1601 * or not we've got I915_GEM_DOMAIN_CPU.
1602 */
1603 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1604 DRM_MEM_DRIVER);
1605 obj_priv->page_cpu_valid = NULL;
1606}
1607
1608/**
1609 * Set the CPU read domain on a range of the object.
1610 *
1611 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
1612 * not entirely valid. The page_cpu_valid member of the object flags which
1613 * pages have been flushed, and will be respected by
1614 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
1615 * of the whole object.
1616 *
1617 * This function returns when the move is complete, including waiting on
1618 * flushes to occur.
1619 */
1620static int
1621i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
1622 uint64_t offset, uint64_t size)
1623{
1624 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1625 int i, ret;
1626
1627 if (offset == 0 && size == obj->size)
1628 return i915_gem_object_set_to_cpu_domain(obj, 0);
1629
1630 i915_gem_object_flush_gpu_write_domain(obj);
1631 /* Wait on any GPU rendering and flushing to occur. */
1632 ret = i915_gem_object_wait_rendering(obj);
1633 if (ret != 0)
1634 return ret;
1635 i915_gem_object_flush_gtt_write_domain(obj);
1636
1637 /* If we're already fully in the CPU read domain, we're done. */
1638 if (obj_priv->page_cpu_valid == NULL &&
1639 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
1640 return 0;
1641
1642 /* Otherwise, create/clear the per-page CPU read domain flag if we're
1643 * newly adding I915_GEM_DOMAIN_CPU
1644 */
1445 if (obj_priv->page_cpu_valid == NULL) { 1645 if (obj_priv->page_cpu_valid == NULL) {
1446 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE, 1646 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1447 DRM_MEM_DRIVER); 1647 DRM_MEM_DRIVER);
1448 } 1648 if (obj_priv->page_cpu_valid == NULL)
1649 return -ENOMEM;
1650 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
1651 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
1449 1652
1450 /* Flush the cache on any pages that are still invalid from the CPU's 1653 /* Flush the cache on any pages that are still invalid from the CPU's
1451 * perspective. 1654 * perspective.
1452 */ 1655 */
1453 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) { 1656 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1657 i++) {
1454 if (obj_priv->page_cpu_valid[i]) 1658 if (obj_priv->page_cpu_valid[i])
1455 continue; 1659 continue;
1456 1660
@@ -1459,39 +1663,14 @@ i915_gem_object_set_domain_range(struct drm_gem_object *obj,
1459 obj_priv->page_cpu_valid[i] = 1; 1663 obj_priv->page_cpu_valid[i] = 1;
1460 } 1664 }
1461 1665
1462 return 0; 1666 /* It should now be out of any other write domains, and we can update
1463} 1667 * the domain values for our changes.
1464
1465/**
1466 * Once all of the objects have been set in the proper domain,
1467 * perform the necessary flush and invalidate operations.
1468 *
1469 * Returns the write domains flushed, for use in flush tracking.
1470 */
1471static uint32_t
1472i915_gem_dev_set_domain(struct drm_device *dev)
1473{
1474 uint32_t flush_domains = dev->flush_domains;
1475
1476 /*
1477 * Now that all the buffers are synced to the proper domains,
1478 * flush and invalidate the collected domains
1479 */ 1668 */
1480 if (dev->invalidate_domains | dev->flush_domains) { 1669 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1481#if WATCH_EXEC
1482 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
1483 __func__,
1484 dev->invalidate_domains,
1485 dev->flush_domains);
1486#endif
1487 i915_gem_flush(dev,
1488 dev->invalidate_domains,
1489 dev->flush_domains);
1490 dev->invalidate_domains = 0;
1491 dev->flush_domains = 0;
1492 }
1493 1670
1494 return flush_domains; 1671 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1672
1673 return 0;
1495} 1674}
1496 1675
1497/** 1676/**
@@ -1503,12 +1682,12 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1503 struct drm_i915_gem_exec_object *entry) 1682 struct drm_i915_gem_exec_object *entry)
1504{ 1683{
1505 struct drm_device *dev = obj->dev; 1684 struct drm_device *dev = obj->dev;
1685 drm_i915_private_t *dev_priv = dev->dev_private;
1506 struct drm_i915_gem_relocation_entry reloc; 1686 struct drm_i915_gem_relocation_entry reloc;
1507 struct drm_i915_gem_relocation_entry __user *relocs; 1687 struct drm_i915_gem_relocation_entry __user *relocs;
1508 struct drm_i915_gem_object *obj_priv = obj->driver_private; 1688 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1509 int i, ret; 1689 int i, ret;
1510 uint32_t last_reloc_offset = -1; 1690 void __iomem *reloc_page;
1511 void __iomem *reloc_page = NULL;
1512 1691
1513 /* Choose the GTT offset for our buffer and put it there. */ 1692 /* Choose the GTT offset for our buffer and put it there. */
1514 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment); 1693 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
@@ -1572,6 +1751,18 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1572 return -EINVAL; 1751 return -EINVAL;
1573 } 1752 }
1574 1753
1754 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
1755 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
1756 DRM_ERROR("reloc with read/write CPU domains: "
1757 "obj %p target %d offset %d "
1758 "read %08x write %08x",
1759 obj, reloc.target_handle,
1760 (int) reloc.offset,
1761 reloc.read_domains,
1762 reloc.write_domain);
1763 return -EINVAL;
1764 }
1765
1575 if (reloc.write_domain && target_obj->pending_write_domain && 1766 if (reloc.write_domain && target_obj->pending_write_domain &&
1576 reloc.write_domain != target_obj->pending_write_domain) { 1767 reloc.write_domain != target_obj->pending_write_domain) {
1577 DRM_ERROR("Write domain conflict: " 1768 DRM_ERROR("Write domain conflict: "
@@ -1612,45 +1803,22 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1612 continue; 1803 continue;
1613 } 1804 }
1614 1805
1615 /* Now that we're going to actually write some data in, 1806 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1616 * make sure that any rendering using this buffer's contents 1807 if (ret != 0) {
1617 * is completed. 1808 drm_gem_object_unreference(target_obj);
1618 */ 1809 i915_gem_object_unpin(obj);
1619 i915_gem_object_wait_rendering(obj); 1810 return -EINVAL;
1620
1621 /* As we're writing through the gtt, flush
1622 * any CPU writes before we write the relocations
1623 */
1624 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
1625 i915_gem_clflush_object(obj);
1626 drm_agp_chipset_flush(dev);
1627 obj->write_domain = 0;
1628 } 1811 }
1629 1812
1630 /* Map the page containing the relocation we're going to 1813 /* Map the page containing the relocation we're going to
1631 * perform. 1814 * perform.
1632 */ 1815 */
1633 reloc_offset = obj_priv->gtt_offset + reloc.offset; 1816 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1634 if (reloc_page == NULL || 1817 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
1635 (last_reloc_offset & ~(PAGE_SIZE - 1)) != 1818 (reloc_offset &
1636 (reloc_offset & ~(PAGE_SIZE - 1))) { 1819 ~(PAGE_SIZE - 1)));
1637 if (reloc_page != NULL)
1638 iounmap(reloc_page);
1639
1640 reloc_page = ioremap_wc(dev->agp->base +
1641 (reloc_offset &
1642 ~(PAGE_SIZE - 1)),
1643 PAGE_SIZE);
1644 last_reloc_offset = reloc_offset;
1645 if (reloc_page == NULL) {
1646 drm_gem_object_unreference(target_obj);
1647 i915_gem_object_unpin(obj);
1648 return -ENOMEM;
1649 }
1650 }
1651
1652 reloc_entry = (uint32_t __iomem *)(reloc_page + 1820 reloc_entry = (uint32_t __iomem *)(reloc_page +
1653 (reloc_offset & (PAGE_SIZE - 1))); 1821 (reloc_offset & (PAGE_SIZE - 1)));
1654 reloc_val = target_obj_priv->gtt_offset + reloc.delta; 1822 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1655 1823
1656#if WATCH_BUF 1824#if WATCH_BUF
@@ -1659,6 +1827,7 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1659 readl(reloc_entry), reloc_val); 1827 readl(reloc_entry), reloc_val);
1660#endif 1828#endif
1661 writel(reloc_val, reloc_entry); 1829 writel(reloc_val, reloc_entry);
1830 io_mapping_unmap_atomic(reloc_page);
1662 1831
1663 /* Write the updated presumed offset for this entry back out 1832 /* Write the updated presumed offset for this entry back out
1664 * to the user. 1833 * to the user.
@@ -1674,9 +1843,6 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1674 drm_gem_object_unreference(target_obj); 1843 drm_gem_object_unreference(target_obj);
1675 } 1844 }
1676 1845
1677 if (reloc_page != NULL)
1678 iounmap(reloc_page);
1679
1680#if WATCH_BUF 1846#if WATCH_BUF
1681 if (0) 1847 if (0)
1682 i915_gem_dump_object(obj, 128, __func__, ~0); 1848 i915_gem_dump_object(obj, 128, __func__, ~0);
@@ -1783,6 +1949,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1783 int ret, i, pinned = 0; 1949 int ret, i, pinned = 0;
1784 uint64_t exec_offset; 1950 uint64_t exec_offset;
1785 uint32_t seqno, flush_domains; 1951 uint32_t seqno, flush_domains;
1952 int pin_tries;
1786 1953
1787#if WATCH_EXEC 1954#if WATCH_EXEC
1788 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", 1955 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
@@ -1831,14 +1998,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1831 return -EBUSY; 1998 return -EBUSY;
1832 } 1999 }
1833 2000
1834 /* Zero the gloabl flush/invalidate flags. These 2001 /* Look up object handles */
1835 * will be modified as each object is bound to the
1836 * gtt
1837 */
1838 dev->invalidate_domains = 0;
1839 dev->flush_domains = 0;
1840
1841 /* Look up object handles and perform the relocations */
1842 for (i = 0; i < args->buffer_count; i++) { 2002 for (i = 0; i < args->buffer_count; i++) {
1843 object_list[i] = drm_gem_object_lookup(dev, file_priv, 2003 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1844 exec_list[i].handle); 2004 exec_list[i].handle);
@@ -1848,17 +2008,39 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1848 ret = -EBADF; 2008 ret = -EBADF;
1849 goto err; 2009 goto err;
1850 } 2010 }
2011 }
1851 2012
1852 object_list[i]->pending_read_domains = 0; 2013 /* Pin and relocate */
1853 object_list[i]->pending_write_domain = 0; 2014 for (pin_tries = 0; ; pin_tries++) {
1854 ret = i915_gem_object_pin_and_relocate(object_list[i], 2015 ret = 0;
1855 file_priv, 2016 for (i = 0; i < args->buffer_count; i++) {
1856 &exec_list[i]); 2017 object_list[i]->pending_read_domains = 0;
1857 if (ret) { 2018 object_list[i]->pending_write_domain = 0;
1858 DRM_ERROR("object bind and relocate failed %d\n", ret); 2019 ret = i915_gem_object_pin_and_relocate(object_list[i],
2020 file_priv,
2021 &exec_list[i]);
2022 if (ret)
2023 break;
2024 pinned = i + 1;
2025 }
2026 /* success */
2027 if (ret == 0)
2028 break;
2029
2030 /* error other than GTT full, or we've already tried again */
2031 if (ret != -ENOMEM || pin_tries >= 1) {
2032 DRM_ERROR("Failed to pin buffers %d\n", ret);
1859 goto err; 2033 goto err;
1860 } 2034 }
1861 pinned = i + 1; 2035
2036 /* unpin all of our buffers */
2037 for (i = 0; i < pinned; i++)
2038 i915_gem_object_unpin(object_list[i]);
2039
2040 /* evict everyone we can from the aperture */
2041 ret = i915_gem_evict_everything(dev);
2042 if (ret)
2043 goto err;
1862 } 2044 }
1863 2045
1864 /* Set the pending read domains for the batch buffer to COMMAND */ 2046 /* Set the pending read domains for the batch buffer to COMMAND */
@@ -1868,32 +2050,37 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1868 2050
1869 i915_verify_inactive(dev, __FILE__, __LINE__); 2051 i915_verify_inactive(dev, __FILE__, __LINE__);
1870 2052
2053 /* Zero the global flush/invalidate flags. These
2054 * will be modified as new domains are computed
2055 * for each object
2056 */
2057 dev->invalidate_domains = 0;
2058 dev->flush_domains = 0;
2059
1871 for (i = 0; i < args->buffer_count; i++) { 2060 for (i = 0; i < args->buffer_count; i++) {
1872 struct drm_gem_object *obj = object_list[i]; 2061 struct drm_gem_object *obj = object_list[i];
1873 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1874
1875 if (obj_priv->gtt_space == NULL) {
1876 /* We evicted the buffer in the process of validating
1877 * our set of buffers in. We could try to recover by
1878 * kicking them everything out and trying again from
1879 * the start.
1880 */
1881 ret = -ENOMEM;
1882 goto err;
1883 }
1884 2062
1885 /* make sure all previous memory operations have passed */ 2063 /* Compute new gpu domains and update invalidate/flush */
1886 ret = i915_gem_object_set_domain(obj, 2064 i915_gem_object_set_to_gpu_domain(obj,
1887 obj->pending_read_domains, 2065 obj->pending_read_domains,
1888 obj->pending_write_domain); 2066 obj->pending_write_domain);
1889 if (ret)
1890 goto err;
1891 } 2067 }
1892 2068
1893 i915_verify_inactive(dev, __FILE__, __LINE__); 2069 i915_verify_inactive(dev, __FILE__, __LINE__);
1894 2070
1895 /* Flush/invalidate caches and chipset buffer */ 2071 if (dev->invalidate_domains | dev->flush_domains) {
1896 flush_domains = i915_gem_dev_set_domain(dev); 2072#if WATCH_EXEC
2073 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
2074 __func__,
2075 dev->invalidate_domains,
2076 dev->flush_domains);
2077#endif
2078 i915_gem_flush(dev,
2079 dev->invalidate_domains,
2080 dev->flush_domains);
2081 if (dev->flush_domains)
2082 (void)i915_add_request(dev, dev->flush_domains);
2083 }
1897 2084
1898 i915_verify_inactive(dev, __FILE__, __LINE__); 2085 i915_verify_inactive(dev, __FILE__, __LINE__);
1899 2086
@@ -1913,8 +2100,6 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1913 ~0); 2100 ~0);
1914#endif 2101#endif
1915 2102
1916 (void)i915_add_request(dev, flush_domains);
1917
1918 /* Exec the batchbuffer */ 2103 /* Exec the batchbuffer */
1919 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset); 2104 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
1920 if (ret) { 2105 if (ret) {
@@ -1942,10 +2127,8 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
1942 i915_file_priv->mm.last_gem_seqno = seqno; 2127 i915_file_priv->mm.last_gem_seqno = seqno;
1943 for (i = 0; i < args->buffer_count; i++) { 2128 for (i = 0; i < args->buffer_count; i++) {
1944 struct drm_gem_object *obj = object_list[i]; 2129 struct drm_gem_object *obj = object_list[i];
1945 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1946 2130
1947 i915_gem_object_move_to_active(obj); 2131 i915_gem_object_move_to_active(obj, seqno);
1948 obj_priv->last_rendering_seqno = seqno;
1949#if WATCH_LRU 2132#if WATCH_LRU
1950 DRM_INFO("%s: move to exec list %p\n", __func__, obj); 2133 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
1951#endif 2134#endif
@@ -2076,11 +2259,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2076 /* XXX - flush the CPU caches for pinned objects 2259 /* XXX - flush the CPU caches for pinned objects
2077 * as the X server doesn't manage domains yet 2260 * as the X server doesn't manage domains yet
2078 */ 2261 */
2079 if (obj->write_domain & I915_GEM_DOMAIN_CPU) { 2262 i915_gem_object_flush_cpu_write_domain(obj);
2080 i915_gem_clflush_object(obj);
2081 drm_agp_chipset_flush(dev);
2082 obj->write_domain = 0;
2083 }
2084 args->offset = obj_priv->gtt_offset; 2263 args->offset = obj_priv->gtt_offset;
2085 drm_gem_object_unreference(obj); 2264 drm_gem_object_unreference(obj);
2086 mutex_unlock(&dev->struct_mutex); 2265 mutex_unlock(&dev->struct_mutex);
@@ -2182,29 +2361,6 @@ void i915_gem_free_object(struct drm_gem_object *obj)
2182 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER); 2361 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2183} 2362}
2184 2363
2185static int
2186i915_gem_set_domain(struct drm_gem_object *obj,
2187 struct drm_file *file_priv,
2188 uint32_t read_domains,
2189 uint32_t write_domain)
2190{
2191 struct drm_device *dev = obj->dev;
2192 int ret;
2193 uint32_t flush_domains;
2194
2195 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2196
2197 ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
2198 if (ret)
2199 return ret;
2200 flush_domains = i915_gem_dev_set_domain(obj->dev);
2201
2202 if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
2203 (void) i915_add_request(dev, flush_domains);
2204
2205 return 0;
2206}
2207
2208/** Unbinds all objects that are on the given buffer list. */ 2364/** Unbinds all objects that are on the given buffer list. */
2209static int 2365static int
2210i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head) 2366i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
@@ -2299,29 +2455,52 @@ i915_gem_idle(struct drm_device *dev)
2299 2455
2300 i915_gem_retire_requests(dev); 2456 i915_gem_retire_requests(dev);
2301 2457
2302 /* Active and flushing should now be empty as we've 2458 if (!dev_priv->mm.wedged) {
2303 * waited for a sequence higher than any pending execbuffer 2459 /* Active and flushing should now be empty as we've
2304 */ 2460 * waited for a sequence higher than any pending execbuffer
2305 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 2461 */
2306 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 2462 WARN_ON(!list_empty(&dev_priv->mm.active_list));
2463 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
2464 /* Request should now be empty as we've also waited
2465 * for the last request in the list
2466 */
2467 WARN_ON(!list_empty(&dev_priv->mm.request_list));
2468 }
2307 2469
2308 /* Request should now be empty as we've also waited 2470 /* Empty the active and flushing lists to inactive. If there's
2309 * for the last request in the list 2471 * anything left at this point, it means that we're wedged and
2472 * nothing good's going to happen by leaving them there. So strip
2473 * the GPU domains and just stuff them onto inactive.
2310 */ 2474 */
2311 BUG_ON(!list_empty(&dev_priv->mm.request_list)); 2475 while (!list_empty(&dev_priv->mm.active_list)) {
2476 struct drm_i915_gem_object *obj_priv;
2477
2478 obj_priv = list_first_entry(&dev_priv->mm.active_list,
2479 struct drm_i915_gem_object,
2480 list);
2481 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2482 i915_gem_object_move_to_inactive(obj_priv->obj);
2483 }
2484
2485 while (!list_empty(&dev_priv->mm.flushing_list)) {
2486 struct drm_i915_gem_object *obj_priv;
2312 2487
2313 /* Move all buffers out of the GTT. */ 2488 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
2489 struct drm_i915_gem_object,
2490 list);
2491 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2492 i915_gem_object_move_to_inactive(obj_priv->obj);
2493 }
2494
2495
2496 /* Move all inactive buffers out of the GTT. */
2314 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list); 2497 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2498 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
2315 if (ret) { 2499 if (ret) {
2316 mutex_unlock(&dev->struct_mutex); 2500 mutex_unlock(&dev->struct_mutex);
2317 return ret; 2501 return ret;
2318 } 2502 }
2319 2503
2320 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2321 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2322 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2323 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2324
2325 i915_gem_cleanup_ringbuffer(dev); 2504 i915_gem_cleanup_ringbuffer(dev);
2326 mutex_unlock(&dev->struct_mutex); 2505 mutex_unlock(&dev->struct_mutex);
2327 2506
@@ -2518,6 +2697,10 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2518 if (ret != 0) 2697 if (ret != 0)
2519 return ret; 2698 return ret;
2520 2699
2700 dev_priv->mm.gtt_mapping = io_mapping_create_wc(dev->agp->base,
2701 dev->agp->agp_info.aper_size
2702 * 1024 * 1024);
2703
2521 mutex_lock(&dev->struct_mutex); 2704 mutex_lock(&dev->struct_mutex);
2522 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 2705 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2523 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 2706 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
@@ -2535,11 +2718,13 @@ int
2535i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 2718i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2536 struct drm_file *file_priv) 2719 struct drm_file *file_priv)
2537{ 2720{
2721 drm_i915_private_t *dev_priv = dev->dev_private;
2538 int ret; 2722 int ret;
2539 2723
2540 ret = i915_gem_idle(dev); 2724 ret = i915_gem_idle(dev);
2541 drm_irq_uninstall(dev); 2725 drm_irq_uninstall(dev);
2542 2726
2727 io_mapping_free(dev_priv->mm.gtt_mapping);
2543 return ret; 2728 return ret;
2544} 2729}
2545 2730
diff --git a/drivers/gpu/drm/i915/i915_gem_proc.c b/drivers/gpu/drm/i915/i915_gem_proc.c
index 93de15b4c9a7..e8d5abe1250e 100644
--- a/drivers/gpu/drm/i915/i915_gem_proc.c
+++ b/drivers/gpu/drm/i915/i915_gem_proc.c
@@ -166,10 +166,9 @@ static int i915_gem_request_info(char *buf, char **start, off_t offset,
166 list_for_each_entry(gem_request, &dev_priv->mm.request_list, 166 list_for_each_entry(gem_request, &dev_priv->mm.request_list,
167 list) 167 list)
168 { 168 {
169 DRM_PROC_PRINT(" %d @ %d %08x\n", 169 DRM_PROC_PRINT(" %d @ %d\n",
170 gem_request->seqno, 170 gem_request->seqno,
171 (int) (jiffies - gem_request->emitted_jiffies), 171 (int) (jiffies - gem_request->emitted_jiffies));
172 gem_request->flush_domains);
173 } 172 }
174 if (len > request + offset) 173 if (len > request + offset)
175 return request; 174 return request;
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index e8b85ac4ca04..a8cb69469c64 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -119,9 +119,10 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
119 dcc & DCC_CHANNEL_XOR_DISABLE) { 119 dcc & DCC_CHANNEL_XOR_DISABLE) {
120 swizzle_x = I915_BIT_6_SWIZZLE_9_10; 120 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
121 swizzle_y = I915_BIT_6_SWIZZLE_9; 121 swizzle_y = I915_BIT_6_SWIZZLE_9;
122 } else if (IS_I965GM(dev) || IS_GM45(dev)) { 122 } else if ((IS_I965GM(dev) || IS_GM45(dev)) &&
123 /* GM965 only does bit 11-based channel 123 (dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
124 * randomization 124 /* GM965/GM45 does either bit 11 or bit 17
125 * swizzling.
125 */ 126 */
126 swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; 127 swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
127 swizzle_y = I915_BIT_6_SWIZZLE_9_11; 128 swizzle_y = I915_BIT_6_SWIZZLE_9_11;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 26f48932a51e..69b9a42da95e 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -33,11 +33,23 @@
33 33
34#define MAX_NOPID ((u32)~0) 34#define MAX_NOPID ((u32)~0)
35 35
36/** These are the interrupts used by the driver */ 36/**
37#define I915_INTERRUPT_ENABLE_MASK (I915_USER_INTERRUPT | \ 37 * Interrupts that are always left unmasked.
38 I915_ASLE_INTERRUPT | \ 38 *
39 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \ 39 * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
40 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) 40 * we leave them always unmasked in IMR and then control enabling them through
41 * PIPESTAT alone.
42 */
43#define I915_INTERRUPT_ENABLE_FIX (I915_ASLE_INTERRUPT | \
44 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
45 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
46
47/** Interrupts that we mask and unmask at runtime. */
48#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
49
50/** These are all of the interrupts used by the driver */
51#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
52 I915_INTERRUPT_ENABLE_VAR)
41 53
42void 54void
43i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) 55i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
@@ -59,6 +71,41 @@ i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
59 } 71 }
60} 72}
61 73
74static inline u32
75i915_pipestat(int pipe)
76{
77 if (pipe == 0)
78 return PIPEASTAT;
79 if (pipe == 1)
80 return PIPEBSTAT;
81 BUG();
82}
83
84void
85i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
86{
87 if ((dev_priv->pipestat[pipe] & mask) != mask) {
88 u32 reg = i915_pipestat(pipe);
89
90 dev_priv->pipestat[pipe] |= mask;
91 /* Enable the interrupt, clear any pending status */
92 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
93 (void) I915_READ(reg);
94 }
95}
96
97void
98i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
99{
100 if ((dev_priv->pipestat[pipe] & mask) != 0) {
101 u32 reg = i915_pipestat(pipe);
102
103 dev_priv->pipestat[pipe] &= ~mask;
104 I915_WRITE(reg, dev_priv->pipestat[pipe]);
105 (void) I915_READ(reg);
106 }
107}
108
62/** 109/**
63 * i915_pipe_enabled - check if a pipe is enabled 110 * i915_pipe_enabled - check if a pipe is enabled
64 * @dev: DRM device 111 * @dev: DRM device
@@ -80,211 +127,6 @@ i915_pipe_enabled(struct drm_device *dev, int pipe)
80 return 0; 127 return 0;
81} 128}
82 129
83/**
84 * Emit blits for scheduled buffer swaps.
85 *
86 * This function will be called with the HW lock held.
87 * Because this function must grab the ring mutex (dev->struct_mutex),
88 * it can no longer run at soft irq time. We'll fix this when we do
89 * the DRI2 swap buffer work.
90 */
91static void i915_vblank_tasklet(struct drm_device *dev)
92{
93 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
94 unsigned long irqflags;
95 struct list_head *list, *tmp, hits, *hit;
96 int nhits, nrects, slice[2], upper[2], lower[2], i;
97 unsigned counter[2];
98 struct drm_drawable_info *drw;
99 drm_i915_sarea_t *sarea_priv = dev_priv->sarea_priv;
100 u32 cpp = dev_priv->cpp;
101 u32 cmd = (cpp == 4) ? (XY_SRC_COPY_BLT_CMD |
102 XY_SRC_COPY_BLT_WRITE_ALPHA |
103 XY_SRC_COPY_BLT_WRITE_RGB)
104 : XY_SRC_COPY_BLT_CMD;
105 u32 src_pitch = sarea_priv->pitch * cpp;
106 u32 dst_pitch = sarea_priv->pitch * cpp;
107 u32 ropcpp = (0xcc << 16) | ((cpp - 1) << 24);
108 RING_LOCALS;
109
110 mutex_lock(&dev->struct_mutex);
111
112 if (IS_I965G(dev) && sarea_priv->front_tiled) {
113 cmd |= XY_SRC_COPY_BLT_DST_TILED;
114 dst_pitch >>= 2;
115 }
116 if (IS_I965G(dev) && sarea_priv->back_tiled) {
117 cmd |= XY_SRC_COPY_BLT_SRC_TILED;
118 src_pitch >>= 2;
119 }
120
121 counter[0] = drm_vblank_count(dev, 0);
122 counter[1] = drm_vblank_count(dev, 1);
123
124 DRM_DEBUG("\n");
125
126 INIT_LIST_HEAD(&hits);
127
128 nhits = nrects = 0;
129
130 spin_lock_irqsave(&dev_priv->swaps_lock, irqflags);
131
132 /* Find buffer swaps scheduled for this vertical blank */
133 list_for_each_safe(list, tmp, &dev_priv->vbl_swaps.head) {
134 drm_i915_vbl_swap_t *vbl_swap =
135 list_entry(list, drm_i915_vbl_swap_t, head);
136 int pipe = vbl_swap->pipe;
137
138 if ((counter[pipe] - vbl_swap->sequence) > (1<<23))
139 continue;
140
141 list_del(list);
142 dev_priv->swaps_pending--;
143 drm_vblank_put(dev, pipe);
144
145 spin_unlock(&dev_priv->swaps_lock);
146 spin_lock(&dev->drw_lock);
147
148 drw = drm_get_drawable_info(dev, vbl_swap->drw_id);
149
150 list_for_each(hit, &hits) {
151 drm_i915_vbl_swap_t *swap_cmp =
152 list_entry(hit, drm_i915_vbl_swap_t, head);
153 struct drm_drawable_info *drw_cmp =
154 drm_get_drawable_info(dev, swap_cmp->drw_id);
155
156 /* Make sure both drawables are still
157 * around and have some rectangles before
158 * we look inside to order them for the
159 * blts below.
160 */
161 if (drw_cmp && drw_cmp->num_rects > 0 &&
162 drw && drw->num_rects > 0 &&
163 drw_cmp->rects[0].y1 > drw->rects[0].y1) {
164 list_add_tail(list, hit);
165 break;
166 }
167 }
168
169 spin_unlock(&dev->drw_lock);
170
171 /* List of hits was empty, or we reached the end of it */
172 if (hit == &hits)
173 list_add_tail(list, hits.prev);
174
175 nhits++;
176
177 spin_lock(&dev_priv->swaps_lock);
178 }
179
180 if (nhits == 0) {
181 spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
182 mutex_unlock(&dev->struct_mutex);
183 return;
184 }
185
186 spin_unlock(&dev_priv->swaps_lock);
187
188 i915_kernel_lost_context(dev);
189
190 if (IS_I965G(dev)) {
191 BEGIN_LP_RING(4);
192
193 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
194 OUT_RING(0);
195 OUT_RING(((sarea_priv->width - 1) & 0xffff) | ((sarea_priv->height - 1) << 16));
196 OUT_RING(0);
197 ADVANCE_LP_RING();
198 } else {
199 BEGIN_LP_RING(6);
200
201 OUT_RING(GFX_OP_DRAWRECT_INFO);
202 OUT_RING(0);
203 OUT_RING(0);
204 OUT_RING(sarea_priv->width | sarea_priv->height << 16);
205 OUT_RING(sarea_priv->width | sarea_priv->height << 16);
206 OUT_RING(0);
207
208 ADVANCE_LP_RING();
209 }
210
211 sarea_priv->ctxOwner = DRM_KERNEL_CONTEXT;
212
213 upper[0] = upper[1] = 0;
214 slice[0] = max(sarea_priv->pipeA_h / nhits, 1);
215 slice[1] = max(sarea_priv->pipeB_h / nhits, 1);
216 lower[0] = sarea_priv->pipeA_y + slice[0];
217 lower[1] = sarea_priv->pipeB_y + slice[0];
218
219 spin_lock(&dev->drw_lock);
220
221 /* Emit blits for buffer swaps, partitioning both outputs into as many
222 * slices as there are buffer swaps scheduled in order to avoid tearing
223 * (based on the assumption that a single buffer swap would always
224 * complete before scanout starts).
225 */
226 for (i = 0; i++ < nhits;
227 upper[0] = lower[0], lower[0] += slice[0],
228 upper[1] = lower[1], lower[1] += slice[1]) {
229 if (i == nhits)
230 lower[0] = lower[1] = sarea_priv->height;
231
232 list_for_each(hit, &hits) {
233 drm_i915_vbl_swap_t *swap_hit =
234 list_entry(hit, drm_i915_vbl_swap_t, head);
235 struct drm_clip_rect *rect;
236 int num_rects, pipe;
237 unsigned short top, bottom;
238
239 drw = drm_get_drawable_info(dev, swap_hit->drw_id);
240
241 /* The drawable may have been destroyed since
242 * the vblank swap was queued
243 */
244 if (!drw)
245 continue;
246
247 rect = drw->rects;
248 pipe = swap_hit->pipe;
249 top = upper[pipe];
250 bottom = lower[pipe];
251
252 for (num_rects = drw->num_rects; num_rects--; rect++) {
253 int y1 = max(rect->y1, top);
254 int y2 = min(rect->y2, bottom);
255
256 if (y1 >= y2)
257 continue;
258
259 BEGIN_LP_RING(8);
260
261 OUT_RING(cmd);
262 OUT_RING(ropcpp | dst_pitch);
263 OUT_RING((y1 << 16) | rect->x1);
264 OUT_RING((y2 << 16) | rect->x2);
265 OUT_RING(sarea_priv->front_offset);
266 OUT_RING((y1 << 16) | rect->x1);
267 OUT_RING(src_pitch);
268 OUT_RING(sarea_priv->back_offset);
269
270 ADVANCE_LP_RING();
271 }
272 }
273 }
274
275 spin_unlock_irqrestore(&dev->drw_lock, irqflags);
276 mutex_unlock(&dev->struct_mutex);
277
278 list_for_each_safe(hit, tmp, &hits) {
279 drm_i915_vbl_swap_t *swap_hit =
280 list_entry(hit, drm_i915_vbl_swap_t, head);
281
282 list_del(hit);
283
284 drm_free(swap_hit, sizeof(*swap_hit), DRM_MEM_DRIVER);
285 }
286}
287
288/* Called from drm generic code, passed a 'crtc', which 130/* Called from drm generic code, passed a 'crtc', which
289 * we use as a pipe index 131 * we use as a pipe index
290 */ 132 */
@@ -322,121 +164,106 @@ u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
322 return count; 164 return count;
323} 165}
324 166
325void
326i915_vblank_work_handler(struct work_struct *work)
327{
328 drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
329 vblank_work);
330 struct drm_device *dev = dev_priv->dev;
331 unsigned long irqflags;
332
333 if (dev->lock.hw_lock == NULL) {
334 i915_vblank_tasklet(dev);
335 return;
336 }
337
338 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
339 dev->locked_tasklet_func = i915_vblank_tasklet;
340 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
341
342 /* Try to get the lock now, if this fails, the lock
343 * holder will execute the tasklet during unlock
344 */
345 if (!drm_lock_take(&dev->lock, DRM_KERNEL_CONTEXT))
346 return;
347
348 dev->lock.lock_time = jiffies;
349 atomic_inc(&dev->counts[_DRM_STAT_LOCKS]);
350
351 spin_lock_irqsave(&dev->tasklet_lock, irqflags);
352 dev->locked_tasklet_func = NULL;
353 spin_unlock_irqrestore(&dev->tasklet_lock, irqflags);
354
355 i915_vblank_tasklet(dev);
356 drm_lock_free(&dev->lock, DRM_KERNEL_CONTEXT);
357}
358
359irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) 167irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
360{ 168{
361 struct drm_device *dev = (struct drm_device *) arg; 169 struct drm_device *dev = (struct drm_device *) arg;
362 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 170 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
363 u32 iir; 171 u32 iir, new_iir;
364 u32 pipea_stats, pipeb_stats; 172 u32 pipea_stats, pipeb_stats;
173 u32 vblank_status;
174 u32 vblank_enable;
365 int vblank = 0; 175 int vblank = 0;
176 unsigned long irqflags;
177 int irq_received;
178 int ret = IRQ_NONE;
366 179
367 atomic_inc(&dev_priv->irq_received); 180 atomic_inc(&dev_priv->irq_received);
368 181
369 if (dev->pdev->msi_enabled)
370 I915_WRITE(IMR, ~0);
371 iir = I915_READ(IIR); 182 iir = I915_READ(IIR);
372 183
373 if (iir == 0) { 184 if (IS_I965G(dev)) {
374 if (dev->pdev->msi_enabled) { 185 vblank_status = I915_START_VBLANK_INTERRUPT_STATUS;
375 I915_WRITE(IMR, dev_priv->irq_mask_reg); 186 vblank_enable = PIPE_START_VBLANK_INTERRUPT_ENABLE;
376 (void) I915_READ(IMR); 187 } else {
377 } 188 vblank_status = I915_VBLANK_INTERRUPT_STATUS;
378 return IRQ_NONE; 189 vblank_enable = I915_VBLANK_INTERRUPT_ENABLE;
379 } 190 }
380 191
381 /* 192 for (;;) {
382 * Clear the PIPE(A|B)STAT regs before the IIR otherwise 193 irq_received = iir != 0;
383 * we may get extra interrupts. 194
384 */ 195 /* Can't rely on pipestat interrupt bit in iir as it might
385 if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) { 196 * have been cleared after the pipestat interrupt was received.
197 * It doesn't set the bit in iir again, but it still produces
198 * interrupts (for non-MSI).
199 */
200 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
386 pipea_stats = I915_READ(PIPEASTAT); 201 pipea_stats = I915_READ(PIPEASTAT);
387 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A)) 202 pipeb_stats = I915_READ(PIPEBSTAT);
388 pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE | 203 /*
389 PIPE_VBLANK_INTERRUPT_ENABLE); 204 * Clear the PIPE(A|B)STAT regs before the IIR
390 else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS| 205 */
391 PIPE_VBLANK_INTERRUPT_STATUS)) { 206 if (pipea_stats & 0x8000ffff) {
207 I915_WRITE(PIPEASTAT, pipea_stats);
208 irq_received = 1;
209 }
210
211 if (pipeb_stats & 0x8000ffff) {
212 I915_WRITE(PIPEBSTAT, pipeb_stats);
213 irq_received = 1;
214 }
215 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
216
217 if (!irq_received)
218 break;
219
220 ret = IRQ_HANDLED;
221
222 I915_WRITE(IIR, iir);
223 new_iir = I915_READ(IIR); /* Flush posted writes */
224
225 if (dev_priv->sarea_priv)
226 dev_priv->sarea_priv->last_dispatch =
227 READ_BREADCRUMB(dev_priv);
228
229 if (iir & I915_USER_INTERRUPT) {
230 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
231 DRM_WAKEUP(&dev_priv->irq_queue);
232 }
233
234 if (pipea_stats & vblank_status) {
392 vblank++; 235 vblank++;
393 drm_handle_vblank(dev, 0); 236 drm_handle_vblank(dev, 0);
394 } 237 }
395 238
396 I915_WRITE(PIPEASTAT, pipea_stats); 239 if (pipeb_stats & vblank_status) {
397 }
398 if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
399 pipeb_stats = I915_READ(PIPEBSTAT);
400 /* Ack the event */
401 I915_WRITE(PIPEBSTAT, pipeb_stats);
402
403 /* The vblank interrupt gets enabled even if we didn't ask for
404 it, so make sure it's shut down again */
405 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
406 pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
407 PIPE_VBLANK_INTERRUPT_ENABLE);
408 else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
409 PIPE_VBLANK_INTERRUPT_STATUS)) {
410 vblank++; 240 vblank++;
411 drm_handle_vblank(dev, 1); 241 drm_handle_vblank(dev, 1);
412 } 242 }
413 243
414 if (pipeb_stats & I915_LEGACY_BLC_EVENT_STATUS) 244 if ((pipeb_stats & I915_LEGACY_BLC_EVENT_STATUS) ||
245 (iir & I915_ASLE_INTERRUPT))
415 opregion_asle_intr(dev); 246 opregion_asle_intr(dev);
416 I915_WRITE(PIPEBSTAT, pipeb_stats);
417 }
418
419 I915_WRITE(IIR, iir);
420 if (dev->pdev->msi_enabled)
421 I915_WRITE(IMR, dev_priv->irq_mask_reg);
422 (void) I915_READ(IIR); /* Flush posted writes */
423
424 if (dev_priv->sarea_priv)
425 dev_priv->sarea_priv->last_dispatch =
426 READ_BREADCRUMB(dev_priv);
427 247
428 if (iir & I915_USER_INTERRUPT) { 248 /* With MSI, interrupts are only generated when iir
429 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev); 249 * transitions from zero to nonzero. If another bit got
430 DRM_WAKEUP(&dev_priv->irq_queue); 250 * set while we were handling the existing iir bits, then
251 * we would never get another interrupt.
252 *
253 * This is fine on non-MSI as well, as if we hit this path
254 * we avoid exiting the interrupt handler only to generate
255 * another one.
256 *
257 * Note that for MSI this could cause a stray interrupt report
258 * if an interrupt landed in the time between writing IIR and
259 * the posting read. This should be rare enough to never
260 * trigger the 99% of 100,000 interrupts test for disabling
261 * stray interrupts.
262 */
263 iir = new_iir;
431 } 264 }
432 265
433 if (iir & I915_ASLE_INTERRUPT) 266 return ret;
434 opregion_asle_intr(dev);
435
436 if (vblank && dev_priv->swaps_pending > 0)
437 schedule_work(&dev_priv->vblank_work);
438
439 return IRQ_HANDLED;
440} 267}
441 268
442static int i915_emit_irq(struct drm_device * dev) 269static int i915_emit_irq(struct drm_device * dev)
@@ -454,12 +281,10 @@ static int i915_emit_irq(struct drm_device * dev)
454 if (dev_priv->sarea_priv) 281 if (dev_priv->sarea_priv)
455 dev_priv->sarea_priv->last_enqueue = dev_priv->counter; 282 dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
456 283
457 BEGIN_LP_RING(6); 284 BEGIN_LP_RING(4);
458 OUT_RING(MI_STORE_DWORD_INDEX); 285 OUT_RING(MI_STORE_DWORD_INDEX);
459 OUT_RING(5 << MI_STORE_DWORD_INDEX_SHIFT); 286 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
460 OUT_RING(dev_priv->counter); 287 OUT_RING(dev_priv->counter);
461 OUT_RING(0);
462 OUT_RING(0);
463 OUT_RING(MI_USER_INTERRUPT); 288 OUT_RING(MI_USER_INTERRUPT);
464 ADVANCE_LP_RING(); 289 ADVANCE_LP_RING();
465 290
@@ -574,48 +399,16 @@ int i915_irq_wait(struct drm_device *dev, void *data,
574int i915_enable_vblank(struct drm_device *dev, int pipe) 399int i915_enable_vblank(struct drm_device *dev, int pipe)
575{ 400{
576 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 401 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
577 u32 pipestat_reg = 0;
578 u32 pipestat;
579 u32 interrupt = 0;
580 unsigned long irqflags; 402 unsigned long irqflags;
581 403
582 switch (pipe) {
583 case 0:
584 pipestat_reg = PIPEASTAT;
585 interrupt = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
586 break;
587 case 1:
588 pipestat_reg = PIPEBSTAT;
589 interrupt = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
590 break;
591 default:
592 DRM_ERROR("tried to enable vblank on non-existent pipe %d\n",
593 pipe);
594 return 0;
595 }
596
597 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 404 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
598 /* Enabling vblank events in IMR comes before PIPESTAT write, or
599 * there's a race where the PIPESTAT vblank bit gets set to 1, so
600 * the OR of enabled PIPESTAT bits goes to 1, so the PIPExEVENT in
601 * ISR flashes to 1, but the IIR bit doesn't get set to 1 because
602 * IMR masks it. It doesn't ever get set after we clear the masking
603 * in IMR because the ISR bit is edge, not level-triggered, on the
604 * OR of PIPESTAT bits.
605 */
606 i915_enable_irq(dev_priv, interrupt);
607 pipestat = I915_READ(pipestat_reg);
608 if (IS_I965G(dev)) 405 if (IS_I965G(dev))
609 pipestat |= PIPE_START_VBLANK_INTERRUPT_ENABLE; 406 i915_enable_pipestat(dev_priv, pipe,
407 PIPE_START_VBLANK_INTERRUPT_ENABLE);
610 else 408 else
611 pipestat |= PIPE_VBLANK_INTERRUPT_ENABLE; 409 i915_enable_pipestat(dev_priv, pipe,
612 /* Clear any stale interrupt status */ 410 PIPE_VBLANK_INTERRUPT_ENABLE);
613 pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
614 PIPE_VBLANK_INTERRUPT_STATUS);
615 I915_WRITE(pipestat_reg, pipestat);
616 (void) I915_READ(pipestat_reg); /* Posting read */
617 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 411 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
618
619 return 0; 412 return 0;
620} 413}
621 414
@@ -625,37 +418,12 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
625void i915_disable_vblank(struct drm_device *dev, int pipe) 418void i915_disable_vblank(struct drm_device *dev, int pipe)
626{ 419{
627 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 420 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
628 u32 pipestat_reg = 0;
629 u32 pipestat;
630 u32 interrupt = 0;
631 unsigned long irqflags; 421 unsigned long irqflags;
632 422
633 switch (pipe) {
634 case 0:
635 pipestat_reg = PIPEASTAT;
636 interrupt = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
637 break;
638 case 1:
639 pipestat_reg = PIPEBSTAT;
640 interrupt = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
641 break;
642 default:
643 DRM_ERROR("tried to disable vblank on non-existent pipe %d\n",
644 pipe);
645 return;
646 break;
647 }
648
649 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 423 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
650 i915_disable_irq(dev_priv, interrupt); 424 i915_disable_pipestat(dev_priv, pipe,
651 pipestat = I915_READ(pipestat_reg); 425 PIPE_VBLANK_INTERRUPT_ENABLE |
652 pipestat &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE | 426 PIPE_START_VBLANK_INTERRUPT_ENABLE);
653 PIPE_VBLANK_INTERRUPT_ENABLE);
654 /* Clear any stale interrupt status */
655 pipestat |= (PIPE_START_VBLANK_INTERRUPT_STATUS |
656 PIPE_VBLANK_INTERRUPT_STATUS);
657 I915_WRITE(pipestat_reg, pipestat);
658 (void) I915_READ(pipestat_reg); /* Posting read */
659 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 427 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
660} 428}
661 429
@@ -696,123 +464,21 @@ int i915_vblank_pipe_get(struct drm_device *dev, void *data,
696int i915_vblank_swap(struct drm_device *dev, void *data, 464int i915_vblank_swap(struct drm_device *dev, void *data,
697 struct drm_file *file_priv) 465 struct drm_file *file_priv)
698{ 466{
699 drm_i915_private_t *dev_priv = dev->dev_private; 467 /* The delayed swap mechanism was fundamentally racy, and has been
700 drm_i915_vblank_swap_t *swap = data; 468 * removed. The model was that the client requested a delayed flip/swap
701 drm_i915_vbl_swap_t *vbl_swap, *vbl_old; 469 * from the kernel, then waited for vblank before continuing to perform
702 unsigned int pipe, seqtype, curseq; 470 * rendering. The problem was that the kernel might wake the client
703 unsigned long irqflags; 471 * up before it dispatched the vblank swap (since the lock has to be
704 struct list_head *list; 472 * held while touching the ringbuffer), in which case the client would
705 int ret; 473 * clear and start the next frame before the swap occurred, and
706 474 * flicker would occur in addition to likely missing the vblank.
707 if (!dev_priv || !dev_priv->sarea_priv) { 475 *
708 DRM_ERROR("%s called with no initialization\n", __func__); 476 * In the absence of this ioctl, userland falls back to a correct path
709 return -EINVAL; 477 * of waiting for a vblank, then dispatching the swap on its own.
710 } 478 * Context switching to userland and back is plenty fast enough for
711 479 * meeting the requirements of vblank swapping.
712 if (dev_priv->sarea_priv->rotation) {
713 DRM_DEBUG("Rotation not supported\n");
714 return -EINVAL;
715 }
716
717 if (swap->seqtype & ~(_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE |
718 _DRM_VBLANK_SECONDARY | _DRM_VBLANK_NEXTONMISS)) {
719 DRM_ERROR("Invalid sequence type 0x%x\n", swap->seqtype);
720 return -EINVAL;
721 }
722
723 pipe = (swap->seqtype & _DRM_VBLANK_SECONDARY) ? 1 : 0;
724
725 seqtype = swap->seqtype & (_DRM_VBLANK_RELATIVE | _DRM_VBLANK_ABSOLUTE);
726
727 if (!(dev_priv->vblank_pipe & (1 << pipe))) {
728 DRM_ERROR("Invalid pipe %d\n", pipe);
729 return -EINVAL;
730 }
731
732 spin_lock_irqsave(&dev->drw_lock, irqflags);
733
734 if (!drm_get_drawable_info(dev, swap->drawable)) {
735 spin_unlock_irqrestore(&dev->drw_lock, irqflags);
736 DRM_DEBUG("Invalid drawable ID %d\n", swap->drawable);
737 return -EINVAL;
738 }
739
740 spin_unlock_irqrestore(&dev->drw_lock, irqflags);
741
742 /*
743 * We take the ref here and put it when the swap actually completes
744 * in the tasklet.
745 */ 480 */
746 ret = drm_vblank_get(dev, pipe); 481 return -EINVAL;
747 if (ret)
748 return ret;
749 curseq = drm_vblank_count(dev, pipe);
750
751 if (seqtype == _DRM_VBLANK_RELATIVE)
752 swap->sequence += curseq;
753
754 if ((curseq - swap->sequence) <= (1<<23)) {
755 if (swap->seqtype & _DRM_VBLANK_NEXTONMISS) {
756 swap->sequence = curseq + 1;
757 } else {
758 DRM_DEBUG("Missed target sequence\n");
759 drm_vblank_put(dev, pipe);
760 return -EINVAL;
761 }
762 }
763
764 vbl_swap = drm_calloc(1, sizeof(*vbl_swap), DRM_MEM_DRIVER);
765
766 if (!vbl_swap) {
767 DRM_ERROR("Failed to allocate memory to queue swap\n");
768 drm_vblank_put(dev, pipe);
769 return -ENOMEM;
770 }
771
772 vbl_swap->drw_id = swap->drawable;
773 vbl_swap->pipe = pipe;
774 vbl_swap->sequence = swap->sequence;
775
776 spin_lock_irqsave(&dev_priv->swaps_lock, irqflags);
777
778 list_for_each(list, &dev_priv->vbl_swaps.head) {
779 vbl_old = list_entry(list, drm_i915_vbl_swap_t, head);
780
781 if (vbl_old->drw_id == swap->drawable &&
782 vbl_old->pipe == pipe &&
783 vbl_old->sequence == swap->sequence) {
784 spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
785 drm_vblank_put(dev, pipe);
786 drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
787 DRM_DEBUG("Already scheduled\n");
788 return 0;
789 }
790 }
791
792 if (dev_priv->swaps_pending >= 10) {
793 DRM_DEBUG("Too many swaps queued\n");
794 DRM_DEBUG(" pipe 0: %d pipe 1: %d\n",
795 drm_vblank_count(dev, 0),
796 drm_vblank_count(dev, 1));
797
798 list_for_each(list, &dev_priv->vbl_swaps.head) {
799 vbl_old = list_entry(list, drm_i915_vbl_swap_t, head);
800 DRM_DEBUG("\tdrw %x pipe %d seq %x\n",
801 vbl_old->drw_id, vbl_old->pipe,
802 vbl_old->sequence);
803 }
804 spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
805 drm_vblank_put(dev, pipe);
806 drm_free(vbl_swap, sizeof(*vbl_swap), DRM_MEM_DRIVER);
807 return -EBUSY;
808 }
809
810 list_add_tail(&vbl_swap->head, &dev_priv->vbl_swaps.head);
811 dev_priv->swaps_pending++;
812
813 spin_unlock_irqrestore(&dev_priv->swaps_lock, irqflags);
814
815 return 0;
816} 482}
817 483
818/* drm_dma.h hooks 484/* drm_dma.h hooks
@@ -822,37 +488,35 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
822 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 488 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
823 489
824 I915_WRITE(HWSTAM, 0xeffe); 490 I915_WRITE(HWSTAM, 0xeffe);
491 I915_WRITE(PIPEASTAT, 0);
492 I915_WRITE(PIPEBSTAT, 0);
825 I915_WRITE(IMR, 0xffffffff); 493 I915_WRITE(IMR, 0xffffffff);
826 I915_WRITE(IER, 0x0); 494 I915_WRITE(IER, 0x0);
495 (void) I915_READ(IER);
827} 496}
828 497
829int i915_driver_irq_postinstall(struct drm_device *dev) 498int i915_driver_irq_postinstall(struct drm_device *dev)
830{ 499{
831 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 500 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
832 int ret, num_pipes = 2;
833
834 spin_lock_init(&dev_priv->swaps_lock);
835 INIT_LIST_HEAD(&dev_priv->vbl_swaps.head);
836 INIT_WORK(&dev_priv->vblank_work, i915_vblank_work_handler);
837 dev_priv->swaps_pending = 0;
838
839 /* Set initial unmasked IRQs to just the selected vblank pipes. */
840 dev_priv->irq_mask_reg = ~0;
841
842 ret = drm_vblank_init(dev, num_pipes);
843 if (ret)
844 return ret;
845 501
846 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; 502 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
847 dev_priv->irq_mask_reg &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
848 dev_priv->irq_mask_reg &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
849 503
850 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ 504 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
851 505
852 dev_priv->irq_mask_reg &= I915_INTERRUPT_ENABLE_MASK; 506 /* Unmask the interrupts that we always want on. */
507 dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
508
509 dev_priv->pipestat[0] = 0;
510 dev_priv->pipestat[1] = 0;
511
512 /* Disable pipe interrupt enables, clear pending pipe status */
513 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
514 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
515 /* Clear pending interrupt status */
516 I915_WRITE(IIR, I915_READ(IIR));
853 517
854 I915_WRITE(IMR, dev_priv->irq_mask_reg);
855 I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK); 518 I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
519 I915_WRITE(IMR, dev_priv->irq_mask_reg);
856 (void) I915_READ(IER); 520 (void) I915_READ(IER);
857 521
858 opregion_enable_asle(dev); 522 opregion_enable_asle(dev);
@@ -864,7 +528,6 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
864void i915_driver_irq_uninstall(struct drm_device * dev) 528void i915_driver_irq_uninstall(struct drm_device * dev)
865{ 529{
866 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 530 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
867 u32 temp;
868 531
869 if (!dev_priv) 532 if (!dev_priv)
870 return; 533 return;
@@ -872,13 +535,12 @@ void i915_driver_irq_uninstall(struct drm_device * dev)
872 dev_priv->vblank_pipe = 0; 535 dev_priv->vblank_pipe = 0;
873 536
874 I915_WRITE(HWSTAM, 0xffffffff); 537 I915_WRITE(HWSTAM, 0xffffffff);
538 I915_WRITE(PIPEASTAT, 0);
539 I915_WRITE(PIPEBSTAT, 0);
875 I915_WRITE(IMR, 0xffffffff); 540 I915_WRITE(IMR, 0xffffffff);
876 I915_WRITE(IER, 0x0); 541 I915_WRITE(IER, 0x0);
877 542
878 temp = I915_READ(PIPEASTAT); 543 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
879 I915_WRITE(PIPEASTAT, temp); 544 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
880 temp = I915_READ(PIPEBSTAT); 545 I915_WRITE(IIR, I915_READ(IIR));
881 I915_WRITE(PIPEBSTAT, temp);
882 temp = I915_READ(IIR);
883 I915_WRITE(IIR, temp);
884} 546}
diff --git a/drivers/gpu/drm/i915/i915_opregion.c b/drivers/gpu/drm/i915/i915_opregion.c
index 1787a0c7e3ab..13ae731a33db 100644
--- a/drivers/gpu/drm/i915/i915_opregion.c
+++ b/drivers/gpu/drm/i915/i915_opregion.c
@@ -235,17 +235,15 @@ void opregion_enable_asle(struct drm_device *dev)
235 struct opregion_asle *asle = dev_priv->opregion.asle; 235 struct opregion_asle *asle = dev_priv->opregion.asle;
236 236
237 if (asle) { 237 if (asle) {
238 u32 pipeb_stats = I915_READ(PIPEBSTAT);
239 if (IS_MOBILE(dev)) { 238 if (IS_MOBILE(dev)) {
240 /* Many devices trigger events with a write to the 239 unsigned long irqflags;
241 legacy backlight controller, so we need to ensure 240
242 that it's able to generate interrupts */ 241 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
243 I915_WRITE(PIPEBSTAT, pipeb_stats |= 242 i915_enable_pipestat(dev_priv, 1,
244 I915_LEGACY_BLC_EVENT_ENABLE); 243 I915_LEGACY_BLC_EVENT_ENABLE);
245 i915_enable_irq(dev_priv, I915_ASLE_INTERRUPT | 244 spin_unlock_irqrestore(&dev_priv->user_irq_lock,
246 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT); 245 irqflags);
247 } else 246 }
248 i915_enable_irq(dev_priv, I915_ASLE_INTERRUPT);
249 247
250 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN | 248 asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN |
251 ASLE_PFMB_EN; 249 ASLE_PFMB_EN;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 5c2d9f206d05..9d24aaeb8a45 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -522,11 +522,15 @@
522#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0) 522#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0)
523#define DCC_ADDRESSING_MODE_MASK (3 << 0) 523#define DCC_ADDRESSING_MODE_MASK (3 << 0)
524#define DCC_CHANNEL_XOR_DISABLE (1 << 10) 524#define DCC_CHANNEL_XOR_DISABLE (1 << 10)
525#define DCC_CHANNEL_XOR_BIT_17 (1 << 9)
525 526
526/** 965 MCH register controlling DRAM channel configuration */ 527/** 965 MCH register controlling DRAM channel configuration */
527#define C0DRB3 0x10206 528#define C0DRB3 0x10206
528#define C1DRB3 0x10606 529#define C1DRB3 0x10606
529 530
531/** GM965 GM45 render standby register */
532#define MCHBAR_RENDER_STANDBY 0x111B8
533
530/* 534/*
531 * Overlay regs 535 * Overlay regs
532 */ 536 */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 603fe742ccd4..5d84027ee8f3 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -240,6 +240,13 @@ int i915_save_state(struct drm_device *dev)
240 240
241 pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB); 241 pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
242 242
243 /* Render Standby */
244 if (IS_I965G(dev) && IS_MOBILE(dev))
245 dev_priv->saveRENDERSTANDBY = I915_READ(MCHBAR_RENDER_STANDBY);
246
247 /* Hardware status page */
248 dev_priv->saveHWS = I915_READ(HWS_PGA);
249
243 /* Display arbitration control */ 250 /* Display arbitration control */
244 dev_priv->saveDSPARB = I915_READ(DSPARB); 251 dev_priv->saveDSPARB = I915_READ(DSPARB);
245 252
@@ -365,6 +372,14 @@ int i915_restore_state(struct drm_device *dev)
365 372
366 pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB); 373 pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
367 374
375 /* Render Standby */
376 if (IS_I965G(dev) && IS_MOBILE(dev))
377 I915_WRITE(MCHBAR_RENDER_STANDBY, dev_priv->saveRENDERSTANDBY);
378
379 /* Hardware status page */
380 I915_WRITE(HWS_PGA, dev_priv->saveHWS);
381
382 /* Display arbitration */
368 I915_WRITE(DSPARB, dev_priv->saveDSPARB); 383 I915_WRITE(DSPARB, dev_priv->saveDSPARB);
369 384
370 /* Pipe & plane A info */ 385 /* Pipe & plane A info */
diff --git a/drivers/gpu/drm/mga/mga_dma.c b/drivers/gpu/drm/mga/mga_dma.c
index c1d12dbfa8d8..b49c5ff29585 100644
--- a/drivers/gpu/drm/mga/mga_dma.c
+++ b/drivers/gpu/drm/mga/mga_dma.c
@@ -396,6 +396,7 @@ int mga_freelist_put(struct drm_device * dev, struct drm_buf * buf)
396int mga_driver_load(struct drm_device * dev, unsigned long flags) 396int mga_driver_load(struct drm_device * dev, unsigned long flags)
397{ 397{
398 drm_mga_private_t *dev_priv; 398 drm_mga_private_t *dev_priv;
399 int ret;
399 400
400 dev_priv = drm_alloc(sizeof(drm_mga_private_t), DRM_MEM_DRIVER); 401 dev_priv = drm_alloc(sizeof(drm_mga_private_t), DRM_MEM_DRIVER);
401 if (!dev_priv) 402 if (!dev_priv)
@@ -415,6 +416,13 @@ int mga_driver_load(struct drm_device * dev, unsigned long flags)
415 dev->types[7] = _DRM_STAT_PRIMARY; 416 dev->types[7] = _DRM_STAT_PRIMARY;
416 dev->types[8] = _DRM_STAT_SECONDARY; 417 dev->types[8] = _DRM_STAT_SECONDARY;
417 418
419 ret = drm_vblank_init(dev, 1);
420
421 if (ret) {
422 (void) mga_driver_unload(dev);
423 return ret;
424 }
425
418 return 0; 426 return 0;
419} 427}
420 428
diff --git a/drivers/gpu/drm/mga/mga_irq.c b/drivers/gpu/drm/mga/mga_irq.c
index bab42f41188b..daa6041a483a 100644
--- a/drivers/gpu/drm/mga/mga_irq.c
+++ b/drivers/gpu/drm/mga/mga_irq.c
@@ -152,11 +152,6 @@ void mga_driver_irq_preinstall(struct drm_device * dev)
152int mga_driver_irq_postinstall(struct drm_device *dev) 152int mga_driver_irq_postinstall(struct drm_device *dev)
153{ 153{
154 drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private; 154 drm_mga_private_t *dev_priv = (drm_mga_private_t *) dev->dev_private;
155 int ret;
156
157 ret = drm_vblank_init(dev, 1);
158 if (ret)
159 return ret;
160 155
161 DRM_INIT_WAITQUEUE(&dev_priv->fence_queue); 156 DRM_INIT_WAITQUEUE(&dev_priv->fence_queue);
162 157
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index 3265d53ba91f..601f4c0e5da5 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -45,6 +45,7 @@ static struct drm_driver driver = {
45 DRIVER_USE_AGP | DRIVER_USE_MTRR | DRIVER_PCI_DMA | DRIVER_SG | 45 DRIVER_USE_AGP | DRIVER_USE_MTRR | DRIVER_PCI_DMA | DRIVER_SG |
46 DRIVER_HAVE_DMA | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED, 46 DRIVER_HAVE_DMA | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED,
47 .dev_priv_size = sizeof(drm_r128_buf_priv_t), 47 .dev_priv_size = sizeof(drm_r128_buf_priv_t),
48 .load = r128_driver_load,
48 .preclose = r128_driver_preclose, 49 .preclose = r128_driver_preclose,
49 .lastclose = r128_driver_lastclose, 50 .lastclose = r128_driver_lastclose,
50 .get_vblank_counter = r128_get_vblank_counter, 51 .get_vblank_counter = r128_get_vblank_counter,
@@ -84,6 +85,11 @@ static struct drm_driver driver = {
84 .patchlevel = DRIVER_PATCHLEVEL, 85 .patchlevel = DRIVER_PATCHLEVEL,
85}; 86};
86 87
88int r128_driver_load(struct drm_device * dev, unsigned long flags)
89{
90 return drm_vblank_init(dev, 1);
91}
92
87static int __init r128_init(void) 93static int __init r128_init(void)
88{ 94{
89 driver.num_ioctls = r128_max_ioctl; 95 driver.num_ioctls = r128_max_ioctl;
diff --git a/drivers/gpu/drm/r128/r128_drv.h b/drivers/gpu/drm/r128/r128_drv.h
index 5898b274279d..797a26c42dab 100644
--- a/drivers/gpu/drm/r128/r128_drv.h
+++ b/drivers/gpu/drm/r128/r128_drv.h
@@ -159,6 +159,7 @@ extern void r128_driver_irq_preinstall(struct drm_device * dev);
159extern int r128_driver_irq_postinstall(struct drm_device *dev); 159extern int r128_driver_irq_postinstall(struct drm_device *dev);
160extern void r128_driver_irq_uninstall(struct drm_device * dev); 160extern void r128_driver_irq_uninstall(struct drm_device * dev);
161extern void r128_driver_lastclose(struct drm_device * dev); 161extern void r128_driver_lastclose(struct drm_device * dev);
162extern int r128_driver_load(struct drm_device * dev, unsigned long flags);
162extern void r128_driver_preclose(struct drm_device * dev, 163extern void r128_driver_preclose(struct drm_device * dev,
163 struct drm_file *file_priv); 164 struct drm_file *file_priv);
164 165
diff --git a/drivers/gpu/drm/r128/r128_irq.c b/drivers/gpu/drm/r128/r128_irq.c
index d7349012a680..69810fb8ac49 100644
--- a/drivers/gpu/drm/r128/r128_irq.c
+++ b/drivers/gpu/drm/r128/r128_irq.c
@@ -102,7 +102,7 @@ void r128_driver_irq_preinstall(struct drm_device * dev)
102 102
103int r128_driver_irq_postinstall(struct drm_device *dev) 103int r128_driver_irq_postinstall(struct drm_device *dev)
104{ 104{
105 return drm_vblank_init(dev, 1); 105 return 0;
106} 106}
107 107
108void r128_driver_irq_uninstall(struct drm_device * dev) 108void r128_driver_irq_uninstall(struct drm_device * dev)
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index 59a2132a8f57..dcebb4bee7aa 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -653,15 +653,16 @@ static void radeon_cp_init_ring_buffer(struct drm_device * dev,
653 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7); 653 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
654 654
655 /* Turn on bus mastering */ 655 /* Turn on bus mastering */
656 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) || 656 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
657 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
658 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) { 657 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
659 /* rs400, rs690/rs740 */ 658 /* rs600/rs690/rs740 */
660 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS400_BUS_MASTER_DIS; 659 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
661 RADEON_WRITE(RADEON_BUS_CNTL, tmp); 660 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
662 } else if (!(((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) || 661 } else if (((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV350) ||
663 ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R423))) { 662 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
664 /* r1xx, r2xx, r300, r(v)350, r420/r481, rs480 */ 663 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
664 ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
665 /* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
665 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS; 666 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
666 RADEON_WRITE(RADEON_BUS_CNTL, tmp); 667 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
667 } /* PCIE cards appears to not need this */ 668 } /* PCIE cards appears to not need this */
@@ -1750,6 +1751,18 @@ int radeon_driver_load(struct drm_device *dev, unsigned long flags)
1750 else 1751 else
1751 dev_priv->flags |= RADEON_IS_PCI; 1752 dev_priv->flags |= RADEON_IS_PCI;
1752 1753
1754 ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
1755 drm_get_resource_len(dev, 2), _DRM_REGISTERS,
1756 _DRM_READ_ONLY | _DRM_DRIVER, &dev_priv->mmio);
1757 if (ret != 0)
1758 return ret;
1759
1760 ret = drm_vblank_init(dev, 2);
1761 if (ret) {
1762 radeon_driver_unload(dev);
1763 return ret;
1764 }
1765
1753 DRM_DEBUG("%s card detected\n", 1766 DRM_DEBUG("%s card detected\n",
1754 ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI")))); 1767 ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
1755 return ret; 1768 return ret;
@@ -1766,12 +1779,6 @@ int radeon_driver_firstopen(struct drm_device *dev)
1766 1779
1767 dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE; 1780 dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
1768 1781
1769 ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
1770 drm_get_resource_len(dev, 2), _DRM_REGISTERS,
1771 _DRM_READ_ONLY, &dev_priv->mmio);
1772 if (ret != 0)
1773 return ret;
1774
1775 dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0); 1782 dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
1776 ret = drm_addmap(dev, dev_priv->fb_aper_offset, 1783 ret = drm_addmap(dev, dev_priv->fb_aper_offset,
1777 drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER, 1784 drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
@@ -1787,6 +1794,9 @@ int radeon_driver_unload(struct drm_device *dev)
1787 drm_radeon_private_t *dev_priv = dev->dev_private; 1794 drm_radeon_private_t *dev_priv = dev->dev_private;
1788 1795
1789 DRM_DEBUG("\n"); 1796 DRM_DEBUG("\n");
1797
1798 drm_rmmap(dev, dev_priv->mmio);
1799
1790 drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER); 1800 drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
1791 1801
1792 dev->dev_private = NULL; 1802 dev->dev_private = NULL;
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index 4dbb813910c3..3bbb871b25d5 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -287,7 +287,6 @@ typedef struct drm_radeon_private {
287 unsigned long gart_textures_offset; 287 unsigned long gart_textures_offset;
288 288
289 drm_local_map_t *sarea; 289 drm_local_map_t *sarea;
290 drm_local_map_t *mmio;
291 drm_local_map_t *cp_ring; 290 drm_local_map_t *cp_ring;
292 drm_local_map_t *ring_rptr; 291 drm_local_map_t *ring_rptr;
293 drm_local_map_t *gart_textures; 292 drm_local_map_t *gart_textures;
@@ -300,7 +299,6 @@ typedef struct drm_radeon_private {
300 atomic_t swi_emitted; 299 atomic_t swi_emitted;
301 int vblank_crtc; 300 int vblank_crtc;
302 uint32_t irq_enable_reg; 301 uint32_t irq_enable_reg;
303 int irq_enabled;
304 uint32_t r500_disp_irq_reg; 302 uint32_t r500_disp_irq_reg;
305 303
306 struct radeon_surface surfaces[RADEON_MAX_SURFACES]; 304 struct radeon_surface surfaces[RADEON_MAX_SURFACES];
@@ -318,6 +316,7 @@ typedef struct drm_radeon_private {
318 316
319 int num_gb_pipes; 317 int num_gb_pipes;
320 int track_flush; 318 int track_flush;
319 drm_local_map_t *mmio;
321} drm_radeon_private_t; 320} drm_radeon_private_t;
322 321
323typedef struct drm_radeon_buf_priv { 322typedef struct drm_radeon_buf_priv {
@@ -447,12 +446,12 @@ extern int r300_do_cp_cmdbuf(struct drm_device *dev,
447 * handling, not bus mastering itself. 446 * handling, not bus mastering itself.
448 */ 447 */
449#define RADEON_BUS_CNTL 0x0030 448#define RADEON_BUS_CNTL 0x0030
450/* r1xx, r2xx, r300, r(v)350, r420/r481, rs480 */ 449/* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
451# define RADEON_BUS_MASTER_DIS (1 << 6) 450# define RADEON_BUS_MASTER_DIS (1 << 6)
452/* rs400, rs690/rs740 */ 451/* rs600/rs690/rs740 */
453# define RS400_BUS_MASTER_DIS (1 << 14) 452# define RS600_BUS_MASTER_DIS (1 << 14)
454# define RS400_MSI_REARM (1 << 20) 453# define RS600_MSI_REARM (1 << 20)
455/* see RS480_MSI_REARM in AIC_CNTL for rs480 */ 454/* see RS400_MSI_REARM in AIC_CNTL for rs480 */
456 455
457#define RADEON_BUS_CNTL1 0x0034 456#define RADEON_BUS_CNTL1 0x0034
458# define RADEON_PMI_BM_DIS (1 << 2) 457# define RADEON_PMI_BM_DIS (1 << 2)
@@ -937,7 +936,7 @@ extern int r300_do_cp_cmdbuf(struct drm_device *dev,
937 936
938#define RADEON_AIC_CNTL 0x01d0 937#define RADEON_AIC_CNTL 0x01d0
939# define RADEON_PCIGART_TRANSLATE_EN (1 << 0) 938# define RADEON_PCIGART_TRANSLATE_EN (1 << 0)
940# define RS480_MSI_REARM (1 << 3) 939# define RS400_MSI_REARM (1 << 3)
941#define RADEON_AIC_STAT 0x01d4 940#define RADEON_AIC_STAT 0x01d4
942#define RADEON_AIC_PT_BASE 0x01d8 941#define RADEON_AIC_PT_BASE 0x01d8
943#define RADEON_AIC_LO_ADDR 0x01dc 942#define RADEON_AIC_LO_ADDR 0x01dc
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
index 5079f7054a2f..99be11418ac2 100644
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ b/drivers/gpu/drm/radeon/radeon_irq.c
@@ -44,7 +44,8 @@ void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state)
44 else 44 else
45 dev_priv->irq_enable_reg &= ~mask; 45 dev_priv->irq_enable_reg &= ~mask;
46 46
47 RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg); 47 if (!dev->irq_enabled)
48 RADEON_WRITE(RADEON_GEN_INT_CNTL, dev_priv->irq_enable_reg);
48} 49}
49 50
50static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state) 51static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state)
@@ -56,7 +57,8 @@ static void r500_vbl_irq_set_state(struct drm_device *dev, u32 mask, int state)
56 else 57 else
57 dev_priv->r500_disp_irq_reg &= ~mask; 58 dev_priv->r500_disp_irq_reg &= ~mask;
58 59
59 RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg); 60 if (!dev->irq_enabled)
61 RADEON_WRITE(R500_DxMODE_INT_MASK, dev_priv->r500_disp_irq_reg);
60} 62}
61 63
62int radeon_enable_vblank(struct drm_device *dev, int crtc) 64int radeon_enable_vblank(struct drm_device *dev, int crtc)
@@ -337,15 +339,10 @@ int radeon_driver_irq_postinstall(struct drm_device *dev)
337{ 339{
338 drm_radeon_private_t *dev_priv = 340 drm_radeon_private_t *dev_priv =
339 (drm_radeon_private_t *) dev->dev_private; 341 (drm_radeon_private_t *) dev->dev_private;
340 int ret;
341 342
342 atomic_set(&dev_priv->swi_emitted, 0); 343 atomic_set(&dev_priv->swi_emitted, 0);
343 DRM_INIT_WAITQUEUE(&dev_priv->swi_queue); 344 DRM_INIT_WAITQUEUE(&dev_priv->swi_queue);
344 345
345 ret = drm_vblank_init(dev, 2);
346 if (ret)
347 return ret;
348
349 dev->max_vblank_count = 0x001fffff; 346 dev->max_vblank_count = 0x001fffff;
350 347
351 radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1); 348 radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
@@ -360,8 +357,6 @@ void radeon_driver_irq_uninstall(struct drm_device * dev)
360 if (!dev_priv) 357 if (!dev_priv)
361 return; 358 return;
362 359
363 dev_priv->irq_enabled = 0;
364
365 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690) 360 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RS690)
366 RADEON_WRITE(R500_DxMODE_INT_MASK, 0); 361 RADEON_WRITE(R500_DxMODE_INT_MASK, 0);
367 /* Disable *all* interrupts */ 362 /* Disable *all* interrupts */
diff --git a/drivers/gpu/drm/via/via_irq.c b/drivers/gpu/drm/via/via_irq.c
index 665d319b927b..c248c1d37268 100644
--- a/drivers/gpu/drm/via/via_irq.c
+++ b/drivers/gpu/drm/via/via_irq.c
@@ -314,7 +314,6 @@ int via_driver_irq_postinstall(struct drm_device *dev)
314 if (!dev_priv) 314 if (!dev_priv)
315 return -EINVAL; 315 return -EINVAL;
316 316
317 drm_vblank_init(dev, 1);
318 status = VIA_READ(VIA_REG_INTERRUPT); 317 status = VIA_READ(VIA_REG_INTERRUPT);
319 VIA_WRITE(VIA_REG_INTERRUPT, status | VIA_IRQ_GLOBAL 318 VIA_WRITE(VIA_REG_INTERRUPT, status | VIA_IRQ_GLOBAL
320 | dev_priv->irq_enable_mask); 319 | dev_priv->irq_enable_mask);
diff --git a/drivers/gpu/drm/via/via_map.c b/drivers/gpu/drm/via/via_map.c
index a967556be014..2c4f0b485792 100644
--- a/drivers/gpu/drm/via/via_map.c
+++ b/drivers/gpu/drm/via/via_map.c
@@ -107,8 +107,17 @@ int via_driver_load(struct drm_device *dev, unsigned long chipset)
107 ret = drm_sman_init(&dev_priv->sman, 2, 12, 8); 107 ret = drm_sman_init(&dev_priv->sman, 2, 12, 8);
108 if (ret) { 108 if (ret) {
109 drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER); 109 drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
110 return ret;
110 } 111 }
111 return ret; 112
113 ret = drm_vblank_init(dev, 1);
114 if (ret) {
115 drm_sman_takedown(&dev_priv->sman);
116 drm_free(dev_priv, sizeof(drm_via_private_t), DRM_MEM_DRIVER);
117 return ret;
118 }
119
120 return 0;
112} 121}
113 122
114int via_driver_unload(struct drm_device *dev) 123int via_driver_unload(struct drm_device *dev)
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-11 20:01:49 -0500