aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915
diff options
context:
space:
mode:
authorEric Anholt <eric@anholt.net>2008-07-30 15:06:12 -0400
committerDave Airlie <airlied@linux.ie>2008-10-17 17:10:12 -0400
commit673a394b1e3b69be886ff24abfd6df97c52e8d08 (patch)
tree61ca8299333ab50ffc46cf328b20eb25133392ff /drivers/gpu/drm/i915
parentd1d8c925b71dd6753bf438f9e14a9e5c5183bcc6 (diff)
drm: Add GEM ("graphics execution manager") to i915 driver.
GEM allows the creation of persistent buffer objects accessible by the graphics device through new ioctls for managing execution of commands on the device. The userland API is almost entirely driver-specific to ensure that any driver building on this model can easily map the interface to individual driver requirements. GEM is used by the 2d driver for managing its internal state allocations and will be used for pixmap storage to reduce memory consumption and enable zero-copy GLX_EXT_texture_from_pixmap, and in the 3d driver is used to enable GL_EXT_framebuffer_object and GL_ARB_pixel_buffer_object. Signed-off-by: Eric Anholt <eric@anholt.net> Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'drivers/gpu/drm/i915')
-rw-r--r--drivers/gpu/drm/i915/Makefile6
-rw-r--r--drivers/gpu/drm/i915/i915_dma.c94
-rw-r--r--drivers/gpu/drm/i915/i915_drv.c11
-rw-r--r--drivers/gpu/drm/i915/i915_drv.h253
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c2497
-rw-r--r--drivers/gpu/drm/i915/i915_gem_debug.c201
-rw-r--r--drivers/gpu/drm/i915/i915_gem_proc.c292
-rw-r--r--drivers/gpu/drm/i915/i915_gem_tiling.c256
-rw-r--r--drivers/gpu/drm/i915/i915_irq.c19
-rw-r--r--drivers/gpu/drm/i915/i915_reg.h37
10 files changed, 3621 insertions, 45 deletions
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index c4bbda662e26..5ba78e4fd2b5 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -4,7 +4,11 @@
4 4
5ccflags-y := -Iinclude/drm 5ccflags-y := -Iinclude/drm
6i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_opregion.o \ 6i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o i915_opregion.o \
7 i915_suspend.o 7 i915_suspend.o \
8 i915_gem.o \
9 i915_gem_debug.o \
10 i915_gem_proc.o \
11 i915_gem_tiling.o
8 12
9i915-$(CONFIG_COMPAT) += i915_ioc32.o 13i915-$(CONFIG_COMPAT) += i915_ioc32.o
10 14
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 63c6803d471b..f167ff68a4bf 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -170,24 +170,31 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
170 dev_priv->sarea_priv = (drm_i915_sarea_t *) 170 dev_priv->sarea_priv = (drm_i915_sarea_t *)
171 ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset); 171 ((u8 *) dev_priv->sarea->handle + init->sarea_priv_offset);
172 172
173 dev_priv->ring.Start = init->ring_start; 173 if (init->ring_size != 0) {
174 dev_priv->ring.End = init->ring_end; 174 if (dev_priv->ring.ring_obj != NULL) {
175 dev_priv->ring.Size = init->ring_size; 175 i915_dma_cleanup(dev);
176 dev_priv->ring.tail_mask = dev_priv->ring.Size - 1; 176 DRM_ERROR("Client tried to initialize ringbuffer in "
177 "GEM mode\n");
178 return -EINVAL;
179 }
177 180
178 dev_priv->ring.map.offset = init->ring_start; 181 dev_priv->ring.Size = init->ring_size;
179 dev_priv->ring.map.size = init->ring_size; 182 dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
180 dev_priv->ring.map.type = 0;
181 dev_priv->ring.map.flags = 0;
182 dev_priv->ring.map.mtrr = 0;
183 183
184 drm_core_ioremap(&dev_priv->ring.map, dev); 184 dev_priv->ring.map.offset = init->ring_start;
185 dev_priv->ring.map.size = init->ring_size;
186 dev_priv->ring.map.type = 0;
187 dev_priv->ring.map.flags = 0;
188 dev_priv->ring.map.mtrr = 0;
185 189
186 if (dev_priv->ring.map.handle == NULL) { 190 drm_core_ioremap(&dev_priv->ring.map, dev);
187 i915_dma_cleanup(dev); 191
188 DRM_ERROR("can not ioremap virtual address for" 192 if (dev_priv->ring.map.handle == NULL) {
189 " ring buffer\n"); 193 i915_dma_cleanup(dev);
190 return -ENOMEM; 194 DRM_ERROR("can not ioremap virtual address for"
195 " ring buffer\n");
196 return -ENOMEM;
197 }
191 } 198 }
192 199
193 dev_priv->ring.virtual_start = dev_priv->ring.map.handle; 200 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
@@ -377,9 +384,10 @@ static int i915_emit_cmds(struct drm_device * dev, int __user * buffer, int dwor
377 return 0; 384 return 0;
378} 385}
379 386
380static int i915_emit_box(struct drm_device * dev, 387int
381 struct drm_clip_rect __user * boxes, 388i915_emit_box(struct drm_device *dev,
382 int i, int DR1, int DR4) 389 struct drm_clip_rect __user *boxes,
390 int i, int DR1, int DR4)
383{ 391{
384 drm_i915_private_t *dev_priv = dev->dev_private; 392 drm_i915_private_t *dev_priv = dev->dev_private;
385 struct drm_clip_rect box; 393 struct drm_clip_rect box;
@@ -681,6 +689,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
681 case I915_PARAM_LAST_DISPATCH: 689 case I915_PARAM_LAST_DISPATCH:
682 value = READ_BREADCRUMB(dev_priv); 690 value = READ_BREADCRUMB(dev_priv);
683 break; 691 break;
692 case I915_PARAM_HAS_GEM:
693 value = 1;
694 break;
684 default: 695 default:
685 DRM_ERROR("Unknown parameter %d\n", param->param); 696 DRM_ERROR("Unknown parameter %d\n", param->param);
686 return -EINVAL; 697 return -EINVAL;
@@ -784,6 +795,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
784 memset(dev_priv, 0, sizeof(drm_i915_private_t)); 795 memset(dev_priv, 0, sizeof(drm_i915_private_t));
785 796
786 dev->dev_private = (void *)dev_priv; 797 dev->dev_private = (void *)dev_priv;
798 dev_priv->dev = dev;
787 799
788 /* Add register map (needed for suspend/resume) */ 800 /* Add register map (needed for suspend/resume) */
789 base = drm_get_resource_start(dev, mmio_bar); 801 base = drm_get_resource_start(dev, mmio_bar);
@@ -793,6 +805,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
793 _DRM_KERNEL | _DRM_DRIVER, 805 _DRM_KERNEL | _DRM_DRIVER,
794 &dev_priv->mmio_map); 806 &dev_priv->mmio_map);
795 807
808 i915_gem_load(dev);
809
796 /* Init HWS */ 810 /* Init HWS */
797 if (!I915_NEED_GFX_HWS(dev)) { 811 if (!I915_NEED_GFX_HWS(dev)) {
798 ret = i915_init_phys_hws(dev); 812 ret = i915_init_phys_hws(dev);
@@ -838,6 +852,25 @@ int i915_driver_unload(struct drm_device *dev)
838 return 0; 852 return 0;
839} 853}
840 854
855int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv)
856{
857 struct drm_i915_file_private *i915_file_priv;
858
859 DRM_DEBUG("\n");
860 i915_file_priv = (struct drm_i915_file_private *)
861 drm_alloc(sizeof(*i915_file_priv), DRM_MEM_FILES);
862
863 if (!i915_file_priv)
864 return -ENOMEM;
865
866 file_priv->driver_priv = i915_file_priv;
867
868 i915_file_priv->mm.last_gem_seqno = 0;
869 i915_file_priv->mm.last_gem_throttle_seqno = 0;
870
871 return 0;
872}
873
841void i915_driver_lastclose(struct drm_device * dev) 874void i915_driver_lastclose(struct drm_device * dev)
842{ 875{
843 drm_i915_private_t *dev_priv = dev->dev_private; 876 drm_i915_private_t *dev_priv = dev->dev_private;
@@ -845,6 +878,8 @@ void i915_driver_lastclose(struct drm_device * dev)
845 if (!dev_priv) 878 if (!dev_priv)
846 return; 879 return;
847 880
881 i915_gem_lastclose(dev);
882
848 if (dev_priv->agp_heap) 883 if (dev_priv->agp_heap)
849 i915_mem_takedown(&(dev_priv->agp_heap)); 884 i915_mem_takedown(&(dev_priv->agp_heap));
850 885
@@ -857,6 +892,13 @@ void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
857 i915_mem_release(dev, file_priv, dev_priv->agp_heap); 892 i915_mem_release(dev, file_priv, dev_priv->agp_heap);
858} 893}
859 894
895void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
896{
897 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
898
899 drm_free(i915_file_priv, sizeof(*i915_file_priv), DRM_MEM_FILES);
900}
901
860struct drm_ioctl_desc i915_ioctls[] = { 902struct drm_ioctl_desc i915_ioctls[] = {
861 DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), 903 DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
862 DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH), 904 DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
@@ -875,6 +917,22 @@ struct drm_ioctl_desc i915_ioctls[] = {
875 DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE, i915_vblank_pipe_get, DRM_AUTH ), 917 DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE, i915_vblank_pipe_get, DRM_AUTH ),
876 DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH), 918 DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
877 DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH), 919 DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH),
920 DRM_IOCTL_DEF(DRM_I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH),
921 DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
922 DRM_IOCTL_DEF(DRM_I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
923 DRM_IOCTL_DEF(DRM_I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
924 DRM_IOCTL_DEF(DRM_I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH),
925 DRM_IOCTL_DEF(DRM_I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH),
926 DRM_IOCTL_DEF(DRM_I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH),
927 DRM_IOCTL_DEF(DRM_I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH),
928 DRM_IOCTL_DEF(DRM_I915_GEM_CREATE, i915_gem_create_ioctl, 0),
929 DRM_IOCTL_DEF(DRM_I915_GEM_PREAD, i915_gem_pread_ioctl, 0),
930 DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, 0),
931 DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, 0),
932 DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, 0),
933 DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0),
934 DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0),
935 DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0),
878}; 936};
879 937
880int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls); 938int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 37af03f4db36..a80ead215282 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -85,12 +85,15 @@ static struct drm_driver driver = {
85 /* don't use mtrr's here, the Xserver or user space app should 85 /* don't use mtrr's here, the Xserver or user space app should
86 * deal with them for intel hardware. 86 * deal with them for intel hardware.
87 */ 87 */
88 .driver_features = DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | 88 .driver_features =
89 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED, 89 DRIVER_USE_AGP | DRIVER_REQUIRE_AGP | /* DRIVER_USE_MTRR |*/
90 DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM,
90 .load = i915_driver_load, 91 .load = i915_driver_load,
91 .unload = i915_driver_unload, 92 .unload = i915_driver_unload,
93 .open = i915_driver_open,
92 .lastclose = i915_driver_lastclose, 94 .lastclose = i915_driver_lastclose,
93 .preclose = i915_driver_preclose, 95 .preclose = i915_driver_preclose,
96 .postclose = i915_driver_postclose,
94 .suspend = i915_suspend, 97 .suspend = i915_suspend,
95 .resume = i915_resume, 98 .resume = i915_resume,
96 .device_is_agp = i915_driver_device_is_agp, 99 .device_is_agp = i915_driver_device_is_agp,
@@ -104,6 +107,10 @@ static struct drm_driver driver = {
104 .reclaim_buffers = drm_core_reclaim_buffers, 107 .reclaim_buffers = drm_core_reclaim_buffers,
105 .get_map_ofs = drm_core_get_map_ofs, 108 .get_map_ofs = drm_core_get_map_ofs,
106 .get_reg_ofs = drm_core_get_reg_ofs, 109 .get_reg_ofs = drm_core_get_reg_ofs,
110 .proc_init = i915_gem_proc_init,
111 .proc_cleanup = i915_gem_proc_cleanup,
112 .gem_init_object = i915_gem_init_object,
113 .gem_free_object = i915_gem_free_object,
107 .ioctls = i915_ioctls, 114 .ioctls = i915_ioctls,
108 .fops = { 115 .fops = {
109 .owner = THIS_MODULE, 116 .owner = THIS_MODULE,
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index d1a02bead458..87b071ab8647 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -39,7 +39,7 @@
39 39
40#define DRIVER_NAME "i915" 40#define DRIVER_NAME "i915"
41#define DRIVER_DESC "Intel Graphics" 41#define DRIVER_DESC "Intel Graphics"
42#define DRIVER_DATE "20060119" 42#define DRIVER_DATE "20080730"
43 43
44enum pipe { 44enum pipe {
45 PIPE_A = 0, 45 PIPE_A = 0,
@@ -60,16 +60,23 @@ enum pipe {
60#define DRIVER_MINOR 6 60#define DRIVER_MINOR 6
61#define DRIVER_PATCHLEVEL 0 61#define DRIVER_PATCHLEVEL 0
62 62
63#define WATCH_COHERENCY 0
64#define WATCH_BUF 0
65#define WATCH_EXEC 0
66#define WATCH_LRU 0
67#define WATCH_RELOC 0
68#define WATCH_INACTIVE 0
69#define WATCH_PWRITE 0
70
63typedef struct _drm_i915_ring_buffer { 71typedef struct _drm_i915_ring_buffer {
64 int tail_mask; 72 int tail_mask;
65 unsigned long Start;
66 unsigned long End;
67 unsigned long Size; 73 unsigned long Size;
68 u8 *virtual_start; 74 u8 *virtual_start;
69 int head; 75 int head;
70 int tail; 76 int tail;
71 int space; 77 int space;
72 drm_local_map_t map; 78 drm_local_map_t map;
79 struct drm_gem_object *ring_obj;
73} drm_i915_ring_buffer_t; 80} drm_i915_ring_buffer_t;
74 81
75struct mem_block { 82struct mem_block {
@@ -101,6 +108,8 @@ struct intel_opregion {
101}; 108};
102 109
103typedef struct drm_i915_private { 110typedef struct drm_i915_private {
111 struct drm_device *dev;
112
104 drm_local_map_t *sarea; 113 drm_local_map_t *sarea;
105 drm_local_map_t *mmio_map; 114 drm_local_map_t *mmio_map;
106 115
@@ -113,6 +122,7 @@ typedef struct drm_i915_private {
113 uint32_t counter; 122 uint32_t counter;
114 unsigned int status_gfx_addr; 123 unsigned int status_gfx_addr;
115 drm_local_map_t hws_map; 124 drm_local_map_t hws_map;
125 struct drm_gem_object *hws_obj;
116 126
117 unsigned int cpp; 127 unsigned int cpp;
118 int back_offset; 128 int back_offset;
@@ -122,7 +132,6 @@ typedef struct drm_i915_private {
122 132
123 wait_queue_head_t irq_queue; 133 wait_queue_head_t irq_queue;
124 atomic_t irq_received; 134 atomic_t irq_received;
125 atomic_t irq_emitted;
126 /** Protects user_irq_refcount and irq_mask_reg */ 135 /** Protects user_irq_refcount and irq_mask_reg */
127 spinlock_t user_irq_lock; 136 spinlock_t user_irq_lock;
128 /** Refcount for i915_user_irq_get() versus i915_user_irq_put(). */ 137 /** Refcount for i915_user_irq_get() versus i915_user_irq_put(). */
@@ -230,8 +239,174 @@ typedef struct drm_i915_private {
230 u8 saveDACMASK; 239 u8 saveDACMASK;
231 u8 saveDACDATA[256*3]; /* 256 3-byte colors */ 240 u8 saveDACDATA[256*3]; /* 256 3-byte colors */
232 u8 saveCR[37]; 241 u8 saveCR[37];
242
243 struct {
244 struct drm_mm gtt_space;
245
246 /**
247 * List of objects currently involved in rendering from the
248 * ringbuffer.
249 *
250 * A reference is held on the buffer while on this list.
251 */
252 struct list_head active_list;
253
254 /**
255 * List of objects which are not in the ringbuffer but which
256 * still have a write_domain which needs to be flushed before
257 * unbinding.
258 *
259 * A reference is held on the buffer while on this list.
260 */
261 struct list_head flushing_list;
262
263 /**
264 * LRU list of objects which are not in the ringbuffer and
265 * are ready to unbind, but are still in the GTT.
266 *
267 * A reference is not held on the buffer while on this list,
268 * as merely being GTT-bound shouldn't prevent its being
269 * freed, and we'll pull it off the list in the free path.
270 */
271 struct list_head inactive_list;
272
273 /**
274 * List of breadcrumbs associated with GPU requests currently
275 * outstanding.
276 */
277 struct list_head request_list;
278
279 /**
280 * We leave the user IRQ off as much as possible,
281 * but this means that requests will finish and never
282 * be retired once the system goes idle. Set a timer to
283 * fire periodically while the ring is running. When it
284 * fires, go retire requests.
285 */
286 struct delayed_work retire_work;
287
288 uint32_t next_gem_seqno;
289
290 /**
291 * Waiting sequence number, if any
292 */
293 uint32_t waiting_gem_seqno;
294
295 /**
296 * Last seq seen at irq time
297 */
298 uint32_t irq_gem_seqno;
299
300 /**
301 * Flag if the X Server, and thus DRM, is not currently in
302 * control of the device.
303 *
304 * This is set between LeaveVT and EnterVT. It needs to be
305 * replaced with a semaphore. It also needs to be
306 * transitioned away from for kernel modesetting.
307 */
308 int suspended;
309
310 /**
311 * Flag if the hardware appears to be wedged.
312 *
313 * This is set when attempts to idle the device timeout.
314 * It prevents command submission from occuring and makes
315 * every pending request fail
316 */
317 int wedged;
318
319 /** Bit 6 swizzling required for X tiling */
320 uint32_t bit_6_swizzle_x;
321 /** Bit 6 swizzling required for Y tiling */
322 uint32_t bit_6_swizzle_y;
323 } mm;
233} drm_i915_private_t; 324} drm_i915_private_t;
234 325
326/** driver private structure attached to each drm_gem_object */
327struct drm_i915_gem_object {
328 struct drm_gem_object *obj;
329
330 /** Current space allocated to this object in the GTT, if any. */
331 struct drm_mm_node *gtt_space;
332
333 /** This object's place on the active/flushing/inactive lists */
334 struct list_head list;
335
336 /**
337 * This is set if the object is on the active or flushing lists
338 * (has pending rendering), and is not set if it's on inactive (ready
339 * to be unbound).
340 */
341 int active;
342
343 /**
344 * This is set if the object has been written to since last bound
345 * to the GTT
346 */
347 int dirty;
348
349 /** AGP memory structure for our GTT binding. */
350 DRM_AGP_MEM *agp_mem;
351
352 struct page **page_list;
353
354 /**
355 * Current offset of the object in GTT space.
356 *
357 * This is the same as gtt_space->start
358 */
359 uint32_t gtt_offset;
360
361 /** Boolean whether this object has a valid gtt offset. */
362 int gtt_bound;
363
364 /** How many users have pinned this object in GTT space */
365 int pin_count;
366
367 /** Breadcrumb of last rendering to the buffer. */
368 uint32_t last_rendering_seqno;
369
370 /** Current tiling mode for the object. */
371 uint32_t tiling_mode;
372
373 /**
374 * Flagging of which individual pages are valid in GEM_DOMAIN_CPU when
375 * GEM_DOMAIN_CPU is not in the object's read domain.
376 */
377 uint8_t *page_cpu_valid;
378};
379
380/**
381 * Request queue structure.
382 *
383 * The request queue allows us to note sequence numbers that have been emitted
384 * and may be associated with active buffers to be retired.
385 *
386 * By keeping this list, we can avoid having to do questionable
387 * sequence-number comparisons on buffer last_rendering_seqnos, and associate
388 * an emission time with seqnos for tracking how far ahead of the GPU we are.
389 */
390struct drm_i915_gem_request {
391 /** GEM sequence number associated with this request. */
392 uint32_t seqno;
393
394 /** Time at which this request was emitted, in jiffies. */
395 unsigned long emitted_jiffies;
396
397 /** Cache domains that were flushed at the start of the request. */
398 uint32_t flush_domains;
399
400 struct list_head list;
401};
402
403struct drm_i915_file_private {
404 struct {
405 uint32_t last_gem_seqno;
406 uint32_t last_gem_throttle_seqno;
407 } mm;
408};
409
235extern struct drm_ioctl_desc i915_ioctls[]; 410extern struct drm_ioctl_desc i915_ioctls[];
236extern int i915_max_ioctl; 411extern int i915_max_ioctl;
237 412
@@ -239,18 +414,26 @@ extern int i915_max_ioctl;
239extern void i915_kernel_lost_context(struct drm_device * dev); 414extern void i915_kernel_lost_context(struct drm_device * dev);
240extern int i915_driver_load(struct drm_device *, unsigned long flags); 415extern int i915_driver_load(struct drm_device *, unsigned long flags);
241extern int i915_driver_unload(struct drm_device *); 416extern int i915_driver_unload(struct drm_device *);
417extern int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv);
242extern void i915_driver_lastclose(struct drm_device * dev); 418extern void i915_driver_lastclose(struct drm_device * dev);
243extern void i915_driver_preclose(struct drm_device *dev, 419extern void i915_driver_preclose(struct drm_device *dev,
244 struct drm_file *file_priv); 420 struct drm_file *file_priv);
421extern void i915_driver_postclose(struct drm_device *dev,
422 struct drm_file *file_priv);
245extern int i915_driver_device_is_agp(struct drm_device * dev); 423extern int i915_driver_device_is_agp(struct drm_device * dev);
246extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, 424extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
247 unsigned long arg); 425 unsigned long arg);
426extern int i915_emit_box(struct drm_device *dev,
427 struct drm_clip_rect __user *boxes,
428 int i, int DR1, int DR4);
248 429
249/* i915_irq.c */ 430/* i915_irq.c */
250extern int i915_irq_emit(struct drm_device *dev, void *data, 431extern int i915_irq_emit(struct drm_device *dev, void *data,
251 struct drm_file *file_priv); 432 struct drm_file *file_priv);
252extern int i915_irq_wait(struct drm_device *dev, void *data, 433extern int i915_irq_wait(struct drm_device *dev, void *data,
253 struct drm_file *file_priv); 434 struct drm_file *file_priv);
435void i915_user_irq_get(struct drm_device *dev);
436void i915_user_irq_put(struct drm_device *dev);
254 437
255extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS); 438extern irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS);
256extern void i915_driver_irq_preinstall(struct drm_device * dev); 439extern void i915_driver_irq_preinstall(struct drm_device * dev);
@@ -279,6 +462,67 @@ extern int i915_mem_destroy_heap(struct drm_device *dev, void *data,
279extern void i915_mem_takedown(struct mem_block **heap); 462extern void i915_mem_takedown(struct mem_block **heap);
280extern void i915_mem_release(struct drm_device * dev, 463extern void i915_mem_release(struct drm_device * dev,
281 struct drm_file *file_priv, struct mem_block *heap); 464 struct drm_file *file_priv, struct mem_block *heap);
465/* i915_gem.c */
466int i915_gem_init_ioctl(struct drm_device *dev, void *data,
467 struct drm_file *file_priv);
468int i915_gem_create_ioctl(struct drm_device *dev, void *data,
469 struct drm_file *file_priv);
470int i915_gem_pread_ioctl(struct drm_device *dev, void *data,
471 struct drm_file *file_priv);
472int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
473 struct drm_file *file_priv);
474int i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
475 struct drm_file *file_priv);
476int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
477 struct drm_file *file_priv);
478int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
479 struct drm_file *file_priv);
480int i915_gem_execbuffer(struct drm_device *dev, void *data,
481 struct drm_file *file_priv);
482int i915_gem_pin_ioctl(struct drm_device *dev, void *data,
483 struct drm_file *file_priv);
484int i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
485 struct drm_file *file_priv);
486int i915_gem_busy_ioctl(struct drm_device *dev, void *data,
487 struct drm_file *file_priv);
488int i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
489 struct drm_file *file_priv);
490int i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
491 struct drm_file *file_priv);
492int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
493 struct drm_file *file_priv);
494int i915_gem_set_tiling(struct drm_device *dev, void *data,
495 struct drm_file *file_priv);
496int i915_gem_get_tiling(struct drm_device *dev, void *data,
497 struct drm_file *file_priv);
498void i915_gem_load(struct drm_device *dev);
499int i915_gem_proc_init(struct drm_minor *minor);
500void i915_gem_proc_cleanup(struct drm_minor *minor);
501int i915_gem_init_object(struct drm_gem_object *obj);
502void i915_gem_free_object(struct drm_gem_object *obj);
503int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment);
504void i915_gem_object_unpin(struct drm_gem_object *obj);
505void i915_gem_lastclose(struct drm_device *dev);
506uint32_t i915_get_gem_seqno(struct drm_device *dev);
507void i915_gem_retire_requests(struct drm_device *dev);
508void i915_gem_retire_work_handler(struct work_struct *work);
509void i915_gem_clflush_object(struct drm_gem_object *obj);
510
511/* i915_gem_tiling.c */
512void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
513
514/* i915_gem_debug.c */
515void i915_gem_dump_object(struct drm_gem_object *obj, int len,
516 const char *where, uint32_t mark);
517#if WATCH_INACTIVE
518void i915_verify_inactive(struct drm_device *dev, char *file, int line);
519#else
520#define i915_verify_inactive(dev, file, line)
521#endif
522void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle);
523void i915_gem_dump_object(struct drm_gem_object *obj, int len,
524 const char *where, uint32_t mark);
525void i915_dump_lru(struct drm_device *dev, const char *where);
282 526
283/* i915_suspend.c */ 527/* i915_suspend.c */
284extern int i915_save_state(struct drm_device *dev); 528extern int i915_save_state(struct drm_device *dev);
@@ -347,6 +591,7 @@ extern void opregion_enable_asle(struct drm_device *dev);
347 */ 591 */
348#define READ_HWSP(dev_priv, reg) (((volatile u32*)(dev_priv->hw_status_page))[reg]) 592#define READ_HWSP(dev_priv, reg) (((volatile u32*)(dev_priv->hw_status_page))[reg])
349#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, 5) 593#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, 5)
594#define I915_GEM_HWS_INDEX 0x10
350 595
351extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller); 596extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
352 597
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
new file mode 100644
index 000000000000..90ae8a0369f7
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -0,0 +1,2497 @@
1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include <linux/swap.h>
33
34static int
35i915_gem_object_set_domain(struct drm_gem_object *obj,
36 uint32_t read_domains,
37 uint32_t write_domain);
38static int
39i915_gem_object_set_domain_range(struct drm_gem_object *obj,
40 uint64_t offset,
41 uint64_t size,
42 uint32_t read_domains,
43 uint32_t write_domain);
44static int
45i915_gem_set_domain(struct drm_gem_object *obj,
46 struct drm_file *file_priv,
47 uint32_t read_domains,
48 uint32_t write_domain);
49static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
50static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
52
53int
54i915_gem_init_ioctl(struct drm_device *dev, void *data,
55 struct drm_file *file_priv)
56{
57 drm_i915_private_t *dev_priv = dev->dev_private;
58 struct drm_i915_gem_init *args = data;
59
60 mutex_lock(&dev->struct_mutex);
61
62 if (args->gtt_start >= args->gtt_end ||
63 (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
64 (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
65 mutex_unlock(&dev->struct_mutex);
66 return -EINVAL;
67 }
68
69 drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
70 args->gtt_end - args->gtt_start);
71
72 dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
73
74 mutex_unlock(&dev->struct_mutex);
75
76 return 0;
77}
78
79
80/**
81 * Creates a new mm object and returns a handle to it.
82 */
83int
84i915_gem_create_ioctl(struct drm_device *dev, void *data,
85 struct drm_file *file_priv)
86{
87 struct drm_i915_gem_create *args = data;
88 struct drm_gem_object *obj;
89 int handle, ret;
90
91 args->size = roundup(args->size, PAGE_SIZE);
92
93 /* Allocate the new object */
94 obj = drm_gem_object_alloc(dev, args->size);
95 if (obj == NULL)
96 return -ENOMEM;
97
98 ret = drm_gem_handle_create(file_priv, obj, &handle);
99 mutex_lock(&dev->struct_mutex);
100 drm_gem_object_handle_unreference(obj);
101 mutex_unlock(&dev->struct_mutex);
102
103 if (ret)
104 return ret;
105
106 args->handle = handle;
107
108 return 0;
109}
110
111/**
112 * Reads data from the object referenced by handle.
113 *
114 * On error, the contents of *data are undefined.
115 */
116int
117i915_gem_pread_ioctl(struct drm_device *dev, void *data,
118 struct drm_file *file_priv)
119{
120 struct drm_i915_gem_pread *args = data;
121 struct drm_gem_object *obj;
122 struct drm_i915_gem_object *obj_priv;
123 ssize_t read;
124 loff_t offset;
125 int ret;
126
127 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
128 if (obj == NULL)
129 return -EBADF;
130 obj_priv = obj->driver_private;
131
132 /* Bounds check source.
133 *
134 * XXX: This could use review for overflow issues...
135 */
136 if (args->offset > obj->size || args->size > obj->size ||
137 args->offset + args->size > obj->size) {
138 drm_gem_object_unreference(obj);
139 return -EINVAL;
140 }
141
142 mutex_lock(&dev->struct_mutex);
143
144 ret = i915_gem_object_set_domain_range(obj, args->offset, args->size,
145 I915_GEM_DOMAIN_CPU, 0);
146 if (ret != 0) {
147 drm_gem_object_unreference(obj);
148 mutex_unlock(&dev->struct_mutex);
149 }
150
151 offset = args->offset;
152
153 read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
154 args->size, &offset);
155 if (read != args->size) {
156 drm_gem_object_unreference(obj);
157 mutex_unlock(&dev->struct_mutex);
158 if (read < 0)
159 return read;
160 else
161 return -EINVAL;
162 }
163
164 drm_gem_object_unreference(obj);
165 mutex_unlock(&dev->struct_mutex);
166
167 return 0;
168}
169
170static int
171i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
172 struct drm_i915_gem_pwrite *args,
173 struct drm_file *file_priv)
174{
175 struct drm_i915_gem_object *obj_priv = obj->driver_private;
176 ssize_t remain;
177 loff_t offset;
178 char __user *user_data;
179 char *vaddr;
180 int i, o, l;
181 int ret = 0;
182 unsigned long pfn;
183 unsigned long unwritten;
184
185 user_data = (char __user *) (uintptr_t) args->data_ptr;
186 remain = args->size;
187 if (!access_ok(VERIFY_READ, user_data, remain))
188 return -EFAULT;
189
190
191 mutex_lock(&dev->struct_mutex);
192 ret = i915_gem_object_pin(obj, 0);
193 if (ret) {
194 mutex_unlock(&dev->struct_mutex);
195 return ret;
196 }
197 ret = i915_gem_set_domain(obj, file_priv,
198 I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
199 if (ret)
200 goto fail;
201
202 obj_priv = obj->driver_private;
203 offset = obj_priv->gtt_offset + args->offset;
204 obj_priv->dirty = 1;
205
206 while (remain > 0) {
207 /* Operation in this page
208 *
209 * i = page number
210 * o = offset within page
211 * l = bytes to copy
212 */
213 i = offset >> PAGE_SHIFT;
214 o = offset & (PAGE_SIZE-1);
215 l = remain;
216 if ((o + l) > PAGE_SIZE)
217 l = PAGE_SIZE - o;
218
219 pfn = (dev->agp->base >> PAGE_SHIFT) + i;
220
221#ifdef CONFIG_HIGHMEM
222 /* kmap_atomic can't map IO pages on non-HIGHMEM kernels
223 */
224 vaddr = kmap_atomic_pfn(pfn, KM_USER0);
225#if WATCH_PWRITE
226 DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n",
227 i, o, l, pfn, vaddr);
228#endif
229 unwritten = __copy_from_user_inatomic_nocache(vaddr + o,
230 user_data, l);
231 kunmap_atomic(vaddr, KM_USER0);
232
233 if (unwritten)
234#endif /* CONFIG_HIGHMEM */
235 {
236 vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
237#if WATCH_PWRITE
238 DRM_INFO("pwrite slow i %d o %d l %d "
239 "pfn %ld vaddr %p\n",
240 i, o, l, pfn, vaddr);
241#endif
242 if (vaddr == NULL) {
243 ret = -EFAULT;
244 goto fail;
245 }
246 unwritten = __copy_from_user(vaddr + o, user_data, l);
247#if WATCH_PWRITE
248 DRM_INFO("unwritten %ld\n", unwritten);
249#endif
250 iounmap(vaddr);
251 if (unwritten) {
252 ret = -EFAULT;
253 goto fail;
254 }
255 }
256
257 remain -= l;
258 user_data += l;
259 offset += l;
260 }
261#if WATCH_PWRITE && 1
262 i915_gem_clflush_object(obj);
263 i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
264 i915_gem_clflush_object(obj);
265#endif
266
267fail:
268 i915_gem_object_unpin(obj);
269 mutex_unlock(&dev->struct_mutex);
270
271 return ret;
272}
273
274int
275i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
276 struct drm_i915_gem_pwrite *args,
277 struct drm_file *file_priv)
278{
279 int ret;
280 loff_t offset;
281 ssize_t written;
282
283 mutex_lock(&dev->struct_mutex);
284
285 ret = i915_gem_set_domain(obj, file_priv,
286 I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
287 if (ret) {
288 mutex_unlock(&dev->struct_mutex);
289 return ret;
290 }
291
292 offset = args->offset;
293
294 written = vfs_write(obj->filp,
295 (char __user *)(uintptr_t) args->data_ptr,
296 args->size, &offset);
297 if (written != args->size) {
298 mutex_unlock(&dev->struct_mutex);
299 if (written < 0)
300 return written;
301 else
302 return -EINVAL;
303 }
304
305 mutex_unlock(&dev->struct_mutex);
306
307 return 0;
308}
309
310/**
311 * Writes data to the object referenced by handle.
312 *
313 * On error, the contents of the buffer that were to be modified are undefined.
314 */
315int
316i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
317 struct drm_file *file_priv)
318{
319 struct drm_i915_gem_pwrite *args = data;
320 struct drm_gem_object *obj;
321 struct drm_i915_gem_object *obj_priv;
322 int ret = 0;
323
324 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
325 if (obj == NULL)
326 return -EBADF;
327 obj_priv = obj->driver_private;
328
329 /* Bounds check destination.
330 *
331 * XXX: This could use review for overflow issues...
332 */
333 if (args->offset > obj->size || args->size > obj->size ||
334 args->offset + args->size > obj->size) {
335 drm_gem_object_unreference(obj);
336 return -EINVAL;
337 }
338
339 /* We can only do the GTT pwrite on untiled buffers, as otherwise
340 * it would end up going through the fenced access, and we'll get
341 * different detiling behavior between reading and writing.
342 * pread/pwrite currently are reading and writing from the CPU
343 * perspective, requiring manual detiling by the client.
344 */
345 if (obj_priv->tiling_mode == I915_TILING_NONE &&
346 dev->gtt_total != 0)
347 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
348 else
349 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
350
351#if WATCH_PWRITE
352 if (ret)
353 DRM_INFO("pwrite failed %d\n", ret);
354#endif
355
356 drm_gem_object_unreference(obj);
357
358 return ret;
359}
360
361/**
362 * Called when user space prepares to use an object
363 */
364int
365i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
366 struct drm_file *file_priv)
367{
368 struct drm_i915_gem_set_domain *args = data;
369 struct drm_gem_object *obj;
370 int ret;
371
372 if (!(dev->driver->driver_features & DRIVER_GEM))
373 return -ENODEV;
374
375 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
376 if (obj == NULL)
377 return -EBADF;
378
379 mutex_lock(&dev->struct_mutex);
380#if WATCH_BUF
381 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
382 obj, obj->size, args->read_domains, args->write_domain);
383#endif
384 ret = i915_gem_set_domain(obj, file_priv,
385 args->read_domains, args->write_domain);
386 drm_gem_object_unreference(obj);
387 mutex_unlock(&dev->struct_mutex);
388 return ret;
389}
390
391/**
392 * Called when user space has done writes to this buffer
393 */
394int
395i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
396 struct drm_file *file_priv)
397{
398 struct drm_i915_gem_sw_finish *args = data;
399 struct drm_gem_object *obj;
400 struct drm_i915_gem_object *obj_priv;
401 int ret = 0;
402
403 if (!(dev->driver->driver_features & DRIVER_GEM))
404 return -ENODEV;
405
406 mutex_lock(&dev->struct_mutex);
407 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
408 if (obj == NULL) {
409 mutex_unlock(&dev->struct_mutex);
410 return -EBADF;
411 }
412
413#if WATCH_BUF
414 DRM_INFO("%s: sw_finish %d (%p %d)\n",
415 __func__, args->handle, obj, obj->size);
416#endif
417 obj_priv = obj->driver_private;
418
419 /* Pinned buffers may be scanout, so flush the cache */
420 if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
421 i915_gem_clflush_object(obj);
422 drm_agp_chipset_flush(dev);
423 }
424 drm_gem_object_unreference(obj);
425 mutex_unlock(&dev->struct_mutex);
426 return ret;
427}
428
429/**
430 * Maps the contents of an object, returning the address it is mapped
431 * into.
432 *
433 * While the mapping holds a reference on the contents of the object, it doesn't
434 * imply a ref on the object itself.
435 */
436int
437i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
438 struct drm_file *file_priv)
439{
440 struct drm_i915_gem_mmap *args = data;
441 struct drm_gem_object *obj;
442 loff_t offset;
443 unsigned long addr;
444
445 if (!(dev->driver->driver_features & DRIVER_GEM))
446 return -ENODEV;
447
448 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
449 if (obj == NULL)
450 return -EBADF;
451
452 offset = args->offset;
453
454 down_write(&current->mm->mmap_sem);
455 addr = do_mmap(obj->filp, 0, args->size,
456 PROT_READ | PROT_WRITE, MAP_SHARED,
457 args->offset);
458 up_write(&current->mm->mmap_sem);
459 mutex_lock(&dev->struct_mutex);
460 drm_gem_object_unreference(obj);
461 mutex_unlock(&dev->struct_mutex);
462 if (IS_ERR((void *)addr))
463 return addr;
464
465 args->addr_ptr = (uint64_t) addr;
466
467 return 0;
468}
469
470static void
471i915_gem_object_free_page_list(struct drm_gem_object *obj)
472{
473 struct drm_i915_gem_object *obj_priv = obj->driver_private;
474 int page_count = obj->size / PAGE_SIZE;
475 int i;
476
477 if (obj_priv->page_list == NULL)
478 return;
479
480
481 for (i = 0; i < page_count; i++)
482 if (obj_priv->page_list[i] != NULL) {
483 if (obj_priv->dirty)
484 set_page_dirty(obj_priv->page_list[i]);
485 mark_page_accessed(obj_priv->page_list[i]);
486 page_cache_release(obj_priv->page_list[i]);
487 }
488 obj_priv->dirty = 0;
489
490 drm_free(obj_priv->page_list,
491 page_count * sizeof(struct page *),
492 DRM_MEM_DRIVER);
493 obj_priv->page_list = NULL;
494}
495
496static void
497i915_gem_object_move_to_active(struct drm_gem_object *obj)
498{
499 struct drm_device *dev = obj->dev;
500 drm_i915_private_t *dev_priv = dev->dev_private;
501 struct drm_i915_gem_object *obj_priv = obj->driver_private;
502
503 /* Add a reference if we're newly entering the active list. */
504 if (!obj_priv->active) {
505 drm_gem_object_reference(obj);
506 obj_priv->active = 1;
507 }
508 /* Move from whatever list we were on to the tail of execution. */
509 list_move_tail(&obj_priv->list,
510 &dev_priv->mm.active_list);
511}
512
513
514static void
515i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
516{
517 struct drm_device *dev = obj->dev;
518 drm_i915_private_t *dev_priv = dev->dev_private;
519 struct drm_i915_gem_object *obj_priv = obj->driver_private;
520
521 i915_verify_inactive(dev, __FILE__, __LINE__);
522 if (obj_priv->pin_count != 0)
523 list_del_init(&obj_priv->list);
524 else
525 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
526
527 if (obj_priv->active) {
528 obj_priv->active = 0;
529 drm_gem_object_unreference(obj);
530 }
531 i915_verify_inactive(dev, __FILE__, __LINE__);
532}
533
534/**
535 * Creates a new sequence number, emitting a write of it to the status page
536 * plus an interrupt, which will trigger i915_user_interrupt_handler.
537 *
538 * Must be called with struct_lock held.
539 *
540 * Returned sequence numbers are nonzero on success.
541 */
542static uint32_t
543i915_add_request(struct drm_device *dev, uint32_t flush_domains)
544{
545 drm_i915_private_t *dev_priv = dev->dev_private;
546 struct drm_i915_gem_request *request;
547 uint32_t seqno;
548 int was_empty;
549 RING_LOCALS;
550
551 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
552 if (request == NULL)
553 return 0;
554
555 /* Grab the seqno we're going to make this request be, and bump the
556 * next (skipping 0 so it can be the reserved no-seqno value).
557 */
558 seqno = dev_priv->mm.next_gem_seqno;
559 dev_priv->mm.next_gem_seqno++;
560 if (dev_priv->mm.next_gem_seqno == 0)
561 dev_priv->mm.next_gem_seqno++;
562
563 BEGIN_LP_RING(4);
564 OUT_RING(MI_STORE_DWORD_INDEX);
565 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
566 OUT_RING(seqno);
567
568 OUT_RING(MI_USER_INTERRUPT);
569 ADVANCE_LP_RING();
570
571 DRM_DEBUG("%d\n", seqno);
572
573 request->seqno = seqno;
574 request->emitted_jiffies = jiffies;
575 request->flush_domains = flush_domains;
576 was_empty = list_empty(&dev_priv->mm.request_list);
577 list_add_tail(&request->list, &dev_priv->mm.request_list);
578
579 if (was_empty)
580 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
581 return seqno;
582}
583
584/**
585 * Command execution barrier
586 *
587 * Ensures that all commands in the ring are finished
588 * before signalling the CPU
589 */
590uint32_t
591i915_retire_commands(struct drm_device *dev)
592{
593 drm_i915_private_t *dev_priv = dev->dev_private;
594 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
595 uint32_t flush_domains = 0;
596 RING_LOCALS;
597
598 /* The sampler always gets flushed on i965 (sigh) */
599 if (IS_I965G(dev))
600 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
601 BEGIN_LP_RING(2);
602 OUT_RING(cmd);
603 OUT_RING(0); /* noop */
604 ADVANCE_LP_RING();
605 return flush_domains;
606}
607
608/**
609 * Moves buffers associated only with the given active seqno from the active
610 * to inactive list, potentially freeing them.
611 */
612static void
613i915_gem_retire_request(struct drm_device *dev,
614 struct drm_i915_gem_request *request)
615{
616 drm_i915_private_t *dev_priv = dev->dev_private;
617
618 /* Move any buffers on the active list that are no longer referenced
619 * by the ringbuffer to the flushing/inactive lists as appropriate.
620 */
621 while (!list_empty(&dev_priv->mm.active_list)) {
622 struct drm_gem_object *obj;
623 struct drm_i915_gem_object *obj_priv;
624
625 obj_priv = list_first_entry(&dev_priv->mm.active_list,
626 struct drm_i915_gem_object,
627 list);
628 obj = obj_priv->obj;
629
630 /* If the seqno being retired doesn't match the oldest in the
631 * list, then the oldest in the list must still be newer than
632 * this seqno.
633 */
634 if (obj_priv->last_rendering_seqno != request->seqno)
635 return;
636#if WATCH_LRU
637 DRM_INFO("%s: retire %d moves to inactive list %p\n",
638 __func__, request->seqno, obj);
639#endif
640
641 if (obj->write_domain != 0) {
642 list_move_tail(&obj_priv->list,
643 &dev_priv->mm.flushing_list);
644 } else {
645 i915_gem_object_move_to_inactive(obj);
646 }
647 }
648
649 if (request->flush_domains != 0) {
650 struct drm_i915_gem_object *obj_priv, *next;
651
652 /* Clear the write domain and activity from any buffers
653 * that are just waiting for a flush matching the one retired.
654 */
655 list_for_each_entry_safe(obj_priv, next,
656 &dev_priv->mm.flushing_list, list) {
657 struct drm_gem_object *obj = obj_priv->obj;
658
659 if (obj->write_domain & request->flush_domains) {
660 obj->write_domain = 0;
661 i915_gem_object_move_to_inactive(obj);
662 }
663 }
664
665 }
666}
667
668/**
669 * Returns true if seq1 is later than seq2.
670 */
671static int
672i915_seqno_passed(uint32_t seq1, uint32_t seq2)
673{
674 return (int32_t)(seq1 - seq2) >= 0;
675}
676
677uint32_t
678i915_get_gem_seqno(struct drm_device *dev)
679{
680 drm_i915_private_t *dev_priv = dev->dev_private;
681
682 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
683}
684
685/**
686 * This function clears the request list as sequence numbers are passed.
687 */
688void
689i915_gem_retire_requests(struct drm_device *dev)
690{
691 drm_i915_private_t *dev_priv = dev->dev_private;
692 uint32_t seqno;
693
694 seqno = i915_get_gem_seqno(dev);
695
696 while (!list_empty(&dev_priv->mm.request_list)) {
697 struct drm_i915_gem_request *request;
698 uint32_t retiring_seqno;
699
700 request = list_first_entry(&dev_priv->mm.request_list,
701 struct drm_i915_gem_request,
702 list);
703 retiring_seqno = request->seqno;
704
705 if (i915_seqno_passed(seqno, retiring_seqno) ||
706 dev_priv->mm.wedged) {
707 i915_gem_retire_request(dev, request);
708
709 list_del(&request->list);
710 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
711 } else
712 break;
713 }
714}
715
716void
717i915_gem_retire_work_handler(struct work_struct *work)
718{
719 drm_i915_private_t *dev_priv;
720 struct drm_device *dev;
721
722 dev_priv = container_of(work, drm_i915_private_t,
723 mm.retire_work.work);
724 dev = dev_priv->dev;
725
726 mutex_lock(&dev->struct_mutex);
727 i915_gem_retire_requests(dev);
728 if (!list_empty(&dev_priv->mm.request_list))
729 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
730 mutex_unlock(&dev->struct_mutex);
731}
732
733/**
734 * Waits for a sequence number to be signaled, and cleans up the
735 * request and object lists appropriately for that event.
736 */
737int
738i915_wait_request(struct drm_device *dev, uint32_t seqno)
739{
740 drm_i915_private_t *dev_priv = dev->dev_private;
741 int ret = 0;
742
743 BUG_ON(seqno == 0);
744
745 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
746 dev_priv->mm.waiting_gem_seqno = seqno;
747 i915_user_irq_get(dev);
748 ret = wait_event_interruptible(dev_priv->irq_queue,
749 i915_seqno_passed(i915_get_gem_seqno(dev),
750 seqno) ||
751 dev_priv->mm.wedged);
752 i915_user_irq_put(dev);
753 dev_priv->mm.waiting_gem_seqno = 0;
754 }
755 if (dev_priv->mm.wedged)
756 ret = -EIO;
757
758 if (ret && ret != -ERESTARTSYS)
759 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
760 __func__, ret, seqno, i915_get_gem_seqno(dev));
761
762 /* Directly dispatch request retiring. While we have the work queue
763 * to handle this, the waiter on a request often wants an associated
764 * buffer to have made it to the inactive list, and we would need
765 * a separate wait queue to handle that.
766 */
767 if (ret == 0)
768 i915_gem_retire_requests(dev);
769
770 return ret;
771}
772
773static void
774i915_gem_flush(struct drm_device *dev,
775 uint32_t invalidate_domains,
776 uint32_t flush_domains)
777{
778 drm_i915_private_t *dev_priv = dev->dev_private;
779 uint32_t cmd;
780 RING_LOCALS;
781
782#if WATCH_EXEC
783 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
784 invalidate_domains, flush_domains);
785#endif
786
787 if (flush_domains & I915_GEM_DOMAIN_CPU)
788 drm_agp_chipset_flush(dev);
789
790 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
791 I915_GEM_DOMAIN_GTT)) {
792 /*
793 * read/write caches:
794 *
795 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
796 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
797 * also flushed at 2d versus 3d pipeline switches.
798 *
799 * read-only caches:
800 *
801 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
802 * MI_READ_FLUSH is set, and is always flushed on 965.
803 *
804 * I915_GEM_DOMAIN_COMMAND may not exist?
805 *
806 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
807 * invalidated when MI_EXE_FLUSH is set.
808 *
809 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
810 * invalidated with every MI_FLUSH.
811 *
812 * TLBs:
813 *
814 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
815 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
816 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
817 * are flushed at any MI_FLUSH.
818 */
819
820 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
821 if ((invalidate_domains|flush_domains) &
822 I915_GEM_DOMAIN_RENDER)
823 cmd &= ~MI_NO_WRITE_FLUSH;
824 if (!IS_I965G(dev)) {
825 /*
826 * On the 965, the sampler cache always gets flushed
827 * and this bit is reserved.
828 */
829 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
830 cmd |= MI_READ_FLUSH;
831 }
832 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
833 cmd |= MI_EXE_FLUSH;
834
835#if WATCH_EXEC
836 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
837#endif
838 BEGIN_LP_RING(2);
839 OUT_RING(cmd);
840 OUT_RING(0); /* noop */
841 ADVANCE_LP_RING();
842 }
843}
844
845/**
846 * Ensures that all rendering to the object has completed and the object is
847 * safe to unbind from the GTT or access from the CPU.
848 */
849static int
850i915_gem_object_wait_rendering(struct drm_gem_object *obj)
851{
852 struct drm_device *dev = obj->dev;
853 struct drm_i915_gem_object *obj_priv = obj->driver_private;
854 int ret;
855
856 /* If there are writes queued to the buffer, flush and
857 * create a new seqno to wait for.
858 */
859 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
860 uint32_t write_domain = obj->write_domain;
861#if WATCH_BUF
862 DRM_INFO("%s: flushing object %p from write domain %08x\n",
863 __func__, obj, write_domain);
864#endif
865 i915_gem_flush(dev, 0, write_domain);
866
867 i915_gem_object_move_to_active(obj);
868 obj_priv->last_rendering_seqno = i915_add_request(dev,
869 write_domain);
870 BUG_ON(obj_priv->last_rendering_seqno == 0);
871#if WATCH_LRU
872 DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
873#endif
874 }
875
876 /* If there is rendering queued on the buffer being evicted, wait for
877 * it.
878 */
879 if (obj_priv->active) {
880#if WATCH_BUF
881 DRM_INFO("%s: object %p wait for seqno %08x\n",
882 __func__, obj, obj_priv->last_rendering_seqno);
883#endif
884 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
885 if (ret != 0)
886 return ret;
887 }
888
889 return 0;
890}
891
892/**
893 * Unbinds an object from the GTT aperture.
894 */
895static int
896i915_gem_object_unbind(struct drm_gem_object *obj)
897{
898 struct drm_device *dev = obj->dev;
899 struct drm_i915_gem_object *obj_priv = obj->driver_private;
900 int ret = 0;
901
902#if WATCH_BUF
903 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
904 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
905#endif
906 if (obj_priv->gtt_space == NULL)
907 return 0;
908
909 if (obj_priv->pin_count != 0) {
910 DRM_ERROR("Attempting to unbind pinned buffer\n");
911 return -EINVAL;
912 }
913
914 /* Wait for any rendering to complete
915 */
916 ret = i915_gem_object_wait_rendering(obj);
917 if (ret) {
918 DRM_ERROR("wait_rendering failed: %d\n", ret);
919 return ret;
920 }
921
922 /* Move the object to the CPU domain to ensure that
923 * any possible CPU writes while it's not in the GTT
924 * are flushed when we go to remap it. This will
925 * also ensure that all pending GPU writes are finished
926 * before we unbind.
927 */
928 ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU,
929 I915_GEM_DOMAIN_CPU);
930 if (ret) {
931 DRM_ERROR("set_domain failed: %d\n", ret);
932 return ret;
933 }
934
935 if (obj_priv->agp_mem != NULL) {
936 drm_unbind_agp(obj_priv->agp_mem);
937 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
938 obj_priv->agp_mem = NULL;
939 }
940
941 BUG_ON(obj_priv->active);
942
943 i915_gem_object_free_page_list(obj);
944
945 if (obj_priv->gtt_space) {
946 atomic_dec(&dev->gtt_count);
947 atomic_sub(obj->size, &dev->gtt_memory);
948
949 drm_mm_put_block(obj_priv->gtt_space);
950 obj_priv->gtt_space = NULL;
951 }
952
953 /* Remove ourselves from the LRU list if present. */
954 if (!list_empty(&obj_priv->list))
955 list_del_init(&obj_priv->list);
956
957 return 0;
958}
959
960static int
961i915_gem_evict_something(struct drm_device *dev)
962{
963 drm_i915_private_t *dev_priv = dev->dev_private;
964 struct drm_gem_object *obj;
965 struct drm_i915_gem_object *obj_priv;
966 int ret = 0;
967
968 for (;;) {
969 /* If there's an inactive buffer available now, grab it
970 * and be done.
971 */
972 if (!list_empty(&dev_priv->mm.inactive_list)) {
973 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
974 struct drm_i915_gem_object,
975 list);
976 obj = obj_priv->obj;
977 BUG_ON(obj_priv->pin_count != 0);
978#if WATCH_LRU
979 DRM_INFO("%s: evicting %p\n", __func__, obj);
980#endif
981 BUG_ON(obj_priv->active);
982
983 /* Wait on the rendering and unbind the buffer. */
984 ret = i915_gem_object_unbind(obj);
985 break;
986 }
987
988 /* If we didn't get anything, but the ring is still processing
989 * things, wait for one of those things to finish and hopefully
990 * leave us a buffer to evict.
991 */
992 if (!list_empty(&dev_priv->mm.request_list)) {
993 struct drm_i915_gem_request *request;
994
995 request = list_first_entry(&dev_priv->mm.request_list,
996 struct drm_i915_gem_request,
997 list);
998
999 ret = i915_wait_request(dev, request->seqno);
1000 if (ret)
1001 break;
1002
1003 /* if waiting caused an object to become inactive,
1004 * then loop around and wait for it. Otherwise, we
1005 * assume that waiting freed and unbound something,
1006 * so there should now be some space in the GTT
1007 */
1008 if (!list_empty(&dev_priv->mm.inactive_list))
1009 continue;
1010 break;
1011 }
1012
1013 /* If we didn't have anything on the request list but there
1014 * are buffers awaiting a flush, emit one and try again.
1015 * When we wait on it, those buffers waiting for that flush
1016 * will get moved to inactive.
1017 */
1018 if (!list_empty(&dev_priv->mm.flushing_list)) {
1019 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1020 struct drm_i915_gem_object,
1021 list);
1022 obj = obj_priv->obj;
1023
1024 i915_gem_flush(dev,
1025 obj->write_domain,
1026 obj->write_domain);
1027 i915_add_request(dev, obj->write_domain);
1028
1029 obj = NULL;
1030 continue;
1031 }
1032
1033 DRM_ERROR("inactive empty %d request empty %d "
1034 "flushing empty %d\n",
1035 list_empty(&dev_priv->mm.inactive_list),
1036 list_empty(&dev_priv->mm.request_list),
1037 list_empty(&dev_priv->mm.flushing_list));
1038 /* If we didn't do any of the above, there's nothing to be done
1039 * and we just can't fit it in.
1040 */
1041 return -ENOMEM;
1042 }
1043 return ret;
1044}
1045
1046static int
1047i915_gem_object_get_page_list(struct drm_gem_object *obj)
1048{
1049 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1050 int page_count, i;
1051 struct address_space *mapping;
1052 struct inode *inode;
1053 struct page *page;
1054 int ret;
1055
1056 if (obj_priv->page_list)
1057 return 0;
1058
1059 /* Get the list of pages out of our struct file. They'll be pinned
1060 * at this point until we release them.
1061 */
1062 page_count = obj->size / PAGE_SIZE;
1063 BUG_ON(obj_priv->page_list != NULL);
1064 obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1065 DRM_MEM_DRIVER);
1066 if (obj_priv->page_list == NULL) {
1067 DRM_ERROR("Faled to allocate page list\n");
1068 return -ENOMEM;
1069 }
1070
1071 inode = obj->filp->f_path.dentry->d_inode;
1072 mapping = inode->i_mapping;
1073 for (i = 0; i < page_count; i++) {
1074 page = read_mapping_page(mapping, i, NULL);
1075 if (IS_ERR(page)) {
1076 ret = PTR_ERR(page);
1077 DRM_ERROR("read_mapping_page failed: %d\n", ret);
1078 i915_gem_object_free_page_list(obj);
1079 return ret;
1080 }
1081 obj_priv->page_list[i] = page;
1082 }
1083 return 0;
1084}
1085
1086/**
1087 * Finds free space in the GTT aperture and binds the object there.
1088 */
1089static int
1090i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1091{
1092 struct drm_device *dev = obj->dev;
1093 drm_i915_private_t *dev_priv = dev->dev_private;
1094 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1095 struct drm_mm_node *free_space;
1096 int page_count, ret;
1097
1098 if (alignment == 0)
1099 alignment = PAGE_SIZE;
1100 if (alignment & (PAGE_SIZE - 1)) {
1101 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1102 return -EINVAL;
1103 }
1104
1105 search_free:
1106 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1107 obj->size, alignment, 0);
1108 if (free_space != NULL) {
1109 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1110 alignment);
1111 if (obj_priv->gtt_space != NULL) {
1112 obj_priv->gtt_space->private = obj;
1113 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1114 }
1115 }
1116 if (obj_priv->gtt_space == NULL) {
1117 /* If the gtt is empty and we're still having trouble
1118 * fitting our object in, we're out of memory.
1119 */
1120#if WATCH_LRU
1121 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1122#endif
1123 if (list_empty(&dev_priv->mm.inactive_list) &&
1124 list_empty(&dev_priv->mm.flushing_list) &&
1125 list_empty(&dev_priv->mm.active_list)) {
1126 DRM_ERROR("GTT full, but LRU list empty\n");
1127 return -ENOMEM;
1128 }
1129
1130 ret = i915_gem_evict_something(dev);
1131 if (ret != 0) {
1132 DRM_ERROR("Failed to evict a buffer %d\n", ret);
1133 return ret;
1134 }
1135 goto search_free;
1136 }
1137
1138#if WATCH_BUF
1139 DRM_INFO("Binding object of size %d at 0x%08x\n",
1140 obj->size, obj_priv->gtt_offset);
1141#endif
1142 ret = i915_gem_object_get_page_list(obj);
1143 if (ret) {
1144 drm_mm_put_block(obj_priv->gtt_space);
1145 obj_priv->gtt_space = NULL;
1146 return ret;
1147 }
1148
1149 page_count = obj->size / PAGE_SIZE;
1150 /* Create an AGP memory structure pointing at our pages, and bind it
1151 * into the GTT.
1152 */
1153 obj_priv->agp_mem = drm_agp_bind_pages(dev,
1154 obj_priv->page_list,
1155 page_count,
1156 obj_priv->gtt_offset);
1157 if (obj_priv->agp_mem == NULL) {
1158 i915_gem_object_free_page_list(obj);
1159 drm_mm_put_block(obj_priv->gtt_space);
1160 obj_priv->gtt_space = NULL;
1161 return -ENOMEM;
1162 }
1163 atomic_inc(&dev->gtt_count);
1164 atomic_add(obj->size, &dev->gtt_memory);
1165
1166 /* Assert that the object is not currently in any GPU domain. As it
1167 * wasn't in the GTT, there shouldn't be any way it could have been in
1168 * a GPU cache
1169 */
1170 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1171 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1172
1173 return 0;
1174}
1175
1176void
1177i915_gem_clflush_object(struct drm_gem_object *obj)
1178{
1179 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1180
1181 /* If we don't have a page list set up, then we're not pinned
1182 * to GPU, and we can ignore the cache flush because it'll happen
1183 * again at bind time.
1184 */
1185 if (obj_priv->page_list == NULL)
1186 return;
1187
1188 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1189}
1190
1191/*
1192 * Set the next domain for the specified object. This
1193 * may not actually perform the necessary flushing/invaliding though,
1194 * as that may want to be batched with other set_domain operations
1195 *
1196 * This is (we hope) the only really tricky part of gem. The goal
1197 * is fairly simple -- track which caches hold bits of the object
1198 * and make sure they remain coherent. A few concrete examples may
1199 * help to explain how it works. For shorthand, we use the notation
1200 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1201 * a pair of read and write domain masks.
1202 *
1203 * Case 1: the batch buffer
1204 *
1205 * 1. Allocated
1206 * 2. Written by CPU
1207 * 3. Mapped to GTT
1208 * 4. Read by GPU
1209 * 5. Unmapped from GTT
1210 * 6. Freed
1211 *
1212 * Let's take these a step at a time
1213 *
1214 * 1. Allocated
1215 * Pages allocated from the kernel may still have
1216 * cache contents, so we set them to (CPU, CPU) always.
1217 * 2. Written by CPU (using pwrite)
1218 * The pwrite function calls set_domain (CPU, CPU) and
1219 * this function does nothing (as nothing changes)
1220 * 3. Mapped by GTT
1221 * This function asserts that the object is not
1222 * currently in any GPU-based read or write domains
1223 * 4. Read by GPU
1224 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
1225 * As write_domain is zero, this function adds in the
1226 * current read domains (CPU+COMMAND, 0).
1227 * flush_domains is set to CPU.
1228 * invalidate_domains is set to COMMAND
1229 * clflush is run to get data out of the CPU caches
1230 * then i915_dev_set_domain calls i915_gem_flush to
1231 * emit an MI_FLUSH and drm_agp_chipset_flush
1232 * 5. Unmapped from GTT
1233 * i915_gem_object_unbind calls set_domain (CPU, CPU)
1234 * flush_domains and invalidate_domains end up both zero
1235 * so no flushing/invalidating happens
1236 * 6. Freed
1237 * yay, done
1238 *
1239 * Case 2: The shared render buffer
1240 *
1241 * 1. Allocated
1242 * 2. Mapped to GTT
1243 * 3. Read/written by GPU
1244 * 4. set_domain to (CPU,CPU)
1245 * 5. Read/written by CPU
1246 * 6. Read/written by GPU
1247 *
1248 * 1. Allocated
1249 * Same as last example, (CPU, CPU)
1250 * 2. Mapped to GTT
1251 * Nothing changes (assertions find that it is not in the GPU)
1252 * 3. Read/written by GPU
1253 * execbuffer calls set_domain (RENDER, RENDER)
1254 * flush_domains gets CPU
1255 * invalidate_domains gets GPU
1256 * clflush (obj)
1257 * MI_FLUSH and drm_agp_chipset_flush
1258 * 4. set_domain (CPU, CPU)
1259 * flush_domains gets GPU
1260 * invalidate_domains gets CPU
1261 * wait_rendering (obj) to make sure all drawing is complete.
1262 * This will include an MI_FLUSH to get the data from GPU
1263 * to memory
1264 * clflush (obj) to invalidate the CPU cache
1265 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1266 * 5. Read/written by CPU
1267 * cache lines are loaded and dirtied
1268 * 6. Read written by GPU
1269 * Same as last GPU access
1270 *
1271 * Case 3: The constant buffer
1272 *
1273 * 1. Allocated
1274 * 2. Written by CPU
1275 * 3. Read by GPU
1276 * 4. Updated (written) by CPU again
1277 * 5. Read by GPU
1278 *
1279 * 1. Allocated
1280 * (CPU, CPU)
1281 * 2. Written by CPU
1282 * (CPU, CPU)
1283 * 3. Read by GPU
1284 * (CPU+RENDER, 0)
1285 * flush_domains = CPU
1286 * invalidate_domains = RENDER
1287 * clflush (obj)
1288 * MI_FLUSH
1289 * drm_agp_chipset_flush
1290 * 4. Updated (written) by CPU again
1291 * (CPU, CPU)
1292 * flush_domains = 0 (no previous write domain)
1293 * invalidate_domains = 0 (no new read domains)
1294 * 5. Read by GPU
1295 * (CPU+RENDER, 0)
1296 * flush_domains = CPU
1297 * invalidate_domains = RENDER
1298 * clflush (obj)
1299 * MI_FLUSH
1300 * drm_agp_chipset_flush
1301 */
1302static int
1303i915_gem_object_set_domain(struct drm_gem_object *obj,
1304 uint32_t read_domains,
1305 uint32_t write_domain)
1306{
1307 struct drm_device *dev = obj->dev;
1308 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1309 uint32_t invalidate_domains = 0;
1310 uint32_t flush_domains = 0;
1311 int ret;
1312
1313#if WATCH_BUF
1314 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1315 __func__, obj,
1316 obj->read_domains, read_domains,
1317 obj->write_domain, write_domain);
1318#endif
1319 /*
1320 * If the object isn't moving to a new write domain,
1321 * let the object stay in multiple read domains
1322 */
1323 if (write_domain == 0)
1324 read_domains |= obj->read_domains;
1325 else
1326 obj_priv->dirty = 1;
1327
1328 /*
1329 * Flush the current write domain if
1330 * the new read domains don't match. Invalidate
1331 * any read domains which differ from the old
1332 * write domain
1333 */
1334 if (obj->write_domain && obj->write_domain != read_domains) {
1335 flush_domains |= obj->write_domain;
1336 invalidate_domains |= read_domains & ~obj->write_domain;
1337 }
1338 /*
1339 * Invalidate any read caches which may have
1340 * stale data. That is, any new read domains.
1341 */
1342 invalidate_domains |= read_domains & ~obj->read_domains;
1343 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1344#if WATCH_BUF
1345 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1346 __func__, flush_domains, invalidate_domains);
1347#endif
1348 /*
1349 * If we're invaliding the CPU cache and flushing a GPU cache,
1350 * then pause for rendering so that the GPU caches will be
1351 * flushed before the cpu cache is invalidated
1352 */
1353 if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
1354 (flush_domains & ~(I915_GEM_DOMAIN_CPU |
1355 I915_GEM_DOMAIN_GTT))) {
1356 ret = i915_gem_object_wait_rendering(obj);
1357 if (ret)
1358 return ret;
1359 }
1360 i915_gem_clflush_object(obj);
1361 }
1362
1363 if ((write_domain | flush_domains) != 0)
1364 obj->write_domain = write_domain;
1365
1366 /* If we're invalidating the CPU domain, clear the per-page CPU
1367 * domain list as well.
1368 */
1369 if (obj_priv->page_cpu_valid != NULL &&
1370 (write_domain != 0 ||
1371 read_domains & I915_GEM_DOMAIN_CPU)) {
1372 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1373 DRM_MEM_DRIVER);
1374 obj_priv->page_cpu_valid = NULL;
1375 }
1376 obj->read_domains = read_domains;
1377
1378 dev->invalidate_domains |= invalidate_domains;
1379 dev->flush_domains |= flush_domains;
1380#if WATCH_BUF
1381 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1382 __func__,
1383 obj->read_domains, obj->write_domain,
1384 dev->invalidate_domains, dev->flush_domains);
1385#endif
1386 return 0;
1387}
1388
1389/**
1390 * Set the read/write domain on a range of the object.
1391 *
1392 * Currently only implemented for CPU reads, otherwise drops to normal
1393 * i915_gem_object_set_domain().
1394 */
1395static int
1396i915_gem_object_set_domain_range(struct drm_gem_object *obj,
1397 uint64_t offset,
1398 uint64_t size,
1399 uint32_t read_domains,
1400 uint32_t write_domain)
1401{
1402 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1403 int ret, i;
1404
1405 if (obj->read_domains & I915_GEM_DOMAIN_CPU)
1406 return 0;
1407
1408 if (read_domains != I915_GEM_DOMAIN_CPU ||
1409 write_domain != 0)
1410 return i915_gem_object_set_domain(obj,
1411 read_domains, write_domain);
1412
1413 /* Wait on any GPU rendering to the object to be flushed. */
1414 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) {
1415 ret = i915_gem_object_wait_rendering(obj);
1416 if (ret)
1417 return ret;
1418 }
1419
1420 if (obj_priv->page_cpu_valid == NULL) {
1421 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1422 DRM_MEM_DRIVER);
1423 }
1424
1425 /* Flush the cache on any pages that are still invalid from the CPU's
1426 * perspective.
1427 */
1428 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) {
1429 if (obj_priv->page_cpu_valid[i])
1430 continue;
1431
1432 drm_clflush_pages(obj_priv->page_list + i, 1);
1433
1434 obj_priv->page_cpu_valid[i] = 1;
1435 }
1436
1437 return 0;
1438}
1439
1440/**
1441 * Once all of the objects have been set in the proper domain,
1442 * perform the necessary flush and invalidate operations.
1443 *
1444 * Returns the write domains flushed, for use in flush tracking.
1445 */
1446static uint32_t
1447i915_gem_dev_set_domain(struct drm_device *dev)
1448{
1449 uint32_t flush_domains = dev->flush_domains;
1450
1451 /*
1452 * Now that all the buffers are synced to the proper domains,
1453 * flush and invalidate the collected domains
1454 */
1455 if (dev->invalidate_domains | dev->flush_domains) {
1456#if WATCH_EXEC
1457 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
1458 __func__,
1459 dev->invalidate_domains,
1460 dev->flush_domains);
1461#endif
1462 i915_gem_flush(dev,
1463 dev->invalidate_domains,
1464 dev->flush_domains);
1465 dev->invalidate_domains = 0;
1466 dev->flush_domains = 0;
1467 }
1468
1469 return flush_domains;
1470}
1471
1472/**
1473 * Pin an object to the GTT and evaluate the relocations landing in it.
1474 */
1475static int
1476i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1477 struct drm_file *file_priv,
1478 struct drm_i915_gem_exec_object *entry)
1479{
1480 struct drm_device *dev = obj->dev;
1481 struct drm_i915_gem_relocation_entry reloc;
1482 struct drm_i915_gem_relocation_entry __user *relocs;
1483 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1484 int i, ret;
1485 uint32_t last_reloc_offset = -1;
1486 void *reloc_page = NULL;
1487
1488 /* Choose the GTT offset for our buffer and put it there. */
1489 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1490 if (ret)
1491 return ret;
1492
1493 entry->offset = obj_priv->gtt_offset;
1494
1495 relocs = (struct drm_i915_gem_relocation_entry __user *)
1496 (uintptr_t) entry->relocs_ptr;
1497 /* Apply the relocations, using the GTT aperture to avoid cache
1498 * flushing requirements.
1499 */
1500 for (i = 0; i < entry->relocation_count; i++) {
1501 struct drm_gem_object *target_obj;
1502 struct drm_i915_gem_object *target_obj_priv;
1503 uint32_t reloc_val, reloc_offset, *reloc_entry;
1504 int ret;
1505
1506 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1507 if (ret != 0) {
1508 i915_gem_object_unpin(obj);
1509 return ret;
1510 }
1511
1512 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1513 reloc.target_handle);
1514 if (target_obj == NULL) {
1515 i915_gem_object_unpin(obj);
1516 return -EBADF;
1517 }
1518 target_obj_priv = target_obj->driver_private;
1519
1520 /* The target buffer should have appeared before us in the
1521 * exec_object list, so it should have a GTT space bound by now.
1522 */
1523 if (target_obj_priv->gtt_space == NULL) {
1524 DRM_ERROR("No GTT space found for object %d\n",
1525 reloc.target_handle);
1526 drm_gem_object_unreference(target_obj);
1527 i915_gem_object_unpin(obj);
1528 return -EINVAL;
1529 }
1530
1531 if (reloc.offset > obj->size - 4) {
1532 DRM_ERROR("Relocation beyond object bounds: "
1533 "obj %p target %d offset %d size %d.\n",
1534 obj, reloc.target_handle,
1535 (int) reloc.offset, (int) obj->size);
1536 drm_gem_object_unreference(target_obj);
1537 i915_gem_object_unpin(obj);
1538 return -EINVAL;
1539 }
1540 if (reloc.offset & 3) {
1541 DRM_ERROR("Relocation not 4-byte aligned: "
1542 "obj %p target %d offset %d.\n",
1543 obj, reloc.target_handle,
1544 (int) reloc.offset);
1545 drm_gem_object_unreference(target_obj);
1546 i915_gem_object_unpin(obj);
1547 return -EINVAL;
1548 }
1549
1550 if (reloc.write_domain && target_obj->pending_write_domain &&
1551 reloc.write_domain != target_obj->pending_write_domain) {
1552 DRM_ERROR("Write domain conflict: "
1553 "obj %p target %d offset %d "
1554 "new %08x old %08x\n",
1555 obj, reloc.target_handle,
1556 (int) reloc.offset,
1557 reloc.write_domain,
1558 target_obj->pending_write_domain);
1559 drm_gem_object_unreference(target_obj);
1560 i915_gem_object_unpin(obj);
1561 return -EINVAL;
1562 }
1563
1564#if WATCH_RELOC
1565 DRM_INFO("%s: obj %p offset %08x target %d "
1566 "read %08x write %08x gtt %08x "
1567 "presumed %08x delta %08x\n",
1568 __func__,
1569 obj,
1570 (int) reloc.offset,
1571 (int) reloc.target_handle,
1572 (int) reloc.read_domains,
1573 (int) reloc.write_domain,
1574 (int) target_obj_priv->gtt_offset,
1575 (int) reloc.presumed_offset,
1576 reloc.delta);
1577#endif
1578
1579 target_obj->pending_read_domains |= reloc.read_domains;
1580 target_obj->pending_write_domain |= reloc.write_domain;
1581
1582 /* If the relocation already has the right value in it, no
1583 * more work needs to be done.
1584 */
1585 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1586 drm_gem_object_unreference(target_obj);
1587 continue;
1588 }
1589
1590 /* Now that we're going to actually write some data in,
1591 * make sure that any rendering using this buffer's contents
1592 * is completed.
1593 */
1594 i915_gem_object_wait_rendering(obj);
1595
1596 /* As we're writing through the gtt, flush
1597 * any CPU writes before we write the relocations
1598 */
1599 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
1600 i915_gem_clflush_object(obj);
1601 drm_agp_chipset_flush(dev);
1602 obj->write_domain = 0;
1603 }
1604
1605 /* Map the page containing the relocation we're going to
1606 * perform.
1607 */
1608 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1609 if (reloc_page == NULL ||
1610 (last_reloc_offset & ~(PAGE_SIZE - 1)) !=
1611 (reloc_offset & ~(PAGE_SIZE - 1))) {
1612 if (reloc_page != NULL)
1613 iounmap(reloc_page);
1614
1615 reloc_page = ioremap(dev->agp->base +
1616 (reloc_offset & ~(PAGE_SIZE - 1)),
1617 PAGE_SIZE);
1618 last_reloc_offset = reloc_offset;
1619 if (reloc_page == NULL) {
1620 drm_gem_object_unreference(target_obj);
1621 i915_gem_object_unpin(obj);
1622 return -ENOMEM;
1623 }
1624 }
1625
1626 reloc_entry = (uint32_t *)((char *)reloc_page +
1627 (reloc_offset & (PAGE_SIZE - 1)));
1628 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1629
1630#if WATCH_BUF
1631 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1632 obj, (unsigned int) reloc.offset,
1633 readl(reloc_entry), reloc_val);
1634#endif
1635 writel(reloc_val, reloc_entry);
1636
1637 /* Write the updated presumed offset for this entry back out
1638 * to the user.
1639 */
1640 reloc.presumed_offset = target_obj_priv->gtt_offset;
1641 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1642 if (ret != 0) {
1643 drm_gem_object_unreference(target_obj);
1644 i915_gem_object_unpin(obj);
1645 return ret;
1646 }
1647
1648 drm_gem_object_unreference(target_obj);
1649 }
1650
1651 if (reloc_page != NULL)
1652 iounmap(reloc_page);
1653
1654#if WATCH_BUF
1655 if (0)
1656 i915_gem_dump_object(obj, 128, __func__, ~0);
1657#endif
1658 return 0;
1659}
1660
1661/** Dispatch a batchbuffer to the ring
1662 */
1663static int
1664i915_dispatch_gem_execbuffer(struct drm_device *dev,
1665 struct drm_i915_gem_execbuffer *exec,
1666 uint64_t exec_offset)
1667{
1668 drm_i915_private_t *dev_priv = dev->dev_private;
1669 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1670 (uintptr_t) exec->cliprects_ptr;
1671 int nbox = exec->num_cliprects;
1672 int i = 0, count;
1673 uint32_t exec_start, exec_len;
1674 RING_LOCALS;
1675
1676 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1677 exec_len = (uint32_t) exec->batch_len;
1678
1679 if ((exec_start | exec_len) & 0x7) {
1680 DRM_ERROR("alignment\n");
1681 return -EINVAL;
1682 }
1683
1684 if (!exec_start)
1685 return -EINVAL;
1686
1687 count = nbox ? nbox : 1;
1688
1689 for (i = 0; i < count; i++) {
1690 if (i < nbox) {
1691 int ret = i915_emit_box(dev, boxes, i,
1692 exec->DR1, exec->DR4);
1693 if (ret)
1694 return ret;
1695 }
1696
1697 if (IS_I830(dev) || IS_845G(dev)) {
1698 BEGIN_LP_RING(4);
1699 OUT_RING(MI_BATCH_BUFFER);
1700 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1701 OUT_RING(exec_start + exec_len - 4);
1702 OUT_RING(0);
1703 ADVANCE_LP_RING();
1704 } else {
1705 BEGIN_LP_RING(2);
1706 if (IS_I965G(dev)) {
1707 OUT_RING(MI_BATCH_BUFFER_START |
1708 (2 << 6) |
1709 MI_BATCH_NON_SECURE_I965);
1710 OUT_RING(exec_start);
1711 } else {
1712 OUT_RING(MI_BATCH_BUFFER_START |
1713 (2 << 6));
1714 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1715 }
1716 ADVANCE_LP_RING();
1717 }
1718 }
1719
1720 /* XXX breadcrumb */
1721 return 0;
1722}
1723
1724/* Throttle our rendering by waiting until the ring has completed our requests
1725 * emitted over 20 msec ago.
1726 *
1727 * This should get us reasonable parallelism between CPU and GPU but also
1728 * relatively low latency when blocking on a particular request to finish.
1729 */
1730static int
1731i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1732{
1733 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1734 int ret = 0;
1735 uint32_t seqno;
1736
1737 mutex_lock(&dev->struct_mutex);
1738 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1739 i915_file_priv->mm.last_gem_throttle_seqno =
1740 i915_file_priv->mm.last_gem_seqno;
1741 if (seqno)
1742 ret = i915_wait_request(dev, seqno);
1743 mutex_unlock(&dev->struct_mutex);
1744 return ret;
1745}
1746
1747int
1748i915_gem_execbuffer(struct drm_device *dev, void *data,
1749 struct drm_file *file_priv)
1750{
1751 drm_i915_private_t *dev_priv = dev->dev_private;
1752 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1753 struct drm_i915_gem_execbuffer *args = data;
1754 struct drm_i915_gem_exec_object *exec_list = NULL;
1755 struct drm_gem_object **object_list = NULL;
1756 struct drm_gem_object *batch_obj;
1757 int ret, i, pinned = 0;
1758 uint64_t exec_offset;
1759 uint32_t seqno, flush_domains;
1760
1761#if WATCH_EXEC
1762 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1763 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1764#endif
1765
1766 /* Copy in the exec list from userland */
1767 exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1768 DRM_MEM_DRIVER);
1769 object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1770 DRM_MEM_DRIVER);
1771 if (exec_list == NULL || object_list == NULL) {
1772 DRM_ERROR("Failed to allocate exec or object list "
1773 "for %d buffers\n",
1774 args->buffer_count);
1775 ret = -ENOMEM;
1776 goto pre_mutex_err;
1777 }
1778 ret = copy_from_user(exec_list,
1779 (struct drm_i915_relocation_entry __user *)
1780 (uintptr_t) args->buffers_ptr,
1781 sizeof(*exec_list) * args->buffer_count);
1782 if (ret != 0) {
1783 DRM_ERROR("copy %d exec entries failed %d\n",
1784 args->buffer_count, ret);
1785 goto pre_mutex_err;
1786 }
1787
1788 mutex_lock(&dev->struct_mutex);
1789
1790 i915_verify_inactive(dev, __FILE__, __LINE__);
1791
1792 if (dev_priv->mm.wedged) {
1793 DRM_ERROR("Execbuf while wedged\n");
1794 mutex_unlock(&dev->struct_mutex);
1795 return -EIO;
1796 }
1797
1798 if (dev_priv->mm.suspended) {
1799 DRM_ERROR("Execbuf while VT-switched.\n");
1800 mutex_unlock(&dev->struct_mutex);
1801 return -EBUSY;
1802 }
1803
1804 /* Zero the gloabl flush/invalidate flags. These
1805 * will be modified as each object is bound to the
1806 * gtt
1807 */
1808 dev->invalidate_domains = 0;
1809 dev->flush_domains = 0;
1810
1811 /* Look up object handles and perform the relocations */
1812 for (i = 0; i < args->buffer_count; i++) {
1813 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1814 exec_list[i].handle);
1815 if (object_list[i] == NULL) {
1816 DRM_ERROR("Invalid object handle %d at index %d\n",
1817 exec_list[i].handle, i);
1818 ret = -EBADF;
1819 goto err;
1820 }
1821
1822 object_list[i]->pending_read_domains = 0;
1823 object_list[i]->pending_write_domain = 0;
1824 ret = i915_gem_object_pin_and_relocate(object_list[i],
1825 file_priv,
1826 &exec_list[i]);
1827 if (ret) {
1828 DRM_ERROR("object bind and relocate failed %d\n", ret);
1829 goto err;
1830 }
1831 pinned = i + 1;
1832 }
1833
1834 /* Set the pending read domains for the batch buffer to COMMAND */
1835 batch_obj = object_list[args->buffer_count-1];
1836 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1837 batch_obj->pending_write_domain = 0;
1838
1839 i915_verify_inactive(dev, __FILE__, __LINE__);
1840
1841 for (i = 0; i < args->buffer_count; i++) {
1842 struct drm_gem_object *obj = object_list[i];
1843 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1844
1845 if (obj_priv->gtt_space == NULL) {
1846 /* We evicted the buffer in the process of validating
1847 * our set of buffers in. We could try to recover by
1848 * kicking them everything out and trying again from
1849 * the start.
1850 */
1851 ret = -ENOMEM;
1852 goto err;
1853 }
1854
1855 /* make sure all previous memory operations have passed */
1856 ret = i915_gem_object_set_domain(obj,
1857 obj->pending_read_domains,
1858 obj->pending_write_domain);
1859 if (ret)
1860 goto err;
1861 }
1862
1863 i915_verify_inactive(dev, __FILE__, __LINE__);
1864
1865 /* Flush/invalidate caches and chipset buffer */
1866 flush_domains = i915_gem_dev_set_domain(dev);
1867
1868 i915_verify_inactive(dev, __FILE__, __LINE__);
1869
1870#if WATCH_COHERENCY
1871 for (i = 0; i < args->buffer_count; i++) {
1872 i915_gem_object_check_coherency(object_list[i],
1873 exec_list[i].handle);
1874 }
1875#endif
1876
1877 exec_offset = exec_list[args->buffer_count - 1].offset;
1878
1879#if WATCH_EXEC
1880 i915_gem_dump_object(object_list[args->buffer_count - 1],
1881 args->batch_len,
1882 __func__,
1883 ~0);
1884#endif
1885
1886 (void)i915_add_request(dev, flush_domains);
1887
1888 /* Exec the batchbuffer */
1889 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
1890 if (ret) {
1891 DRM_ERROR("dispatch failed %d\n", ret);
1892 goto err;
1893 }
1894
1895 /*
1896 * Ensure that the commands in the batch buffer are
1897 * finished before the interrupt fires
1898 */
1899 flush_domains = i915_retire_commands(dev);
1900
1901 i915_verify_inactive(dev, __FILE__, __LINE__);
1902
1903 /*
1904 * Get a seqno representing the execution of the current buffer,
1905 * which we can wait on. We would like to mitigate these interrupts,
1906 * likely by only creating seqnos occasionally (so that we have
1907 * *some* interrupts representing completion of buffers that we can
1908 * wait on when trying to clear up gtt space).
1909 */
1910 seqno = i915_add_request(dev, flush_domains);
1911 BUG_ON(seqno == 0);
1912 i915_file_priv->mm.last_gem_seqno = seqno;
1913 for (i = 0; i < args->buffer_count; i++) {
1914 struct drm_gem_object *obj = object_list[i];
1915 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1916
1917 i915_gem_object_move_to_active(obj);
1918 obj_priv->last_rendering_seqno = seqno;
1919#if WATCH_LRU
1920 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
1921#endif
1922 }
1923#if WATCH_LRU
1924 i915_dump_lru(dev, __func__);
1925#endif
1926
1927 i915_verify_inactive(dev, __FILE__, __LINE__);
1928
1929 /* Copy the new buffer offsets back to the user's exec list. */
1930 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1931 (uintptr_t) args->buffers_ptr,
1932 exec_list,
1933 sizeof(*exec_list) * args->buffer_count);
1934 if (ret)
1935 DRM_ERROR("failed to copy %d exec entries "
1936 "back to user (%d)\n",
1937 args->buffer_count, ret);
1938err:
1939 if (object_list != NULL) {
1940 for (i = 0; i < pinned; i++)
1941 i915_gem_object_unpin(object_list[i]);
1942
1943 for (i = 0; i < args->buffer_count; i++)
1944 drm_gem_object_unreference(object_list[i]);
1945 }
1946 mutex_unlock(&dev->struct_mutex);
1947
1948pre_mutex_err:
1949 drm_free(object_list, sizeof(*object_list) * args->buffer_count,
1950 DRM_MEM_DRIVER);
1951 drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
1952 DRM_MEM_DRIVER);
1953
1954 return ret;
1955}
1956
1957int
1958i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
1959{
1960 struct drm_device *dev = obj->dev;
1961 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1962 int ret;
1963
1964 i915_verify_inactive(dev, __FILE__, __LINE__);
1965 if (obj_priv->gtt_space == NULL) {
1966 ret = i915_gem_object_bind_to_gtt(obj, alignment);
1967 if (ret != 0) {
1968 DRM_ERROR("Failure to bind: %d", ret);
1969 return ret;
1970 }
1971 }
1972 obj_priv->pin_count++;
1973
1974 /* If the object is not active and not pending a flush,
1975 * remove it from the inactive list
1976 */
1977 if (obj_priv->pin_count == 1) {
1978 atomic_inc(&dev->pin_count);
1979 atomic_add(obj->size, &dev->pin_memory);
1980 if (!obj_priv->active &&
1981 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
1982 I915_GEM_DOMAIN_GTT)) == 0 &&
1983 !list_empty(&obj_priv->list))
1984 list_del_init(&obj_priv->list);
1985 }
1986 i915_verify_inactive(dev, __FILE__, __LINE__);
1987
1988 return 0;
1989}
1990
1991void
1992i915_gem_object_unpin(struct drm_gem_object *obj)
1993{
1994 struct drm_device *dev = obj->dev;
1995 drm_i915_private_t *dev_priv = dev->dev_private;
1996 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1997
1998 i915_verify_inactive(dev, __FILE__, __LINE__);
1999 obj_priv->pin_count--;
2000 BUG_ON(obj_priv->pin_count < 0);
2001 BUG_ON(obj_priv->gtt_space == NULL);
2002
2003 /* If the object is no longer pinned, and is
2004 * neither active nor being flushed, then stick it on
2005 * the inactive list
2006 */
2007 if (obj_priv->pin_count == 0) {
2008 if (!obj_priv->active &&
2009 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2010 I915_GEM_DOMAIN_GTT)) == 0)
2011 list_move_tail(&obj_priv->list,
2012 &dev_priv->mm.inactive_list);
2013 atomic_dec(&dev->pin_count);
2014 atomic_sub(obj->size, &dev->pin_memory);
2015 }
2016 i915_verify_inactive(dev, __FILE__, __LINE__);
2017}
2018
2019int
2020i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2021 struct drm_file *file_priv)
2022{
2023 struct drm_i915_gem_pin *args = data;
2024 struct drm_gem_object *obj;
2025 struct drm_i915_gem_object *obj_priv;
2026 int ret;
2027
2028 mutex_lock(&dev->struct_mutex);
2029
2030 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2031 if (obj == NULL) {
2032 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2033 args->handle);
2034 mutex_unlock(&dev->struct_mutex);
2035 return -EBADF;
2036 }
2037 obj_priv = obj->driver_private;
2038
2039 ret = i915_gem_object_pin(obj, args->alignment);
2040 if (ret != 0) {
2041 drm_gem_object_unreference(obj);
2042 mutex_unlock(&dev->struct_mutex);
2043 return ret;
2044 }
2045
2046 /* XXX - flush the CPU caches for pinned objects
2047 * as the X server doesn't manage domains yet
2048 */
2049 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
2050 i915_gem_clflush_object(obj);
2051 drm_agp_chipset_flush(dev);
2052 obj->write_domain = 0;
2053 }
2054 args->offset = obj_priv->gtt_offset;
2055 drm_gem_object_unreference(obj);
2056 mutex_unlock(&dev->struct_mutex);
2057
2058 return 0;
2059}
2060
2061int
2062i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2063 struct drm_file *file_priv)
2064{
2065 struct drm_i915_gem_pin *args = data;
2066 struct drm_gem_object *obj;
2067
2068 mutex_lock(&dev->struct_mutex);
2069
2070 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2071 if (obj == NULL) {
2072 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2073 args->handle);
2074 mutex_unlock(&dev->struct_mutex);
2075 return -EBADF;
2076 }
2077
2078 i915_gem_object_unpin(obj);
2079
2080 drm_gem_object_unreference(obj);
2081 mutex_unlock(&dev->struct_mutex);
2082 return 0;
2083}
2084
2085int
2086i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2087 struct drm_file *file_priv)
2088{
2089 struct drm_i915_gem_busy *args = data;
2090 struct drm_gem_object *obj;
2091 struct drm_i915_gem_object *obj_priv;
2092
2093 mutex_lock(&dev->struct_mutex);
2094 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2095 if (obj == NULL) {
2096 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2097 args->handle);
2098 mutex_unlock(&dev->struct_mutex);
2099 return -EBADF;
2100 }
2101
2102 obj_priv = obj->driver_private;
2103 args->busy = obj_priv->active;
2104
2105 drm_gem_object_unreference(obj);
2106 mutex_unlock(&dev->struct_mutex);
2107 return 0;
2108}
2109
2110int
2111i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2112 struct drm_file *file_priv)
2113{
2114 return i915_gem_ring_throttle(dev, file_priv);
2115}
2116
2117int i915_gem_init_object(struct drm_gem_object *obj)
2118{
2119 struct drm_i915_gem_object *obj_priv;
2120
2121 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2122 if (obj_priv == NULL)
2123 return -ENOMEM;
2124
2125 /*
2126 * We've just allocated pages from the kernel,
2127 * so they've just been written by the CPU with
2128 * zeros. They'll need to be clflushed before we
2129 * use them with the GPU.
2130 */
2131 obj->write_domain = I915_GEM_DOMAIN_CPU;
2132 obj->read_domains = I915_GEM_DOMAIN_CPU;
2133
2134 obj->driver_private = obj_priv;
2135 obj_priv->obj = obj;
2136 INIT_LIST_HEAD(&obj_priv->list);
2137 return 0;
2138}
2139
2140void i915_gem_free_object(struct drm_gem_object *obj)
2141{
2142 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2143
2144 while (obj_priv->pin_count > 0)
2145 i915_gem_object_unpin(obj);
2146
2147 i915_gem_object_unbind(obj);
2148
2149 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2150 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2151}
2152
2153static int
2154i915_gem_set_domain(struct drm_gem_object *obj,
2155 struct drm_file *file_priv,
2156 uint32_t read_domains,
2157 uint32_t write_domain)
2158{
2159 struct drm_device *dev = obj->dev;
2160 int ret;
2161 uint32_t flush_domains;
2162
2163 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2164
2165 ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
2166 if (ret)
2167 return ret;
2168 flush_domains = i915_gem_dev_set_domain(obj->dev);
2169
2170 if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
2171 (void) i915_add_request(dev, flush_domains);
2172
2173 return 0;
2174}
2175
2176/** Unbinds all objects that are on the given buffer list. */
2177static int
2178i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2179{
2180 struct drm_gem_object *obj;
2181 struct drm_i915_gem_object *obj_priv;
2182 int ret;
2183
2184 while (!list_empty(head)) {
2185 obj_priv = list_first_entry(head,
2186 struct drm_i915_gem_object,
2187 list);
2188 obj = obj_priv->obj;
2189
2190 if (obj_priv->pin_count != 0) {
2191 DRM_ERROR("Pinned object in unbind list\n");
2192 mutex_unlock(&dev->struct_mutex);
2193 return -EINVAL;
2194 }
2195
2196 ret = i915_gem_object_unbind(obj);
2197 if (ret != 0) {
2198 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2199 ret);
2200 mutex_unlock(&dev->struct_mutex);
2201 return ret;
2202 }
2203 }
2204
2205
2206 return 0;
2207}
2208
2209static int
2210i915_gem_idle(struct drm_device *dev)
2211{
2212 drm_i915_private_t *dev_priv = dev->dev_private;
2213 uint32_t seqno, cur_seqno, last_seqno;
2214 int stuck, ret;
2215
2216 if (dev_priv->mm.suspended)
2217 return 0;
2218
2219 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2220 * We need to replace this with a semaphore, or something.
2221 */
2222 dev_priv->mm.suspended = 1;
2223
2224 i915_kernel_lost_context(dev);
2225
2226 /* Flush the GPU along with all non-CPU write domains
2227 */
2228 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2229 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2230 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2231 I915_GEM_DOMAIN_GTT));
2232
2233 if (seqno == 0) {
2234 mutex_unlock(&dev->struct_mutex);
2235 return -ENOMEM;
2236 }
2237
2238 dev_priv->mm.waiting_gem_seqno = seqno;
2239 last_seqno = 0;
2240 stuck = 0;
2241 for (;;) {
2242 cur_seqno = i915_get_gem_seqno(dev);
2243 if (i915_seqno_passed(cur_seqno, seqno))
2244 break;
2245 if (last_seqno == cur_seqno) {
2246 if (stuck++ > 100) {
2247 DRM_ERROR("hardware wedged\n");
2248 dev_priv->mm.wedged = 1;
2249 DRM_WAKEUP(&dev_priv->irq_queue);
2250 break;
2251 }
2252 }
2253 msleep(10);
2254 last_seqno = cur_seqno;
2255 }
2256 dev_priv->mm.waiting_gem_seqno = 0;
2257
2258 i915_gem_retire_requests(dev);
2259
2260 /* Active and flushing should now be empty as we've
2261 * waited for a sequence higher than any pending execbuffer
2262 */
2263 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2264 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2265
2266 /* Request should now be empty as we've also waited
2267 * for the last request in the list
2268 */
2269 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2270
2271 /* Move all buffers out of the GTT. */
2272 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2273 if (ret)
2274 return ret;
2275
2276 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2277 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2278 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2279 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2280 return 0;
2281}
2282
2283static int
2284i915_gem_init_hws(struct drm_device *dev)
2285{
2286 drm_i915_private_t *dev_priv = dev->dev_private;
2287 struct drm_gem_object *obj;
2288 struct drm_i915_gem_object *obj_priv;
2289 int ret;
2290
2291 /* If we need a physical address for the status page, it's already
2292 * initialized at driver load time.
2293 */
2294 if (!I915_NEED_GFX_HWS(dev))
2295 return 0;
2296
2297 obj = drm_gem_object_alloc(dev, 4096);
2298 if (obj == NULL) {
2299 DRM_ERROR("Failed to allocate status page\n");
2300 return -ENOMEM;
2301 }
2302 obj_priv = obj->driver_private;
2303
2304 ret = i915_gem_object_pin(obj, 4096);
2305 if (ret != 0) {
2306 drm_gem_object_unreference(obj);
2307 return ret;
2308 }
2309
2310 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2311 dev_priv->hws_map.offset = dev->agp->base + obj_priv->gtt_offset;
2312 dev_priv->hws_map.size = 4096;
2313 dev_priv->hws_map.type = 0;
2314 dev_priv->hws_map.flags = 0;
2315 dev_priv->hws_map.mtrr = 0;
2316
2317 drm_core_ioremap(&dev_priv->hws_map, dev);
2318 if (dev_priv->hws_map.handle == NULL) {
2319 DRM_ERROR("Failed to map status page.\n");
2320 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2321 drm_gem_object_unreference(obj);
2322 return -EINVAL;
2323 }
2324 dev_priv->hws_obj = obj;
2325 dev_priv->hw_status_page = dev_priv->hws_map.handle;
2326 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2327 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2328 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2329
2330 return 0;
2331}
2332
2333static int
2334i915_gem_init_ringbuffer(struct drm_device *dev)
2335{
2336 drm_i915_private_t *dev_priv = dev->dev_private;
2337 struct drm_gem_object *obj;
2338 struct drm_i915_gem_object *obj_priv;
2339 int ret;
2340
2341 ret = i915_gem_init_hws(dev);
2342 if (ret != 0)
2343 return ret;
2344
2345 obj = drm_gem_object_alloc(dev, 128 * 1024);
2346 if (obj == NULL) {
2347 DRM_ERROR("Failed to allocate ringbuffer\n");
2348 return -ENOMEM;
2349 }
2350 obj_priv = obj->driver_private;
2351
2352 ret = i915_gem_object_pin(obj, 4096);
2353 if (ret != 0) {
2354 drm_gem_object_unreference(obj);
2355 return ret;
2356 }
2357
2358 /* Set up the kernel mapping for the ring. */
2359 dev_priv->ring.Size = obj->size;
2360 dev_priv->ring.tail_mask = obj->size - 1;
2361
2362 dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2363 dev_priv->ring.map.size = obj->size;
2364 dev_priv->ring.map.type = 0;
2365 dev_priv->ring.map.flags = 0;
2366 dev_priv->ring.map.mtrr = 0;
2367
2368 drm_core_ioremap(&dev_priv->ring.map, dev);
2369 if (dev_priv->ring.map.handle == NULL) {
2370 DRM_ERROR("Failed to map ringbuffer.\n");
2371 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2372 drm_gem_object_unreference(obj);
2373 return -EINVAL;
2374 }
2375 dev_priv->ring.ring_obj = obj;
2376 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2377
2378 /* Stop the ring if it's running. */
2379 I915_WRITE(PRB0_CTL, 0);
2380 I915_WRITE(PRB0_HEAD, 0);
2381 I915_WRITE(PRB0_TAIL, 0);
2382 I915_WRITE(PRB0_START, 0);
2383
2384 /* Initialize the ring. */
2385 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2386 I915_WRITE(PRB0_CTL,
2387 ((obj->size - 4096) & RING_NR_PAGES) |
2388 RING_NO_REPORT |
2389 RING_VALID);
2390
2391 /* Update our cache of the ring state */
2392 i915_kernel_lost_context(dev);
2393
2394 return 0;
2395}
2396
2397static void
2398i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2399{
2400 drm_i915_private_t *dev_priv = dev->dev_private;
2401
2402 if (dev_priv->ring.ring_obj == NULL)
2403 return;
2404
2405 drm_core_ioremapfree(&dev_priv->ring.map, dev);
2406
2407 i915_gem_object_unpin(dev_priv->ring.ring_obj);
2408 drm_gem_object_unreference(dev_priv->ring.ring_obj);
2409 dev_priv->ring.ring_obj = NULL;
2410 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2411
2412 if (dev_priv->hws_obj != NULL) {
2413 i915_gem_object_unpin(dev_priv->hws_obj);
2414 drm_gem_object_unreference(dev_priv->hws_obj);
2415 dev_priv->hws_obj = NULL;
2416 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2417
2418 /* Write high address into HWS_PGA when disabling. */
2419 I915_WRITE(HWS_PGA, 0x1ffff000);
2420 }
2421}
2422
2423int
2424i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2425 struct drm_file *file_priv)
2426{
2427 drm_i915_private_t *dev_priv = dev->dev_private;
2428 int ret;
2429
2430 if (dev_priv->mm.wedged) {
2431 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2432 dev_priv->mm.wedged = 0;
2433 }
2434
2435 ret = i915_gem_init_ringbuffer(dev);
2436 if (ret != 0)
2437 return ret;
2438
2439 mutex_lock(&dev->struct_mutex);
2440 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2441 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2442 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2443 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2444 dev_priv->mm.suspended = 0;
2445 mutex_unlock(&dev->struct_mutex);
2446 return 0;
2447}
2448
2449int
2450i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2451 struct drm_file *file_priv)
2452{
2453 int ret;
2454
2455 mutex_lock(&dev->struct_mutex);
2456 ret = i915_gem_idle(dev);
2457 if (ret == 0)
2458 i915_gem_cleanup_ringbuffer(dev);
2459 mutex_unlock(&dev->struct_mutex);
2460
2461 return 0;
2462}
2463
2464void
2465i915_gem_lastclose(struct drm_device *dev)
2466{
2467 int ret;
2468 drm_i915_private_t *dev_priv = dev->dev_private;
2469
2470 mutex_lock(&dev->struct_mutex);
2471
2472 if (dev_priv->ring.ring_obj != NULL) {
2473 ret = i915_gem_idle(dev);
2474 if (ret)
2475 DRM_ERROR("failed to idle hardware: %d\n", ret);
2476
2477 i915_gem_cleanup_ringbuffer(dev);
2478 }
2479
2480 mutex_unlock(&dev->struct_mutex);
2481}
2482
2483void
2484i915_gem_load(struct drm_device *dev)
2485{
2486 drm_i915_private_t *dev_priv = dev->dev_private;
2487
2488 INIT_LIST_HEAD(&dev_priv->mm.active_list);
2489 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2490 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2491 INIT_LIST_HEAD(&dev_priv->mm.request_list);
2492 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2493 i915_gem_retire_work_handler);
2494 dev_priv->mm.next_gem_seqno = 1;
2495
2496 i915_gem_detect_bit_6_swizzle(dev);
2497}
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
new file mode 100644
index 000000000000..131c088f8c8a
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -0,0 +1,201 @@
1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Keith Packard <keithp@keithp.com>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32
33#if WATCH_INACTIVE
34void
35i915_verify_inactive(struct drm_device *dev, char *file, int line)
36{
37 drm_i915_private_t *dev_priv = dev->dev_private;
38 struct drm_gem_object *obj;
39 struct drm_i915_gem_object *obj_priv;
40
41 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
42 obj = obj_priv->obj;
43 if (obj_priv->pin_count || obj_priv->active ||
44 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
45 I915_GEM_DOMAIN_GTT)))
46 DRM_ERROR("inactive %p (p %d a %d w %x) %s:%d\n",
47 obj,
48 obj_priv->pin_count, obj_priv->active,
49 obj->write_domain, file, line);
50 }
51}
52#endif /* WATCH_INACTIVE */
53
54
55#if WATCH_BUF | WATCH_EXEC | WATCH_PWRITE
56static void
57i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end,
58 uint32_t bias, uint32_t mark)
59{
60 uint32_t *mem = kmap_atomic(page, KM_USER0);
61 int i;
62 for (i = start; i < end; i += 4)
63 DRM_INFO("%08x: %08x%s\n",
64 (int) (bias + i), mem[i / 4],
65 (bias + i == mark) ? " ********" : "");
66 kunmap_atomic(mem, KM_USER0);
67 /* give syslog time to catch up */
68 msleep(1);
69}
70
71void
72i915_gem_dump_object(struct drm_gem_object *obj, int len,
73 const char *where, uint32_t mark)
74{
75 struct drm_i915_gem_object *obj_priv = obj->driver_private;
76 int page;
77
78 DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset);
79 for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) {
80 int page_len, chunk, chunk_len;
81
82 page_len = len - page * PAGE_SIZE;
83 if (page_len > PAGE_SIZE)
84 page_len = PAGE_SIZE;
85
86 for (chunk = 0; chunk < page_len; chunk += 128) {
87 chunk_len = page_len - chunk;
88 if (chunk_len > 128)
89 chunk_len = 128;
90 i915_gem_dump_page(obj_priv->page_list[page],
91 chunk, chunk + chunk_len,
92 obj_priv->gtt_offset +
93 page * PAGE_SIZE,
94 mark);
95 }
96 }
97}
98#endif
99
100#if WATCH_LRU
101void
102i915_dump_lru(struct drm_device *dev, const char *where)
103{
104 drm_i915_private_t *dev_priv = dev->dev_private;
105 struct drm_i915_gem_object *obj_priv;
106
107 DRM_INFO("active list %s {\n", where);
108 list_for_each_entry(obj_priv, &dev_priv->mm.active_list,
109 list)
110 {
111 DRM_INFO(" %p: %08x\n", obj_priv,
112 obj_priv->last_rendering_seqno);
113 }
114 DRM_INFO("}\n");
115 DRM_INFO("flushing list %s {\n", where);
116 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list,
117 list)
118 {
119 DRM_INFO(" %p: %08x\n", obj_priv,
120 obj_priv->last_rendering_seqno);
121 }
122 DRM_INFO("}\n");
123 DRM_INFO("inactive %s {\n", where);
124 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
125 DRM_INFO(" %p: %08x\n", obj_priv,
126 obj_priv->last_rendering_seqno);
127 }
128 DRM_INFO("}\n");
129}
130#endif
131
132
133#if WATCH_COHERENCY
134void
135i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle)
136{
137 struct drm_device *dev = obj->dev;
138 struct drm_i915_gem_object *obj_priv = obj->driver_private;
139 int page;
140 uint32_t *gtt_mapping;
141 uint32_t *backing_map = NULL;
142 int bad_count = 0;
143
144 DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %dkb):\n",
145 __func__, obj, obj_priv->gtt_offset, handle,
146 obj->size / 1024);
147
148 gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset,
149 obj->size);
150 if (gtt_mapping == NULL) {
151 DRM_ERROR("failed to map GTT space\n");
152 return;
153 }
154
155 for (page = 0; page < obj->size / PAGE_SIZE; page++) {
156 int i;
157
158 backing_map = kmap_atomic(obj_priv->page_list[page], KM_USER0);
159
160 if (backing_map == NULL) {
161 DRM_ERROR("failed to map backing page\n");
162 goto out;
163 }
164
165 for (i = 0; i < PAGE_SIZE / 4; i++) {
166 uint32_t cpuval = backing_map[i];
167 uint32_t gttval = readl(gtt_mapping +
168 page * 1024 + i);
169
170 if (cpuval != gttval) {
171 DRM_INFO("incoherent CPU vs GPU at 0x%08x: "
172 "0x%08x vs 0x%08x\n",
173 (int)(obj_priv->gtt_offset +
174 page * PAGE_SIZE + i * 4),
175 cpuval, gttval);
176 if (bad_count++ >= 8) {
177 DRM_INFO("...\n");
178 goto out;
179 }
180 }
181 }
182 kunmap_atomic(backing_map, KM_USER0);
183 backing_map = NULL;
184 }
185
186 out:
187 if (backing_map != NULL)
188 kunmap_atomic(backing_map, KM_USER0);
189 iounmap(gtt_mapping);
190
191 /* give syslog time to catch up */
192 msleep(1);
193
194 /* Directly flush the object, since we just loaded values with the CPU
195 * from the backing pages and we don't want to disturb the cache
196 * management that we're trying to observe.
197 */
198
199 i915_gem_clflush_object(obj);
200}
201#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_proc.c b/drivers/gpu/drm/i915/i915_gem_proc.c
new file mode 100644
index 000000000000..15d4160415b0
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_proc.c
@@ -0,0 +1,292 @@
1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Keith Packard <keithp@keithp.com>
26 *
27 */
28
29#include "drmP.h"
30#include "drm.h"
31#include "i915_drm.h"
32#include "i915_drv.h"
33
34static int i915_gem_active_info(char *buf, char **start, off_t offset,
35 int request, int *eof, void *data)
36{
37 struct drm_minor *minor = (struct drm_minor *) data;
38 struct drm_device *dev = minor->dev;
39 drm_i915_private_t *dev_priv = dev->dev_private;
40 struct drm_i915_gem_object *obj_priv;
41 int len = 0;
42
43 if (offset > DRM_PROC_LIMIT) {
44 *eof = 1;
45 return 0;
46 }
47
48 *start = &buf[offset];
49 *eof = 0;
50 DRM_PROC_PRINT("Active:\n");
51 list_for_each_entry(obj_priv, &dev_priv->mm.active_list,
52 list)
53 {
54 struct drm_gem_object *obj = obj_priv->obj;
55 if (obj->name) {
56 DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n",
57 obj, obj->name,
58 obj->read_domains, obj->write_domain,
59 obj_priv->last_rendering_seqno);
60 } else {
61 DRM_PROC_PRINT(" %p: %08x %08x %d\n",
62 obj,
63 obj->read_domains, obj->write_domain,
64 obj_priv->last_rendering_seqno);
65 }
66 }
67 if (len > request + offset)
68 return request;
69 *eof = 1;
70 return len - offset;
71}
72
73static int i915_gem_flushing_info(char *buf, char **start, off_t offset,
74 int request, int *eof, void *data)
75{
76 struct drm_minor *minor = (struct drm_minor *) data;
77 struct drm_device *dev = minor->dev;
78 drm_i915_private_t *dev_priv = dev->dev_private;
79 struct drm_i915_gem_object *obj_priv;
80 int len = 0;
81
82 if (offset > DRM_PROC_LIMIT) {
83 *eof = 1;
84 return 0;
85 }
86
87 *start = &buf[offset];
88 *eof = 0;
89 DRM_PROC_PRINT("Flushing:\n");
90 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list,
91 list)
92 {
93 struct drm_gem_object *obj = obj_priv->obj;
94 if (obj->name) {
95 DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n",
96 obj, obj->name,
97 obj->read_domains, obj->write_domain,
98 obj_priv->last_rendering_seqno);
99 } else {
100 DRM_PROC_PRINT(" %p: %08x %08x %d\n", obj,
101 obj->read_domains, obj->write_domain,
102 obj_priv->last_rendering_seqno);
103 }
104 }
105 if (len > request + offset)
106 return request;
107 *eof = 1;
108 return len - offset;
109}
110
111static int i915_gem_inactive_info(char *buf, char **start, off_t offset,
112 int request, int *eof, void *data)
113{
114 struct drm_minor *minor = (struct drm_minor *) data;
115 struct drm_device *dev = minor->dev;
116 drm_i915_private_t *dev_priv = dev->dev_private;
117 struct drm_i915_gem_object *obj_priv;
118 int len = 0;
119
120 if (offset > DRM_PROC_LIMIT) {
121 *eof = 1;
122 return 0;
123 }
124
125 *start = &buf[offset];
126 *eof = 0;
127 DRM_PROC_PRINT("Inactive:\n");
128 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list,
129 list)
130 {
131 struct drm_gem_object *obj = obj_priv->obj;
132 if (obj->name) {
133 DRM_PROC_PRINT(" %p(%d): %08x %08x %d\n",
134 obj, obj->name,
135 obj->read_domains, obj->write_domain,
136 obj_priv->last_rendering_seqno);
137 } else {
138 DRM_PROC_PRINT(" %p: %08x %08x %d\n", obj,
139 obj->read_domains, obj->write_domain,
140 obj_priv->last_rendering_seqno);
141 }
142 }
143 if (len > request + offset)
144 return request;
145 *eof = 1;
146 return len - offset;
147}
148
149static int i915_gem_request_info(char *buf, char **start, off_t offset,
150 int request, int *eof, void *data)
151{
152 struct drm_minor *minor = (struct drm_minor *) data;
153 struct drm_device *dev = minor->dev;
154 drm_i915_private_t *dev_priv = dev->dev_private;
155 struct drm_i915_gem_request *gem_request;
156 int len = 0;
157
158 if (offset > DRM_PROC_LIMIT) {
159 *eof = 1;
160 return 0;
161 }
162
163 *start = &buf[offset];
164 *eof = 0;
165 DRM_PROC_PRINT("Request:\n");
166 list_for_each_entry(gem_request, &dev_priv->mm.request_list,
167 list)
168 {
169 DRM_PROC_PRINT(" %d @ %d %08x\n",
170 gem_request->seqno,
171 (int) (jiffies - gem_request->emitted_jiffies),
172 gem_request->flush_domains);
173 }
174 if (len > request + offset)
175 return request;
176 *eof = 1;
177 return len - offset;
178}
179
180static int i915_gem_seqno_info(char *buf, char **start, off_t offset,
181 int request, int *eof, void *data)
182{
183 struct drm_minor *minor = (struct drm_minor *) data;
184 struct drm_device *dev = minor->dev;
185 drm_i915_private_t *dev_priv = dev->dev_private;
186 int len = 0;
187
188 if (offset > DRM_PROC_LIMIT) {
189 *eof = 1;
190 return 0;
191 }
192
193 *start = &buf[offset];
194 *eof = 0;
195 DRM_PROC_PRINT("Current sequence: %d\n", i915_get_gem_seqno(dev));
196 DRM_PROC_PRINT("Waiter sequence: %d\n",
197 dev_priv->mm.waiting_gem_seqno);
198 DRM_PROC_PRINT("IRQ sequence: %d\n", dev_priv->mm.irq_gem_seqno);
199 if (len > request + offset)
200 return request;
201 *eof = 1;
202 return len - offset;
203}
204
205
206static int i915_interrupt_info(char *buf, char **start, off_t offset,
207 int request, int *eof, void *data)
208{
209 struct drm_minor *minor = (struct drm_minor *) data;
210 struct drm_device *dev = minor->dev;
211 drm_i915_private_t *dev_priv = dev->dev_private;
212 int len = 0;
213
214 if (offset > DRM_PROC_LIMIT) {
215 *eof = 1;
216 return 0;
217 }
218
219 *start = &buf[offset];
220 *eof = 0;
221 DRM_PROC_PRINT("Interrupt enable: %08x\n",
222 I915_READ(IER));
223 DRM_PROC_PRINT("Interrupt identity: %08x\n",
224 I915_READ(IIR));
225 DRM_PROC_PRINT("Interrupt mask: %08x\n",
226 I915_READ(IMR));
227 DRM_PROC_PRINT("Pipe A stat: %08x\n",
228 I915_READ(PIPEASTAT));
229 DRM_PROC_PRINT("Pipe B stat: %08x\n",
230 I915_READ(PIPEBSTAT));
231 DRM_PROC_PRINT("Interrupts received: %d\n",
232 atomic_read(&dev_priv->irq_received));
233 DRM_PROC_PRINT("Current sequence: %d\n",
234 i915_get_gem_seqno(dev));
235 DRM_PROC_PRINT("Waiter sequence: %d\n",
236 dev_priv->mm.waiting_gem_seqno);
237 DRM_PROC_PRINT("IRQ sequence: %d\n",
238 dev_priv->mm.irq_gem_seqno);
239 if (len > request + offset)
240 return request;
241 *eof = 1;
242 return len - offset;
243}
244
245static struct drm_proc_list {
246 /** file name */
247 const char *name;
248 /** proc callback*/
249 int (*f) (char *, char **, off_t, int, int *, void *);
250} i915_gem_proc_list[] = {
251 {"i915_gem_active", i915_gem_active_info},
252 {"i915_gem_flushing", i915_gem_flushing_info},
253 {"i915_gem_inactive", i915_gem_inactive_info},
254 {"i915_gem_request", i915_gem_request_info},
255 {"i915_gem_seqno", i915_gem_seqno_info},
256 {"i915_gem_interrupt", i915_interrupt_info},
257};
258
259#define I915_GEM_PROC_ENTRIES ARRAY_SIZE(i915_gem_proc_list)
260
261int i915_gem_proc_init(struct drm_minor *minor)
262{
263 struct proc_dir_entry *ent;
264 int i, j;
265
266 for (i = 0; i < I915_GEM_PROC_ENTRIES; i++) {
267 ent = create_proc_entry(i915_gem_proc_list[i].name,
268 S_IFREG | S_IRUGO, minor->dev_root);
269 if (!ent) {
270 DRM_ERROR("Cannot create /proc/dri/.../%s\n",
271 i915_gem_proc_list[i].name);
272 for (j = 0; j < i; j++)
273 remove_proc_entry(i915_gem_proc_list[i].name,
274 minor->dev_root);
275 return -1;
276 }
277 ent->read_proc = i915_gem_proc_list[i].f;
278 ent->data = minor;
279 }
280 return 0;
281}
282
283void i915_gem_proc_cleanup(struct drm_minor *minor)
284{
285 int i;
286
287 if (!minor->dev_root)
288 return;
289
290 for (i = 0; i < I915_GEM_PROC_ENTRIES; i++)
291 remove_proc_entry(i915_gem_proc_list[i].name, minor->dev_root);
292}
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
new file mode 100644
index 000000000000..0c1b3a0834e1
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -0,0 +1,256 @@
1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32
33/** @file i915_gem_tiling.c
34 *
35 * Support for managing tiling state of buffer objects.
36 *
37 * The idea behind tiling is to increase cache hit rates by rearranging
38 * pixel data so that a group of pixel accesses are in the same cacheline.
39 * Performance improvement from doing this on the back/depth buffer are on
40 * the order of 30%.
41 *
42 * Intel architectures make this somewhat more complicated, though, by
43 * adjustments made to addressing of data when the memory is in interleaved
44 * mode (matched pairs of DIMMS) to improve memory bandwidth.
45 * For interleaved memory, the CPU sends every sequential 64 bytes
46 * to an alternate memory channel so it can get the bandwidth from both.
47 *
48 * The GPU also rearranges its accesses for increased bandwidth to interleaved
49 * memory, and it matches what the CPU does for non-tiled. However, when tiled
50 * it does it a little differently, since one walks addresses not just in the
51 * X direction but also Y. So, along with alternating channels when bit
52 * 6 of the address flips, it also alternates when other bits flip -- Bits 9
53 * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines)
54 * are common to both the 915 and 965-class hardware.
55 *
56 * The CPU also sometimes XORs in higher bits as well, to improve
57 * bandwidth doing strided access like we do so frequently in graphics. This
58 * is called "Channel XOR Randomization" in the MCH documentation. The result
59 * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address
60 * decode.
61 *
62 * All of this bit 6 XORing has an effect on our memory management,
63 * as we need to make sure that the 3d driver can correctly address object
64 * contents.
65 *
66 * If we don't have interleaved memory, all tiling is safe and no swizzling is
67 * required.
68 *
69 * When bit 17 is XORed in, we simply refuse to tile at all. Bit
70 * 17 is not just a page offset, so as we page an objet out and back in,
71 * individual pages in it will have different bit 17 addresses, resulting in
72 * each 64 bytes being swapped with its neighbor!
73 *
74 * Otherwise, if interleaved, we have to tell the 3d driver what the address
75 * swizzling it needs to do is, since it's writing with the CPU to the pages
76 * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the
77 * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling
78 * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order
79 * to match what the GPU expects.
80 */
81
82/**
83 * Detects bit 6 swizzling of address lookup between IGD access and CPU
84 * access through main memory.
85 */
86void
87i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
88{
89 drm_i915_private_t *dev_priv = dev->dev_private;
90 uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
91 uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
92
93 if (!IS_I9XX(dev)) {
94 /* As far as we know, the 865 doesn't have these bit 6
95 * swizzling issues.
96 */
97 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
98 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
99 } else if (!IS_I965G(dev) || IS_I965GM(dev)) {
100 uint32_t dcc;
101
102 /* On 915-945 and GM965, channel interleave by the CPU is
103 * determined by DCC. The CPU will alternate based on bit 6
104 * in interleaved mode, and the GPU will then also alternate
105 * on bit 6, 9, and 10 for X, but the CPU may also optionally
106 * alternate based on bit 17 (XOR not disabled and XOR
107 * bit == 17).
108 */
109 dcc = I915_READ(DCC);
110 switch (dcc & DCC_ADDRESSING_MODE_MASK) {
111 case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
112 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
113 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
114 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
115 break;
116 case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
117 if (IS_I915G(dev) || IS_I915GM(dev) ||
118 dcc & DCC_CHANNEL_XOR_DISABLE) {
119 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
120 swizzle_y = I915_BIT_6_SWIZZLE_9;
121 } else if (IS_I965GM(dev)) {
122 /* GM965 only does bit 11-based channel
123 * randomization
124 */
125 swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
126 swizzle_y = I915_BIT_6_SWIZZLE_9_11;
127 } else {
128 /* Bit 17 or perhaps other swizzling */
129 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
130 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
131 }
132 break;
133 }
134 if (dcc == 0xffffffff) {
135 DRM_ERROR("Couldn't read from MCHBAR. "
136 "Disabling tiling.\n");
137 swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
138 swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
139 }
140 } else {
141 /* The 965, G33, and newer, have a very flexible memory
142 * configuration. It will enable dual-channel mode
143 * (interleaving) on as much memory as it can, and the GPU
144 * will additionally sometimes enable different bit 6
145 * swizzling for tiled objects from the CPU.
146 *
147 * Here's what I found on the G965:
148 * slot fill memory size swizzling
149 * 0A 0B 1A 1B 1-ch 2-ch
150 * 512 0 0 0 512 0 O
151 * 512 0 512 0 16 1008 X
152 * 512 0 0 512 16 1008 X
153 * 0 512 0 512 16 1008 X
154 * 1024 1024 1024 0 2048 1024 O
155 *
156 * We could probably detect this based on either the DRB
157 * matching, which was the case for the swizzling required in
158 * the table above, or from the 1-ch value being less than
159 * the minimum size of a rank.
160 */
161 if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) {
162 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
163 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
164 } else {
165 swizzle_x = I915_BIT_6_SWIZZLE_9_10;
166 swizzle_y = I915_BIT_6_SWIZZLE_9;
167 }
168 }
169
170 dev_priv->mm.bit_6_swizzle_x = swizzle_x;
171 dev_priv->mm.bit_6_swizzle_y = swizzle_y;
172}
173
174/**
175 * Sets the tiling mode of an object, returning the required swizzling of
176 * bit 6 of addresses in the object.
177 */
178int
179i915_gem_set_tiling(struct drm_device *dev, void *data,
180 struct drm_file *file_priv)
181{
182 struct drm_i915_gem_set_tiling *args = data;
183 drm_i915_private_t *dev_priv = dev->dev_private;
184 struct drm_gem_object *obj;
185 struct drm_i915_gem_object *obj_priv;
186
187 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
188 if (obj == NULL)
189 return -EINVAL;
190 obj_priv = obj->driver_private;
191
192 mutex_lock(&dev->struct_mutex);
193
194 if (args->tiling_mode == I915_TILING_NONE) {
195 obj_priv->tiling_mode = I915_TILING_NONE;
196 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
197 } else {
198 if (args->tiling_mode == I915_TILING_X)
199 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
200 else
201 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
202 /* If we can't handle the swizzling, make it untiled. */
203 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
204 args->tiling_mode = I915_TILING_NONE;
205 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
206 }
207 }
208 obj_priv->tiling_mode = args->tiling_mode;
209
210 mutex_unlock(&dev->struct_mutex);
211
212 drm_gem_object_unreference(obj);
213
214 return 0;
215}
216
217/**
218 * Returns the current tiling mode and required bit 6 swizzling for the object.
219 */
220int
221i915_gem_get_tiling(struct drm_device *dev, void *data,
222 struct drm_file *file_priv)
223{
224 struct drm_i915_gem_get_tiling *args = data;
225 drm_i915_private_t *dev_priv = dev->dev_private;
226 struct drm_gem_object *obj;
227 struct drm_i915_gem_object *obj_priv;
228
229 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
230 if (obj == NULL)
231 return -EINVAL;
232 obj_priv = obj->driver_private;
233
234 mutex_lock(&dev->struct_mutex);
235
236 args->tiling_mode = obj_priv->tiling_mode;
237 switch (obj_priv->tiling_mode) {
238 case I915_TILING_X:
239 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
240 break;
241 case I915_TILING_Y:
242 args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
243 break;
244 case I915_TILING_NONE:
245 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
246 break;
247 default:
248 DRM_ERROR("unknown tiling mode\n");
249 }
250
251 mutex_unlock(&dev->struct_mutex);
252
253 drm_gem_object_unreference(obj);
254
255 return 0;
256}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index f8759597233b..f295bdf16e2d 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -407,15 +407,20 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
407 I915_WRITE(PIPEBSTAT, pipeb_stats); 407 I915_WRITE(PIPEBSTAT, pipeb_stats);
408 } 408 }
409 409
410 if (iir & I915_ASLE_INTERRUPT) 410 I915_WRITE(IIR, iir);
411 opregion_asle_intr(dev); 411 if (dev->pdev->msi_enabled)
412 I915_WRITE(IMR, dev_priv->irq_mask_reg);
413 (void) I915_READ(IIR); /* Flush posted writes */
412 414
413 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv); 415 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
414 416
415 if (dev->pdev->msi_enabled) 417 if (iir & I915_USER_INTERRUPT) {
416 I915_WRITE(IMR, dev_priv->irq_mask_reg); 418 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
417 I915_WRITE(IIR, iir); 419 DRM_WAKEUP(&dev_priv->irq_queue);
418 (void) I915_READ(IIR); 420 }
421
422 if (iir & I915_ASLE_INTERRUPT)
423 opregion_asle_intr(dev);
419 424
420 if (vblank && dev_priv->swaps_pending > 0) 425 if (vblank && dev_priv->swaps_pending > 0)
421 drm_locked_tasklet(dev, i915_vblank_tasklet); 426 drm_locked_tasklet(dev, i915_vblank_tasklet);
@@ -449,7 +454,7 @@ static int i915_emit_irq(struct drm_device * dev)
449 return dev_priv->counter; 454 return dev_priv->counter;
450} 455}
451 456
452static void i915_user_irq_get(struct drm_device *dev) 457void i915_user_irq_get(struct drm_device *dev)
453{ 458{
454 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 459 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
455 460
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 43ad2cb8b9f5..5c2d9f206d05 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -25,19 +25,6 @@
25#ifndef _I915_REG_H_ 25#ifndef _I915_REG_H_
26#define _I915_REG_H_ 26#define _I915_REG_H_
27 27
28/* MCH MMIO space */
29/** 915-945 and GM965 MCH register controlling DRAM channel access */
30#define DCC 0x200
31#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0)
32#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0)
33#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0)
34#define DCC_ADDRESSING_MODE_MASK (3 << 0)
35#define DCC_CHANNEL_XOR_DISABLE (1 << 10)
36
37/** 965 MCH register controlling DRAM channel configuration */
38#define CHDECMISC 0x111
39#define CHDECMISC_FLEXMEMORY (1 << 1)
40
41/* 28/*
42 * The Bridge device's PCI config space has information about the 29 * The Bridge device's PCI config space has information about the
43 * fb aperture size and the amount of pre-reserved memory. 30 * fb aperture size and the amount of pre-reserved memory.
@@ -516,6 +503,30 @@
516#define PALETTE_A 0x0a000 503#define PALETTE_A 0x0a000
517#define PALETTE_B 0x0a800 504#define PALETTE_B 0x0a800
518 505
506/* MCH MMIO space */
507
508/*
509 * MCHBAR mirror.
510 *
511 * This mirrors the MCHBAR MMIO space whose location is determined by
512 * device 0 function 0's pci config register 0x44 or 0x48 and matches it in
513 * every way. It is not accessible from the CP register read instructions.
514 *
515 */
516#define MCHBAR_MIRROR_BASE 0x10000
517
518/** 915-945 and GM965 MCH register controlling DRAM channel access */
519#define DCC 0x10200
520#define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0)
521#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC (1 << 0)
522#define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED (2 << 0)
523#define DCC_ADDRESSING_MODE_MASK (3 << 0)
524#define DCC_CHANNEL_XOR_DISABLE (1 << 10)
525
526/** 965 MCH register controlling DRAM channel configuration */
527#define C0DRB3 0x10206
528#define C1DRB3 0x10606
529
519/* 530/*
520 * Overlay regs 531 * Overlay regs
521 */ 532 */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-23 05:26:05 -0500