aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_ringbuffer.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/intel_ringbuffer.c')
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c1264
1 files changed, 861 insertions, 403 deletions
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 09f2dc353ae2..6218fa97aa1e 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -34,6 +34,14 @@
34#include "i915_trace.h" 34#include "i915_trace.h"
35#include "intel_drv.h" 35#include "intel_drv.h"
36 36
37static inline int ring_space(struct intel_ring_buffer *ring)
38{
39 int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
40 if (space < 0)
41 space += ring->size;
42 return space;
43}
44
37static u32 i915_gem_get_seqno(struct drm_device *dev) 45static u32 i915_gem_get_seqno(struct drm_device *dev)
38{ 46{
39 drm_i915_private_t *dev_priv = dev->dev_private; 47 drm_i915_private_t *dev_priv = dev->dev_private;
@@ -48,14 +56,15 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
48 return seqno; 56 return seqno;
49} 57}
50 58
51static void 59static int
52render_ring_flush(struct drm_device *dev, 60render_ring_flush(struct intel_ring_buffer *ring,
53 struct intel_ring_buffer *ring,
54 u32 invalidate_domains, 61 u32 invalidate_domains,
55 u32 flush_domains) 62 u32 flush_domains)
56{ 63{
64 struct drm_device *dev = ring->dev;
57 drm_i915_private_t *dev_priv = dev->dev_private; 65 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd; 66 u32 cmd;
67 int ret;
59 68
60#if WATCH_EXEC 69#if WATCH_EXEC
61 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__, 70 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
@@ -109,79 +118,87 @@ render_ring_flush(struct drm_device *dev,
109 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) 118 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
110 cmd |= MI_EXE_FLUSH; 119 cmd |= MI_EXE_FLUSH;
111 120
121 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
122 (IS_G4X(dev) || IS_GEN5(dev)))
123 cmd |= MI_INVALIDATE_ISP;
124
112#if WATCH_EXEC 125#if WATCH_EXEC
113 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd); 126 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
114#endif 127#endif
115 intel_ring_begin(dev, ring, 2); 128 ret = intel_ring_begin(ring, 2);
116 intel_ring_emit(dev, ring, cmd); 129 if (ret)
117 intel_ring_emit(dev, ring, MI_NOOP); 130 return ret;
118 intel_ring_advance(dev, ring); 131
132 intel_ring_emit(ring, cmd);
133 intel_ring_emit(ring, MI_NOOP);
134 intel_ring_advance(ring);
119 } 135 }
136
137 return 0;
120} 138}
121 139
122static void ring_write_tail(struct drm_device *dev, 140static void ring_write_tail(struct intel_ring_buffer *ring,
123 struct intel_ring_buffer *ring,
124 u32 value) 141 u32 value)
125{ 142{
126 drm_i915_private_t *dev_priv = dev->dev_private; 143 drm_i915_private_t *dev_priv = ring->dev->dev_private;
127 I915_WRITE_TAIL(ring, value); 144 I915_WRITE_TAIL(ring, value);
128} 145}
129 146
130u32 intel_ring_get_active_head(struct drm_device *dev, 147u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
131 struct intel_ring_buffer *ring)
132{ 148{
133 drm_i915_private_t *dev_priv = dev->dev_private; 149 drm_i915_private_t *dev_priv = ring->dev->dev_private;
134 u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ? 150 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
135 RING_ACTHD(ring->mmio_base) : ACTHD; 151 RING_ACTHD(ring->mmio_base) : ACTHD;
136 152
137 return I915_READ(acthd_reg); 153 return I915_READ(acthd_reg);
138} 154}
139 155
140static int init_ring_common(struct drm_device *dev, 156static int init_ring_common(struct intel_ring_buffer *ring)
141 struct intel_ring_buffer *ring)
142{ 157{
158 drm_i915_private_t *dev_priv = ring->dev->dev_private;
159 struct drm_i915_gem_object *obj = ring->obj;
143 u32 head; 160 u32 head;
144 drm_i915_private_t *dev_priv = dev->dev_private;
145 struct drm_i915_gem_object *obj_priv;
146 obj_priv = to_intel_bo(ring->gem_object);
147 161
148 /* Stop the ring if it's running. */ 162 /* Stop the ring if it's running. */
149 I915_WRITE_CTL(ring, 0); 163 I915_WRITE_CTL(ring, 0);
150 I915_WRITE_HEAD(ring, 0); 164 I915_WRITE_HEAD(ring, 0);
151 ring->write_tail(dev, ring, 0); 165 ring->write_tail(ring, 0);
152 166
153 /* Initialize the ring. */ 167 /* Initialize the ring. */
154 I915_WRITE_START(ring, obj_priv->gtt_offset); 168 I915_WRITE_START(ring, obj->gtt_offset);
155 head = I915_READ_HEAD(ring) & HEAD_ADDR; 169 head = I915_READ_HEAD(ring) & HEAD_ADDR;
156 170
157 /* G45 ring initialization fails to reset head to zero */ 171 /* G45 ring initialization fails to reset head to zero */
158 if (head != 0) { 172 if (head != 0) {
159 DRM_ERROR("%s head not reset to zero " 173 DRM_DEBUG_KMS("%s head not reset to zero "
160 "ctl %08x head %08x tail %08x start %08x\n", 174 "ctl %08x head %08x tail %08x start %08x\n",
161 ring->name, 175 ring->name,
162 I915_READ_CTL(ring), 176 I915_READ_CTL(ring),
163 I915_READ_HEAD(ring), 177 I915_READ_HEAD(ring),
164 I915_READ_TAIL(ring), 178 I915_READ_TAIL(ring),
165 I915_READ_START(ring)); 179 I915_READ_START(ring));
166 180
167 I915_WRITE_HEAD(ring, 0); 181 I915_WRITE_HEAD(ring, 0);
168 182
169 DRM_ERROR("%s head forced to zero " 183 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
170 "ctl %08x head %08x tail %08x start %08x\n", 184 DRM_ERROR("failed to set %s head to zero "
171 ring->name, 185 "ctl %08x head %08x tail %08x start %08x\n",
172 I915_READ_CTL(ring), 186 ring->name,
173 I915_READ_HEAD(ring), 187 I915_READ_CTL(ring),
174 I915_READ_TAIL(ring), 188 I915_READ_HEAD(ring),
175 I915_READ_START(ring)); 189 I915_READ_TAIL(ring),
190 I915_READ_START(ring));
191 }
176 } 192 }
177 193
178 I915_WRITE_CTL(ring, 194 I915_WRITE_CTL(ring,
179 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES) 195 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
180 | RING_NO_REPORT | RING_VALID); 196 | RING_REPORT_64K | RING_VALID);
181 197
182 head = I915_READ_HEAD(ring) & HEAD_ADDR;
183 /* If the head is still not zero, the ring is dead */ 198 /* If the head is still not zero, the ring is dead */
184 if (head != 0) { 199 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
200 I915_READ_START(ring) != obj->gtt_offset ||
201 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
185 DRM_ERROR("%s initialization failed " 202 DRM_ERROR("%s initialization failed "
186 "ctl %08x head %08x tail %08x start %08x\n", 203 "ctl %08x head %08x tail %08x start %08x\n",
187 ring->name, 204 ring->name,
@@ -192,344 +209,569 @@ static int init_ring_common(struct drm_device *dev,
192 return -EIO; 209 return -EIO;
193 } 210 }
194 211
195 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 212 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
196 i915_kernel_lost_context(dev); 213 i915_kernel_lost_context(ring->dev);
197 else { 214 else {
198 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; 215 ring->head = I915_READ_HEAD(ring);
199 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; 216 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
200 ring->space = ring->head - (ring->tail + 8); 217 ring->space = ring_space(ring);
201 if (ring->space < 0)
202 ring->space += ring->size;
203 } 218 }
219
204 return 0; 220 return 0;
205} 221}
206 222
207static int init_render_ring(struct drm_device *dev, 223/*
208 struct intel_ring_buffer *ring) 224 * 965+ support PIPE_CONTROL commands, which provide finer grained control
225 * over cache flushing.
226 */
227struct pipe_control {
228 struct drm_i915_gem_object *obj;
229 volatile u32 *cpu_page;
230 u32 gtt_offset;
231};
232
233static int
234init_pipe_control(struct intel_ring_buffer *ring)
209{ 235{
210 drm_i915_private_t *dev_priv = dev->dev_private; 236 struct pipe_control *pc;
211 int ret = init_ring_common(dev, ring); 237 struct drm_i915_gem_object *obj;
212 int mode; 238 int ret;
239
240 if (ring->private)
241 return 0;
242
243 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
244 if (!pc)
245 return -ENOMEM;
246
247 obj = i915_gem_alloc_object(ring->dev, 4096);
248 if (obj == NULL) {
249 DRM_ERROR("Failed to allocate seqno page\n");
250 ret = -ENOMEM;
251 goto err;
252 }
253 obj->agp_type = AGP_USER_CACHED_MEMORY;
254
255 ret = i915_gem_object_pin(obj, 4096, true);
256 if (ret)
257 goto err_unref;
258
259 pc->gtt_offset = obj->gtt_offset;
260 pc->cpu_page = kmap(obj->pages[0]);
261 if (pc->cpu_page == NULL)
262 goto err_unpin;
263
264 pc->obj = obj;
265 ring->private = pc;
266 return 0;
267
268err_unpin:
269 i915_gem_object_unpin(obj);
270err_unref:
271 drm_gem_object_unreference(&obj->base);
272err:
273 kfree(pc);
274 return ret;
275}
276
277static void
278cleanup_pipe_control(struct intel_ring_buffer *ring)
279{
280 struct pipe_control *pc = ring->private;
281 struct drm_i915_gem_object *obj;
282
283 if (!ring->private)
284 return;
285
286 obj = pc->obj;
287 kunmap(obj->pages[0]);
288 i915_gem_object_unpin(obj);
289 drm_gem_object_unreference(&obj->base);
290
291 kfree(pc);
292 ring->private = NULL;
293}
294
295static int init_render_ring(struct intel_ring_buffer *ring)
296{
297 struct drm_device *dev = ring->dev;
298 struct drm_i915_private *dev_priv = dev->dev_private;
299 int ret = init_ring_common(ring);
213 300
214 if (INTEL_INFO(dev)->gen > 3) { 301 if (INTEL_INFO(dev)->gen > 3) {
215 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; 302 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
216 if (IS_GEN6(dev)) 303 if (IS_GEN6(dev))
217 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE; 304 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
218 I915_WRITE(MI_MODE, mode); 305 I915_WRITE(MI_MODE, mode);
219 } 306 }
307
308 if (INTEL_INFO(dev)->gen >= 6) {
309 } else if (IS_GEN5(dev)) {
310 ret = init_pipe_control(ring);
311 if (ret)
312 return ret;
313 }
314
220 return ret; 315 return ret;
221} 316}
222 317
223#define PIPE_CONTROL_FLUSH(addr) \ 318static void render_ring_cleanup(struct intel_ring_buffer *ring)
319{
320 if (!ring->private)
321 return;
322
323 cleanup_pipe_control(ring);
324}
325
326static void
327update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
328{
329 struct drm_device *dev = ring->dev;
330 struct drm_i915_private *dev_priv = dev->dev_private;
331 int id;
332
333 /*
334 * cs -> 1 = vcs, 0 = bcs
335 * vcs -> 1 = bcs, 0 = cs,
336 * bcs -> 1 = cs, 0 = vcs.
337 */
338 id = ring - dev_priv->ring;
339 id += 2 - i;
340 id %= 3;
341
342 intel_ring_emit(ring,
343 MI_SEMAPHORE_MBOX |
344 MI_SEMAPHORE_REGISTER |
345 MI_SEMAPHORE_UPDATE);
346 intel_ring_emit(ring, seqno);
347 intel_ring_emit(ring,
348 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
349}
350
351static int
352gen6_add_request(struct intel_ring_buffer *ring,
353 u32 *result)
354{
355 u32 seqno;
356 int ret;
357
358 ret = intel_ring_begin(ring, 10);
359 if (ret)
360 return ret;
361
362 seqno = i915_gem_get_seqno(ring->dev);
363 update_semaphore(ring, 0, seqno);
364 update_semaphore(ring, 1, seqno);
365
366 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
367 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
368 intel_ring_emit(ring, seqno);
369 intel_ring_emit(ring, MI_USER_INTERRUPT);
370 intel_ring_advance(ring);
371
372 *result = seqno;
373 return 0;
374}
375
376int
377intel_ring_sync(struct intel_ring_buffer *ring,
378 struct intel_ring_buffer *to,
379 u32 seqno)
380{
381 int ret;
382
383 ret = intel_ring_begin(ring, 4);
384 if (ret)
385 return ret;
386
387 intel_ring_emit(ring,
388 MI_SEMAPHORE_MBOX |
389 MI_SEMAPHORE_REGISTER |
390 intel_ring_sync_index(ring, to) << 17 |
391 MI_SEMAPHORE_COMPARE);
392 intel_ring_emit(ring, seqno);
393 intel_ring_emit(ring, 0);
394 intel_ring_emit(ring, MI_NOOP);
395 intel_ring_advance(ring);
396
397 return 0;
398}
399
400#define PIPE_CONTROL_FLUSH(ring__, addr__) \
224do { \ 401do { \
225 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ 402 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
226 PIPE_CONTROL_DEPTH_STALL | 2); \ 403 PIPE_CONTROL_DEPTH_STALL | 2); \
227 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ 404 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
228 OUT_RING(0); \ 405 intel_ring_emit(ring__, 0); \
229 OUT_RING(0); \ 406 intel_ring_emit(ring__, 0); \
230} while (0) 407} while (0)
231 408
232/** 409static int
233 * Creates a new sequence number, emitting a write of it to the status page 410pc_render_add_request(struct intel_ring_buffer *ring,
234 * plus an interrupt, which will trigger i915_user_interrupt_handler. 411 u32 *result)
235 *
236 * Must be called with struct_lock held.
237 *
238 * Returned sequence numbers are nonzero on success.
239 */
240static u32
241render_ring_add_request(struct drm_device *dev,
242 struct intel_ring_buffer *ring,
243 u32 flush_domains)
244{ 412{
245 drm_i915_private_t *dev_priv = dev->dev_private; 413 struct drm_device *dev = ring->dev;
246 u32 seqno; 414 u32 seqno = i915_gem_get_seqno(dev);
415 struct pipe_control *pc = ring->private;
416 u32 scratch_addr = pc->gtt_offset + 128;
417 int ret;
247 418
248 seqno = i915_gem_get_seqno(dev); 419 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
249 420 * incoherent with writes to memory, i.e. completely fubar,
250 if (IS_GEN6(dev)) { 421 * so we need to use PIPE_NOTIFY instead.
251 BEGIN_LP_RING(6); 422 *
252 OUT_RING(GFX_OP_PIPE_CONTROL | 3); 423 * However, we also need to workaround the qword write
253 OUT_RING(PIPE_CONTROL_QW_WRITE | 424 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
254 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH | 425 * memory before requesting an interrupt.
255 PIPE_CONTROL_NOTIFY); 426 */
256 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); 427 ret = intel_ring_begin(ring, 32);
257 OUT_RING(seqno); 428 if (ret)
258 OUT_RING(0); 429 return ret;
259 OUT_RING(0); 430
260 ADVANCE_LP_RING(); 431 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
261 } else if (HAS_PIPE_CONTROL(dev)) { 432 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
262 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128; 433 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
434 intel_ring_emit(ring, seqno);
435 intel_ring_emit(ring, 0);
436 PIPE_CONTROL_FLUSH(ring, scratch_addr);
437 scratch_addr += 128; /* write to separate cachelines */
438 PIPE_CONTROL_FLUSH(ring, scratch_addr);
439 scratch_addr += 128;
440 PIPE_CONTROL_FLUSH(ring, scratch_addr);
441 scratch_addr += 128;
442 PIPE_CONTROL_FLUSH(ring, scratch_addr);
443 scratch_addr += 128;
444 PIPE_CONTROL_FLUSH(ring, scratch_addr);
445 scratch_addr += 128;
446 PIPE_CONTROL_FLUSH(ring, scratch_addr);
447 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
448 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
449 PIPE_CONTROL_NOTIFY);
450 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
451 intel_ring_emit(ring, seqno);
452 intel_ring_emit(ring, 0);
453 intel_ring_advance(ring);
454
455 *result = seqno;
456 return 0;
457}
263 458
264 /* 459static int
265 * Workaround qword write incoherence by flushing the 460render_ring_add_request(struct intel_ring_buffer *ring,
266 * PIPE_NOTIFY buffers out to memory before requesting 461 u32 *result)
267 * an interrupt. 462{
268 */ 463 struct drm_device *dev = ring->dev;
269 BEGIN_LP_RING(32); 464 u32 seqno = i915_gem_get_seqno(dev);
270 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | 465 int ret;
271 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
272 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
273 OUT_RING(seqno);
274 OUT_RING(0);
275 PIPE_CONTROL_FLUSH(scratch_addr);
276 scratch_addr += 128; /* write to separate cachelines */
277 PIPE_CONTROL_FLUSH(scratch_addr);
278 scratch_addr += 128;
279 PIPE_CONTROL_FLUSH(scratch_addr);
280 scratch_addr += 128;
281 PIPE_CONTROL_FLUSH(scratch_addr);
282 scratch_addr += 128;
283 PIPE_CONTROL_FLUSH(scratch_addr);
284 scratch_addr += 128;
285 PIPE_CONTROL_FLUSH(scratch_addr);
286 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
287 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
288 PIPE_CONTROL_NOTIFY);
289 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
290 OUT_RING(seqno);
291 OUT_RING(0);
292 ADVANCE_LP_RING();
293 } else {
294 BEGIN_LP_RING(4);
295 OUT_RING(MI_STORE_DWORD_INDEX);
296 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
297 OUT_RING(seqno);
298 466
299 OUT_RING(MI_USER_INTERRUPT); 467 ret = intel_ring_begin(ring, 4);
300 ADVANCE_LP_RING(); 468 if (ret)
301 } 469 return ret;
302 return seqno; 470
471 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
472 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
473 intel_ring_emit(ring, seqno);
474 intel_ring_emit(ring, MI_USER_INTERRUPT);
475 intel_ring_advance(ring);
476
477 *result = seqno;
478 return 0;
303} 479}
304 480
305static u32 481static u32
306render_ring_get_seqno(struct drm_device *dev, 482ring_get_seqno(struct intel_ring_buffer *ring)
307 struct intel_ring_buffer *ring)
308{ 483{
309 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 484 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
310 if (HAS_PIPE_CONTROL(dev)) 485}
311 return ((volatile u32 *)(dev_priv->seqno_page))[0]; 486
312 else 487static u32
313 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 488pc_render_get_seqno(struct intel_ring_buffer *ring)
489{
490 struct pipe_control *pc = ring->private;
491 return pc->cpu_page[0];
314} 492}
315 493
316static void 494static void
317render_ring_get_user_irq(struct drm_device *dev, 495ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
318 struct intel_ring_buffer *ring)
319{ 496{
320 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 497 dev_priv->gt_irq_mask &= ~mask;
321 unsigned long irqflags; 498 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
499 POSTING_READ(GTIMR);
500}
322 501
323 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 502static void
324 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) { 503ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
504{
505 dev_priv->gt_irq_mask |= mask;
506 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
507 POSTING_READ(GTIMR);
508}
509
510static void
511i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
512{
513 dev_priv->irq_mask &= ~mask;
514 I915_WRITE(IMR, dev_priv->irq_mask);
515 POSTING_READ(IMR);
516}
517
518static void
519i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
520{
521 dev_priv->irq_mask |= mask;
522 I915_WRITE(IMR, dev_priv->irq_mask);
523 POSTING_READ(IMR);
524}
525
526static bool
527render_ring_get_irq(struct intel_ring_buffer *ring)
528{
529 struct drm_device *dev = ring->dev;
530 drm_i915_private_t *dev_priv = dev->dev_private;
531
532 if (!dev->irq_enabled)
533 return false;
534
535 spin_lock(&ring->irq_lock);
536 if (ring->irq_refcount++ == 0) {
325 if (HAS_PCH_SPLIT(dev)) 537 if (HAS_PCH_SPLIT(dev))
326 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 538 ironlake_enable_irq(dev_priv,
539 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
327 else 540 else
328 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 541 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
329 } 542 }
330 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 543 spin_unlock(&ring->irq_lock);
544
545 return true;
331} 546}
332 547
333static void 548static void
334render_ring_put_user_irq(struct drm_device *dev, 549render_ring_put_irq(struct intel_ring_buffer *ring)
335 struct intel_ring_buffer *ring)
336{ 550{
337 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 551 struct drm_device *dev = ring->dev;
338 unsigned long irqflags; 552 drm_i915_private_t *dev_priv = dev->dev_private;
339 553
340 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 554 spin_lock(&ring->irq_lock);
341 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0); 555 if (--ring->irq_refcount == 0) {
342 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
343 if (HAS_PCH_SPLIT(dev)) 556 if (HAS_PCH_SPLIT(dev))
344 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 557 ironlake_disable_irq(dev_priv,
558 GT_USER_INTERRUPT |
559 GT_PIPE_NOTIFY);
345 else 560 else
346 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 561 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
347 } 562 }
348 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 563 spin_unlock(&ring->irq_lock);
349} 564}
350 565
351void intel_ring_setup_status_page(struct drm_device *dev, 566void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
352 struct intel_ring_buffer *ring)
353{ 567{
354 drm_i915_private_t *dev_priv = dev->dev_private; 568 drm_i915_private_t *dev_priv = ring->dev->dev_private;
355 if (IS_GEN6(dev)) { 569 u32 mmio = IS_GEN6(ring->dev) ?
356 I915_WRITE(RING_HWS_PGA_GEN6(ring->mmio_base), 570 RING_HWS_PGA_GEN6(ring->mmio_base) :
357 ring->status_page.gfx_addr); 571 RING_HWS_PGA(ring->mmio_base);
358 I915_READ(RING_HWS_PGA_GEN6(ring->mmio_base)); /* posting read */ 572 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
359 } else { 573 POSTING_READ(mmio);
360 I915_WRITE(RING_HWS_PGA(ring->mmio_base),
361 ring->status_page.gfx_addr);
362 I915_READ(RING_HWS_PGA(ring->mmio_base)); /* posting read */
363 }
364
365} 574}
366 575
367static void 576static int
368bsd_ring_flush(struct drm_device *dev, 577bsd_ring_flush(struct intel_ring_buffer *ring,
369 struct intel_ring_buffer *ring, 578 u32 invalidate_domains,
370 u32 invalidate_domains, 579 u32 flush_domains)
371 u32 flush_domains)
372{ 580{
373 intel_ring_begin(dev, ring, 2); 581 int ret;
374 intel_ring_emit(dev, ring, MI_FLUSH);
375 intel_ring_emit(dev, ring, MI_NOOP);
376 intel_ring_advance(dev, ring);
377}
378 582
379static int init_bsd_ring(struct drm_device *dev, 583 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
380 struct intel_ring_buffer *ring) 584 return 0;
381{ 585
382 return init_ring_common(dev, ring); 586 ret = intel_ring_begin(ring, 2);
587 if (ret)
588 return ret;
589
590 intel_ring_emit(ring, MI_FLUSH);
591 intel_ring_emit(ring, MI_NOOP);
592 intel_ring_advance(ring);
593 return 0;
383} 594}
384 595
385static u32 596static int
386ring_add_request(struct drm_device *dev, 597ring_add_request(struct intel_ring_buffer *ring,
387 struct intel_ring_buffer *ring, 598 u32 *result)
388 u32 flush_domains)
389{ 599{
390 u32 seqno; 600 u32 seqno;
601 int ret;
602
603 ret = intel_ring_begin(ring, 4);
604 if (ret)
605 return ret;
391 606
392 seqno = i915_gem_get_seqno(dev); 607 seqno = i915_gem_get_seqno(ring->dev);
393 608
394 intel_ring_begin(dev, ring, 4); 609 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
395 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); 610 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
396 intel_ring_emit(dev, ring, 611 intel_ring_emit(ring, seqno);
397 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 612 intel_ring_emit(ring, MI_USER_INTERRUPT);
398 intel_ring_emit(dev, ring, seqno); 613 intel_ring_advance(ring);
399 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
400 intel_ring_advance(dev, ring);
401 614
402 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 615 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
616 *result = seqno;
617 return 0;
618}
403 619
404 return seqno; 620static bool
621ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
622{
623 struct drm_device *dev = ring->dev;
624 drm_i915_private_t *dev_priv = dev->dev_private;
625
626 if (!dev->irq_enabled)
627 return false;
628
629 spin_lock(&ring->irq_lock);
630 if (ring->irq_refcount++ == 0)
631 ironlake_enable_irq(dev_priv, flag);
632 spin_unlock(&ring->irq_lock);
633
634 return true;
405} 635}
406 636
407static void 637static void
408bsd_ring_get_user_irq(struct drm_device *dev, 638ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
409 struct intel_ring_buffer *ring) 639{
640 struct drm_device *dev = ring->dev;
641 drm_i915_private_t *dev_priv = dev->dev_private;
642
643 spin_lock(&ring->irq_lock);
644 if (--ring->irq_refcount == 0)
645 ironlake_disable_irq(dev_priv, flag);
646 spin_unlock(&ring->irq_lock);
647}
648
649static bool
650gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
410{ 651{
411 /* do nothing */ 652 struct drm_device *dev = ring->dev;
653 drm_i915_private_t *dev_priv = dev->dev_private;
654
655 if (!dev->irq_enabled)
656 return false;
657
658 spin_lock(&ring->irq_lock);
659 if (ring->irq_refcount++ == 0) {
660 ring->irq_mask &= ~rflag;
661 I915_WRITE_IMR(ring, ring->irq_mask);
662 ironlake_enable_irq(dev_priv, gflag);
663 }
664 spin_unlock(&ring->irq_lock);
665
666 return true;
412} 667}
668
413static void 669static void
414bsd_ring_put_user_irq(struct drm_device *dev, 670gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
415 struct intel_ring_buffer *ring)
416{ 671{
417 /* do nothing */ 672 struct drm_device *dev = ring->dev;
673 drm_i915_private_t *dev_priv = dev->dev_private;
674
675 spin_lock(&ring->irq_lock);
676 if (--ring->irq_refcount == 0) {
677 ring->irq_mask |= rflag;
678 I915_WRITE_IMR(ring, ring->irq_mask);
679 ironlake_disable_irq(dev_priv, gflag);
680 }
681 spin_unlock(&ring->irq_lock);
418} 682}
419 683
420static u32 684static bool
421ring_status_page_get_seqno(struct drm_device *dev, 685bsd_ring_get_irq(struct intel_ring_buffer *ring)
422 struct intel_ring_buffer *ring)
423{ 686{
424 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 687 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
688}
689static void
690bsd_ring_put_irq(struct intel_ring_buffer *ring)
691{
692 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
425} 693}
426 694
427static int 695static int
428ring_dispatch_gem_execbuffer(struct drm_device *dev, 696ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
429 struct intel_ring_buffer *ring, 697{
430 struct drm_i915_gem_execbuffer2 *exec, 698 int ret;
431 struct drm_clip_rect *cliprects, 699
432 uint64_t exec_offset) 700 ret = intel_ring_begin(ring, 2);
433{ 701 if (ret)
434 uint32_t exec_start; 702 return ret;
435 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 703
436 intel_ring_begin(dev, ring, 2); 704 intel_ring_emit(ring,
437 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START | 705 MI_BATCH_BUFFER_START | (2 << 6) |
438 (2 << 6) | MI_BATCH_NON_SECURE_I965); 706 MI_BATCH_NON_SECURE_I965);
439 intel_ring_emit(dev, ring, exec_start); 707 intel_ring_emit(ring, offset);
440 intel_ring_advance(dev, ring); 708 intel_ring_advance(ring);
709
441 return 0; 710 return 0;
442} 711}
443 712
444static int 713static int
445render_ring_dispatch_gem_execbuffer(struct drm_device *dev, 714render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
446 struct intel_ring_buffer *ring, 715 u32 offset, u32 len)
447 struct drm_i915_gem_execbuffer2 *exec,
448 struct drm_clip_rect *cliprects,
449 uint64_t exec_offset)
450{ 716{
717 struct drm_device *dev = ring->dev;
451 drm_i915_private_t *dev_priv = dev->dev_private; 718 drm_i915_private_t *dev_priv = dev->dev_private;
452 int nbox = exec->num_cliprects; 719 int ret;
453 int i = 0, count;
454 uint32_t exec_start, exec_len;
455 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
456 exec_len = (uint32_t) exec->batch_len;
457 720
458 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1); 721 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
459 722
460 count = nbox ? nbox : 1; 723 if (IS_I830(dev) || IS_845G(dev)) {
724 ret = intel_ring_begin(ring, 4);
725 if (ret)
726 return ret;
461 727
462 for (i = 0; i < count; i++) { 728 intel_ring_emit(ring, MI_BATCH_BUFFER);
463 if (i < nbox) { 729 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
464 int ret = i915_emit_box(dev, cliprects, i, 730 intel_ring_emit(ring, offset + len - 8);
465 exec->DR1, exec->DR4); 731 intel_ring_emit(ring, 0);
466 if (ret) 732 } else {
467 return ret; 733 ret = intel_ring_begin(ring, 2);
468 } 734 if (ret)
735 return ret;
469 736
470 if (IS_I830(dev) || IS_845G(dev)) { 737 if (INTEL_INFO(dev)->gen >= 4) {
471 intel_ring_begin(dev, ring, 4); 738 intel_ring_emit(ring,
472 intel_ring_emit(dev, ring, MI_BATCH_BUFFER); 739 MI_BATCH_BUFFER_START | (2 << 6) |
473 intel_ring_emit(dev, ring, 740 MI_BATCH_NON_SECURE_I965);
474 exec_start | MI_BATCH_NON_SECURE); 741 intel_ring_emit(ring, offset);
475 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
476 intel_ring_emit(dev, ring, 0);
477 } else { 742 } else {
478 intel_ring_begin(dev, ring, 2); 743 intel_ring_emit(ring,
479 if (INTEL_INFO(dev)->gen >= 4) { 744 MI_BATCH_BUFFER_START | (2 << 6));
480 intel_ring_emit(dev, ring, 745 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
481 MI_BATCH_BUFFER_START | (2 << 6)
482 | MI_BATCH_NON_SECURE_I965);
483 intel_ring_emit(dev, ring, exec_start);
484 } else {
485 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
486 | (2 << 6));
487 intel_ring_emit(dev, ring, exec_start |
488 MI_BATCH_NON_SECURE);
489 }
490 } 746 }
491 intel_ring_advance(dev, ring);
492 } 747 }
493 748 intel_ring_advance(ring);
494 if (IS_G4X(dev) || IS_GEN5(dev)) {
495 intel_ring_begin(dev, ring, 2);
496 intel_ring_emit(dev, ring, MI_FLUSH |
497 MI_NO_WRITE_FLUSH |
498 MI_INVALIDATE_ISP );
499 intel_ring_emit(dev, ring, MI_NOOP);
500 intel_ring_advance(dev, ring);
501 }
502 /* XXX breadcrumb */
503 749
504 return 0; 750 return 0;
505} 751}
506 752
507static void cleanup_status_page(struct drm_device *dev, 753static void cleanup_status_page(struct intel_ring_buffer *ring)
508 struct intel_ring_buffer *ring)
509{ 754{
510 drm_i915_private_t *dev_priv = dev->dev_private; 755 drm_i915_private_t *dev_priv = ring->dev->dev_private;
511 struct drm_gem_object *obj; 756 struct drm_i915_gem_object *obj;
512 struct drm_i915_gem_object *obj_priv;
513 757
514 obj = ring->status_page.obj; 758 obj = ring->status_page.obj;
515 if (obj == NULL) 759 if (obj == NULL)
516 return; 760 return;
517 obj_priv = to_intel_bo(obj);
518 761
519 kunmap(obj_priv->pages[0]); 762 kunmap(obj->pages[0]);
520 i915_gem_object_unpin(obj); 763 i915_gem_object_unpin(obj);
521 drm_gem_object_unreference(obj); 764 drm_gem_object_unreference(&obj->base);
522 ring->status_page.obj = NULL; 765 ring->status_page.obj = NULL;
523 766
524 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 767 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
525} 768}
526 769
527static int init_status_page(struct drm_device *dev, 770static int init_status_page(struct intel_ring_buffer *ring)
528 struct intel_ring_buffer *ring)
529{ 771{
772 struct drm_device *dev = ring->dev;
530 drm_i915_private_t *dev_priv = dev->dev_private; 773 drm_i915_private_t *dev_priv = dev->dev_private;
531 struct drm_gem_object *obj; 774 struct drm_i915_gem_object *obj;
532 struct drm_i915_gem_object *obj_priv;
533 int ret; 775 int ret;
534 776
535 obj = i915_gem_alloc_object(dev, 4096); 777 obj = i915_gem_alloc_object(dev, 4096);
@@ -538,16 +780,15 @@ static int init_status_page(struct drm_device *dev,
538 ret = -ENOMEM; 780 ret = -ENOMEM;
539 goto err; 781 goto err;
540 } 782 }
541 obj_priv = to_intel_bo(obj); 783 obj->agp_type = AGP_USER_CACHED_MEMORY;
542 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
543 784
544 ret = i915_gem_object_pin(obj, 4096); 785 ret = i915_gem_object_pin(obj, 4096, true);
545 if (ret != 0) { 786 if (ret != 0) {
546 goto err_unref; 787 goto err_unref;
547 } 788 }
548 789
549 ring->status_page.gfx_addr = obj_priv->gtt_offset; 790 ring->status_page.gfx_addr = obj->gtt_offset;
550 ring->status_page.page_addr = kmap(obj_priv->pages[0]); 791 ring->status_page.page_addr = kmap(obj->pages[0]);
551 if (ring->status_page.page_addr == NULL) { 792 if (ring->status_page.page_addr == NULL) {
552 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 793 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
553 goto err_unpin; 794 goto err_unpin;
@@ -555,7 +796,7 @@ static int init_status_page(struct drm_device *dev,
555 ring->status_page.obj = obj; 796 ring->status_page.obj = obj;
556 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 797 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
557 798
558 intel_ring_setup_status_page(dev, ring); 799 intel_ring_setup_status_page(ring);
559 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", 800 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
560 ring->name, ring->status_page.gfx_addr); 801 ring->name, ring->status_page.gfx_addr);
561 802
@@ -564,7 +805,7 @@ static int init_status_page(struct drm_device *dev,
564err_unpin: 805err_unpin:
565 i915_gem_object_unpin(obj); 806 i915_gem_object_unpin(obj);
566err_unref: 807err_unref:
567 drm_gem_object_unreference(obj); 808 drm_gem_object_unreference(&obj->base);
568err: 809err:
569 return ret; 810 return ret;
570} 811}
@@ -572,9 +813,7 @@ err:
572int intel_init_ring_buffer(struct drm_device *dev, 813int intel_init_ring_buffer(struct drm_device *dev,
573 struct intel_ring_buffer *ring) 814 struct intel_ring_buffer *ring)
574{ 815{
575 struct drm_i915_private *dev_priv = dev->dev_private; 816 struct drm_i915_gem_object *obj;
576 struct drm_i915_gem_object *obj_priv;
577 struct drm_gem_object *obj;
578 int ret; 817 int ret;
579 818
580 ring->dev = dev; 819 ring->dev = dev;
@@ -582,8 +821,11 @@ int intel_init_ring_buffer(struct drm_device *dev,
582 INIT_LIST_HEAD(&ring->request_list); 821 INIT_LIST_HEAD(&ring->request_list);
583 INIT_LIST_HEAD(&ring->gpu_write_list); 822 INIT_LIST_HEAD(&ring->gpu_write_list);
584 823
824 spin_lock_init(&ring->irq_lock);
825 ring->irq_mask = ~0;
826
585 if (I915_NEED_GFX_HWS(dev)) { 827 if (I915_NEED_GFX_HWS(dev)) {
586 ret = init_status_page(dev, ring); 828 ret = init_status_page(ring);
587 if (ret) 829 if (ret)
588 return ret; 830 return ret;
589 } 831 }
@@ -595,15 +837,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
595 goto err_hws; 837 goto err_hws;
596 } 838 }
597 839
598 ring->gem_object = obj; 840 ring->obj = obj;
599 841
600 ret = i915_gem_object_pin(obj, PAGE_SIZE); 842 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
601 if (ret) 843 if (ret)
602 goto err_unref; 844 goto err_unref;
603 845
604 obj_priv = to_intel_bo(obj);
605 ring->map.size = ring->size; 846 ring->map.size = ring->size;
606 ring->map.offset = dev->agp->base + obj_priv->gtt_offset; 847 ring->map.offset = dev->agp->base + obj->gtt_offset;
607 ring->map.type = 0; 848 ring->map.type = 0;
608 ring->map.flags = 0; 849 ring->map.flags = 0;
609 ring->map.mtrr = 0; 850 ring->map.mtrr = 0;
@@ -616,56 +857,64 @@ int intel_init_ring_buffer(struct drm_device *dev,
616 } 857 }
617 858
618 ring->virtual_start = ring->map.handle; 859 ring->virtual_start = ring->map.handle;
619 ret = ring->init(dev, ring); 860 ret = ring->init(ring);
620 if (ret) 861 if (ret)
621 goto err_unmap; 862 goto err_unmap;
622 863
623 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 864 /* Workaround an erratum on the i830 which causes a hang if
624 i915_kernel_lost_context(dev); 865 * the TAIL pointer points to within the last 2 cachelines
625 else { 866 * of the buffer.
626 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; 867 */
627 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; 868 ring->effective_size = ring->size;
628 ring->space = ring->head - (ring->tail + 8); 869 if (IS_I830(ring->dev))
629 if (ring->space < 0) 870 ring->effective_size -= 128;
630 ring->space += ring->size; 871
631 } 872 return 0;
632 return ret;
633 873
634err_unmap: 874err_unmap:
635 drm_core_ioremapfree(&ring->map, dev); 875 drm_core_ioremapfree(&ring->map, dev);
636err_unpin: 876err_unpin:
637 i915_gem_object_unpin(obj); 877 i915_gem_object_unpin(obj);
638err_unref: 878err_unref:
639 drm_gem_object_unreference(obj); 879 drm_gem_object_unreference(&obj->base);
640 ring->gem_object = NULL; 880 ring->obj = NULL;
641err_hws: 881err_hws:
642 cleanup_status_page(dev, ring); 882 cleanup_status_page(ring);
643 return ret; 883 return ret;
644} 884}
645 885
646void intel_cleanup_ring_buffer(struct drm_device *dev, 886void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
647 struct intel_ring_buffer *ring)
648{ 887{
649 if (ring->gem_object == NULL) 888 struct drm_i915_private *dev_priv;
889 int ret;
890
891 if (ring->obj == NULL)
650 return; 892 return;
651 893
652 drm_core_ioremapfree(&ring->map, dev); 894 /* Disable the ring buffer. The ring must be idle at this point */
895 dev_priv = ring->dev->dev_private;
896 ret = intel_wait_ring_buffer(ring, ring->size - 8);
897 I915_WRITE_CTL(ring, 0);
898
899 drm_core_ioremapfree(&ring->map, ring->dev);
653 900
654 i915_gem_object_unpin(ring->gem_object); 901 i915_gem_object_unpin(ring->obj);
655 drm_gem_object_unreference(ring->gem_object); 902 drm_gem_object_unreference(&ring->obj->base);
656 ring->gem_object = NULL; 903 ring->obj = NULL;
657 cleanup_status_page(dev, ring); 904
905 if (ring->cleanup)
906 ring->cleanup(ring);
907
908 cleanup_status_page(ring);
658} 909}
659 910
660static int intel_wrap_ring_buffer(struct drm_device *dev, 911static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
661 struct intel_ring_buffer *ring)
662{ 912{
663 unsigned int *virt; 913 unsigned int *virt;
664 int rem; 914 int rem = ring->size - ring->tail;
665 rem = ring->size - ring->tail;
666 915
667 if (ring->space < rem) { 916 if (ring->space < rem) {
668 int ret = intel_wait_ring_buffer(dev, ring, rem); 917 int ret = intel_wait_ring_buffer(ring, rem);
669 if (ret) 918 if (ret)
670 return ret; 919 return ret;
671 } 920 }
@@ -678,26 +927,36 @@ static int intel_wrap_ring_buffer(struct drm_device *dev,
678 } 927 }
679 928
680 ring->tail = 0; 929 ring->tail = 0;
681 ring->space = ring->head - 8; 930 ring->space = ring_space(ring);
682 931
683 return 0; 932 return 0;
684} 933}
685 934
686int intel_wait_ring_buffer(struct drm_device *dev, 935int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
687 struct intel_ring_buffer *ring, int n)
688{ 936{
937 struct drm_device *dev = ring->dev;
938 struct drm_i915_private *dev_priv = dev->dev_private;
689 unsigned long end; 939 unsigned long end;
690 drm_i915_private_t *dev_priv = dev->dev_private; 940 u32 head;
941
942 /* If the reported head position has wrapped or hasn't advanced,
943 * fallback to the slow and accurate path.
944 */
945 head = intel_read_status_page(ring, 4);
946 if (head > ring->head) {
947 ring->head = head;
948 ring->space = ring_space(ring);
949 if (ring->space >= n)
950 return 0;
951 }
691 952
692 trace_i915_ring_wait_begin (dev); 953 trace_i915_ring_wait_begin (dev);
693 end = jiffies + 3 * HZ; 954 end = jiffies + 3 * HZ;
694 do { 955 do {
695 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; 956 ring->head = I915_READ_HEAD(ring);
696 ring->space = ring->head - (ring->tail + 8); 957 ring->space = ring_space(ring);
697 if (ring->space < 0)
698 ring->space += ring->size;
699 if (ring->space >= n) { 958 if (ring->space >= n) {
700 trace_i915_ring_wait_end (dev); 959 trace_i915_ring_wait_end(dev);
701 return 0; 960 return 0;
702 } 961 }
703 962
@@ -708,29 +967,39 @@ int intel_wait_ring_buffer(struct drm_device *dev,
708 } 967 }
709 968
710 msleep(1); 969 msleep(1);
970 if (atomic_read(&dev_priv->mm.wedged))
971 return -EAGAIN;
711 } while (!time_after(jiffies, end)); 972 } while (!time_after(jiffies, end));
712 trace_i915_ring_wait_end (dev); 973 trace_i915_ring_wait_end (dev);
713 return -EBUSY; 974 return -EBUSY;
714} 975}
715 976
716void intel_ring_begin(struct drm_device *dev, 977int intel_ring_begin(struct intel_ring_buffer *ring,
717 struct intel_ring_buffer *ring, 978 int num_dwords)
718 int num_dwords)
719{ 979{
720 int n = 4*num_dwords; 980 int n = 4*num_dwords;
721 if (unlikely(ring->tail + n > ring->size)) 981 int ret;
722 intel_wrap_ring_buffer(dev, ring); 982
723 if (unlikely(ring->space < n)) 983 if (unlikely(ring->tail + n > ring->effective_size)) {
724 intel_wait_ring_buffer(dev, ring, n); 984 ret = intel_wrap_ring_buffer(ring);
985 if (unlikely(ret))
986 return ret;
987 }
988
989 if (unlikely(ring->space < n)) {
990 ret = intel_wait_ring_buffer(ring, n);
991 if (unlikely(ret))
992 return ret;
993 }
725 994
726 ring->space -= n; 995 ring->space -= n;
996 return 0;
727} 997}
728 998
729void intel_ring_advance(struct drm_device *dev, 999void intel_ring_advance(struct intel_ring_buffer *ring)
730 struct intel_ring_buffer *ring)
731{ 1000{
732 ring->tail &= ring->size - 1; 1001 ring->tail &= ring->size - 1;
733 ring->write_tail(dev, ring, ring->tail); 1002 ring->write_tail(ring, ring->tail);
734} 1003}
735 1004
736static const struct intel_ring_buffer render_ring = { 1005static const struct intel_ring_buffer render_ring = {
@@ -742,10 +1011,11 @@ static const struct intel_ring_buffer render_ring = {
742 .write_tail = ring_write_tail, 1011 .write_tail = ring_write_tail,
743 .flush = render_ring_flush, 1012 .flush = render_ring_flush,
744 .add_request = render_ring_add_request, 1013 .add_request = render_ring_add_request,
745 .get_seqno = render_ring_get_seqno, 1014 .get_seqno = ring_get_seqno,
746 .user_irq_get = render_ring_get_user_irq, 1015 .irq_get = render_ring_get_irq,
747 .user_irq_put = render_ring_put_user_irq, 1016 .irq_put = render_ring_put_irq,
748 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer, 1017 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
1018 .cleanup = render_ring_cleanup,
749}; 1019};
750 1020
751/* ring buffer for bit-stream decoder */ 1021/* ring buffer for bit-stream decoder */
@@ -755,22 +1025,21 @@ static const struct intel_ring_buffer bsd_ring = {
755 .id = RING_BSD, 1025 .id = RING_BSD,
756 .mmio_base = BSD_RING_BASE, 1026 .mmio_base = BSD_RING_BASE,
757 .size = 32 * PAGE_SIZE, 1027 .size = 32 * PAGE_SIZE,
758 .init = init_bsd_ring, 1028 .init = init_ring_common,
759 .write_tail = ring_write_tail, 1029 .write_tail = ring_write_tail,
760 .flush = bsd_ring_flush, 1030 .flush = bsd_ring_flush,
761 .add_request = ring_add_request, 1031 .add_request = ring_add_request,
762 .get_seqno = ring_status_page_get_seqno, 1032 .get_seqno = ring_get_seqno,
763 .user_irq_get = bsd_ring_get_user_irq, 1033 .irq_get = bsd_ring_get_irq,
764 .user_irq_put = bsd_ring_put_user_irq, 1034 .irq_put = bsd_ring_put_irq,
765 .dispatch_gem_execbuffer = ring_dispatch_gem_execbuffer, 1035 .dispatch_execbuffer = ring_dispatch_execbuffer,
766}; 1036};
767 1037
768 1038
769static void gen6_bsd_ring_write_tail(struct drm_device *dev, 1039static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
770 struct intel_ring_buffer *ring,
771 u32 value) 1040 u32 value)
772{ 1041{
773 drm_i915_private_t *dev_priv = dev->dev_private; 1042 drm_i915_private_t *dev_priv = ring->dev->dev_private;
774 1043
775 /* Every tail move must follow the sequence below */ 1044 /* Every tail move must follow the sequence below */
776 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, 1045 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
@@ -789,69 +1058,206 @@ static void gen6_bsd_ring_write_tail(struct drm_device *dev,
789 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); 1058 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
790} 1059}
791 1060
792static void gen6_ring_flush(struct drm_device *dev, 1061static int gen6_ring_flush(struct intel_ring_buffer *ring,
793 struct intel_ring_buffer *ring, 1062 u32 invalidate_domains,
794 u32 invalidate_domains, 1063 u32 flush_domains)
795 u32 flush_domains)
796{ 1064{
797 intel_ring_begin(dev, ring, 4); 1065 int ret;
798 intel_ring_emit(dev, ring, MI_FLUSH_DW); 1066
799 intel_ring_emit(dev, ring, 0); 1067 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
800 intel_ring_emit(dev, ring, 0); 1068 return 0;
801 intel_ring_emit(dev, ring, 0); 1069
802 intel_ring_advance(dev, ring); 1070 ret = intel_ring_begin(ring, 4);
1071 if (ret)
1072 return ret;
1073
1074 intel_ring_emit(ring, MI_FLUSH_DW);
1075 intel_ring_emit(ring, 0);
1076 intel_ring_emit(ring, 0);
1077 intel_ring_emit(ring, 0);
1078 intel_ring_advance(ring);
1079 return 0;
803} 1080}
804 1081
805static int 1082static int
806gen6_ring_dispatch_gem_execbuffer(struct drm_device *dev, 1083gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
807 struct intel_ring_buffer *ring, 1084 u32 offset, u32 len)
808 struct drm_i915_gem_execbuffer2 *exec,
809 struct drm_clip_rect *cliprects,
810 uint64_t exec_offset)
811{ 1085{
812 uint32_t exec_start; 1086 int ret;
813 1087
814 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 1088 ret = intel_ring_begin(ring, 2);
1089 if (ret)
1090 return ret;
815 1091
816 intel_ring_begin(dev, ring, 2); 1092 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
817 intel_ring_emit(dev, ring,
818 MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
819 /* bit0-7 is the length on GEN6+ */ 1093 /* bit0-7 is the length on GEN6+ */
820 intel_ring_emit(dev, ring, exec_start); 1094 intel_ring_emit(ring, offset);
821 intel_ring_advance(dev, ring); 1095 intel_ring_advance(ring);
822 1096
823 return 0; 1097 return 0;
824} 1098}
825 1099
1100static bool
1101gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1102{
1103 return gen6_ring_get_irq(ring,
1104 GT_USER_INTERRUPT,
1105 GEN6_RENDER_USER_INTERRUPT);
1106}
1107
1108static void
1109gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1110{
1111 return gen6_ring_put_irq(ring,
1112 GT_USER_INTERRUPT,
1113 GEN6_RENDER_USER_INTERRUPT);
1114}
1115
1116static bool
1117gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1118{
1119 return gen6_ring_get_irq(ring,
1120 GT_GEN6_BSD_USER_INTERRUPT,
1121 GEN6_BSD_USER_INTERRUPT);
1122}
1123
1124static void
1125gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1126{
1127 return gen6_ring_put_irq(ring,
1128 GT_GEN6_BSD_USER_INTERRUPT,
1129 GEN6_BSD_USER_INTERRUPT);
1130}
1131
826/* ring buffer for Video Codec for Gen6+ */ 1132/* ring buffer for Video Codec for Gen6+ */
827static const struct intel_ring_buffer gen6_bsd_ring = { 1133static const struct intel_ring_buffer gen6_bsd_ring = {
828 .name = "gen6 bsd ring", 1134 .name = "gen6 bsd ring",
829 .id = RING_BSD, 1135 .id = RING_BSD,
830 .mmio_base = GEN6_BSD_RING_BASE, 1136 .mmio_base = GEN6_BSD_RING_BASE,
831 .size = 32 * PAGE_SIZE, 1137 .size = 32 * PAGE_SIZE,
832 .init = init_bsd_ring, 1138 .init = init_ring_common,
833 .write_tail = gen6_bsd_ring_write_tail, 1139 .write_tail = gen6_bsd_ring_write_tail,
834 .flush = gen6_ring_flush, 1140 .flush = gen6_ring_flush,
835 .add_request = ring_add_request, 1141 .add_request = gen6_add_request,
836 .get_seqno = ring_status_page_get_seqno, 1142 .get_seqno = ring_get_seqno,
837 .user_irq_get = bsd_ring_get_user_irq, 1143 .irq_get = gen6_bsd_ring_get_irq,
838 .user_irq_put = bsd_ring_put_user_irq, 1144 .irq_put = gen6_bsd_ring_put_irq,
839 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 1145 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
840}; 1146};
841 1147
842/* Blitter support (SandyBridge+) */ 1148/* Blitter support (SandyBridge+) */
843 1149
844static void 1150static bool
845blt_ring_get_user_irq(struct drm_device *dev, 1151blt_ring_get_irq(struct intel_ring_buffer *ring)
846 struct intel_ring_buffer *ring)
847{ 1152{
848 /* do nothing */ 1153 return gen6_ring_get_irq(ring,
1154 GT_BLT_USER_INTERRUPT,
1155 GEN6_BLITTER_USER_INTERRUPT);
849} 1156}
1157
850static void 1158static void
851blt_ring_put_user_irq(struct drm_device *dev, 1159blt_ring_put_irq(struct intel_ring_buffer *ring)
852 struct intel_ring_buffer *ring)
853{ 1160{
854 /* do nothing */ 1161 gen6_ring_put_irq(ring,
1162 GT_BLT_USER_INTERRUPT,
1163 GEN6_BLITTER_USER_INTERRUPT);
1164}
1165
1166
1167/* Workaround for some stepping of SNB,
1168 * each time when BLT engine ring tail moved,
1169 * the first command in the ring to be parsed
1170 * should be MI_BATCH_BUFFER_START
1171 */
1172#define NEED_BLT_WORKAROUND(dev) \
1173 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1174
1175static inline struct drm_i915_gem_object *
1176to_blt_workaround(struct intel_ring_buffer *ring)
1177{
1178 return ring->private;
1179}
1180
1181static int blt_ring_init(struct intel_ring_buffer *ring)
1182{
1183 if (NEED_BLT_WORKAROUND(ring->dev)) {
1184 struct drm_i915_gem_object *obj;
1185 u32 *ptr;
1186 int ret;
1187
1188 obj = i915_gem_alloc_object(ring->dev, 4096);
1189 if (obj == NULL)
1190 return -ENOMEM;
1191
1192 ret = i915_gem_object_pin(obj, 4096, true);
1193 if (ret) {
1194 drm_gem_object_unreference(&obj->base);
1195 return ret;
1196 }
1197
1198 ptr = kmap(obj->pages[0]);
1199 *ptr++ = MI_BATCH_BUFFER_END;
1200 *ptr++ = MI_NOOP;
1201 kunmap(obj->pages[0]);
1202
1203 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1204 if (ret) {
1205 i915_gem_object_unpin(obj);
1206 drm_gem_object_unreference(&obj->base);
1207 return ret;
1208 }
1209
1210 ring->private = obj;
1211 }
1212
1213 return init_ring_common(ring);
1214}
1215
1216static int blt_ring_begin(struct intel_ring_buffer *ring,
1217 int num_dwords)
1218{
1219 if (ring->private) {
1220 int ret = intel_ring_begin(ring, num_dwords+2);
1221 if (ret)
1222 return ret;
1223
1224 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1225 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1226
1227 return 0;
1228 } else
1229 return intel_ring_begin(ring, 4);
1230}
1231
1232static int blt_ring_flush(struct intel_ring_buffer *ring,
1233 u32 invalidate_domains,
1234 u32 flush_domains)
1235{
1236 int ret;
1237
1238 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
1239 return 0;
1240
1241 ret = blt_ring_begin(ring, 4);
1242 if (ret)
1243 return ret;
1244
1245 intel_ring_emit(ring, MI_FLUSH_DW);
1246 intel_ring_emit(ring, 0);
1247 intel_ring_emit(ring, 0);
1248 intel_ring_emit(ring, 0);
1249 intel_ring_advance(ring);
1250 return 0;
1251}
1252
1253static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1254{
1255 if (!ring->private)
1256 return;
1257
1258 i915_gem_object_unpin(ring->private);
1259 drm_gem_object_unreference(ring->private);
1260 ring->private = NULL;
855} 1261}
856 1262
857static const struct intel_ring_buffer gen6_blt_ring = { 1263static const struct intel_ring_buffer gen6_blt_ring = {
@@ -859,49 +1265,101 @@ static const struct intel_ring_buffer gen6_blt_ring = {
859 .id = RING_BLT, 1265 .id = RING_BLT,
860 .mmio_base = BLT_RING_BASE, 1266 .mmio_base = BLT_RING_BASE,
861 .size = 32 * PAGE_SIZE, 1267 .size = 32 * PAGE_SIZE,
862 .init = init_ring_common, 1268 .init = blt_ring_init,
863 .write_tail = ring_write_tail, 1269 .write_tail = ring_write_tail,
864 .flush = gen6_ring_flush, 1270 .flush = blt_ring_flush,
865 .add_request = ring_add_request, 1271 .add_request = gen6_add_request,
866 .get_seqno = ring_status_page_get_seqno, 1272 .get_seqno = ring_get_seqno,
867 .user_irq_get = blt_ring_get_user_irq, 1273 .irq_get = blt_ring_get_irq,
868 .user_irq_put = blt_ring_put_user_irq, 1274 .irq_put = blt_ring_put_irq,
869 .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, 1275 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1276 .cleanup = blt_ring_cleanup,
870}; 1277};
871 1278
872int intel_init_render_ring_buffer(struct drm_device *dev) 1279int intel_init_render_ring_buffer(struct drm_device *dev)
873{ 1280{
874 drm_i915_private_t *dev_priv = dev->dev_private; 1281 drm_i915_private_t *dev_priv = dev->dev_private;
875 1282 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
876 dev_priv->render_ring = render_ring; 1283
1284 *ring = render_ring;
1285 if (INTEL_INFO(dev)->gen >= 6) {
1286 ring->add_request = gen6_add_request;
1287 ring->irq_get = gen6_render_ring_get_irq;
1288 ring->irq_put = gen6_render_ring_put_irq;
1289 } else if (IS_GEN5(dev)) {
1290 ring->add_request = pc_render_add_request;
1291 ring->get_seqno = pc_render_get_seqno;
1292 }
877 1293
878 if (!I915_NEED_GFX_HWS(dev)) { 1294 if (!I915_NEED_GFX_HWS(dev)) {
879 dev_priv->render_ring.status_page.page_addr 1295 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
880 = dev_priv->status_page_dmah->vaddr; 1296 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
881 memset(dev_priv->render_ring.status_page.page_addr,
882 0, PAGE_SIZE);
883 } 1297 }
884 1298
885 return intel_init_ring_buffer(dev, &dev_priv->render_ring); 1299 return intel_init_ring_buffer(dev, ring);
1300}
1301
1302int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1303{
1304 drm_i915_private_t *dev_priv = dev->dev_private;
1305 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1306
1307 *ring = render_ring;
1308 if (INTEL_INFO(dev)->gen >= 6) {
1309 ring->add_request = gen6_add_request;
1310 ring->irq_get = gen6_render_ring_get_irq;
1311 ring->irq_put = gen6_render_ring_put_irq;
1312 } else if (IS_GEN5(dev)) {
1313 ring->add_request = pc_render_add_request;
1314 ring->get_seqno = pc_render_get_seqno;
1315 }
1316
1317 ring->dev = dev;
1318 INIT_LIST_HEAD(&ring->active_list);
1319 INIT_LIST_HEAD(&ring->request_list);
1320 INIT_LIST_HEAD(&ring->gpu_write_list);
1321
1322 ring->size = size;
1323 ring->effective_size = ring->size;
1324 if (IS_I830(ring->dev))
1325 ring->effective_size -= 128;
1326
1327 ring->map.offset = start;
1328 ring->map.size = size;
1329 ring->map.type = 0;
1330 ring->map.flags = 0;
1331 ring->map.mtrr = 0;
1332
1333 drm_core_ioremap_wc(&ring->map, dev);
1334 if (ring->map.handle == NULL) {
1335 DRM_ERROR("can not ioremap virtual address for"
1336 " ring buffer\n");
1337 return -ENOMEM;
1338 }
1339
1340 ring->virtual_start = (void __force __iomem *)ring->map.handle;
1341 return 0;
886} 1342}
887 1343
888int intel_init_bsd_ring_buffer(struct drm_device *dev) 1344int intel_init_bsd_ring_buffer(struct drm_device *dev)
889{ 1345{
890 drm_i915_private_t *dev_priv = dev->dev_private; 1346 drm_i915_private_t *dev_priv = dev->dev_private;
1347 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
891 1348
892 if (IS_GEN6(dev)) 1349 if (IS_GEN6(dev))
893 dev_priv->bsd_ring = gen6_bsd_ring; 1350 *ring = gen6_bsd_ring;
894 else 1351 else
895 dev_priv->bsd_ring = bsd_ring; 1352 *ring = bsd_ring;
896 1353
897 return intel_init_ring_buffer(dev, &dev_priv->bsd_ring); 1354 return intel_init_ring_buffer(dev, ring);
898} 1355}
899 1356
900int intel_init_blt_ring_buffer(struct drm_device *dev) 1357int intel_init_blt_ring_buffer(struct drm_device *dev)
901{ 1358{
902 drm_i915_private_t *dev_priv = dev->dev_private; 1359 drm_i915_private_t *dev_priv = dev->dev_private;
1360 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
903 1361
904 dev_priv->blt_ring = gen6_blt_ring; 1362 *ring = gen6_blt_ring;
905 1363
906 return intel_init_ring_buffer(dev, &dev_priv->blt_ring); 1364 return intel_init_ring_buffer(dev, ring);
907} 1365}
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-06 06:09:04 -0500