aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/intel_ringbuffer.c
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
commitc71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch)
treeecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /drivers/gpu/drm/i915/intel_ringbuffer.c
parentea53c912f8a86a8567697115b6a0d8152beee5c8 (diff)
parent6a00f206debf8a5c8899055726ad127dbeeed098 (diff)
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts: litmus/sched_cedf.c
Diffstat (limited to 'drivers/gpu/drm/i915/intel_ringbuffer.c')
-rw-r--r--drivers/gpu/drm/i915/intel_ringbuffer.c1560
1 files changed, 1031 insertions, 529 deletions
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index cb3508f78bc3..95c4b1429935 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -32,6 +32,15 @@
32#include "i915_drv.h" 32#include "i915_drv.h"
33#include "i915_drm.h" 33#include "i915_drm.h"
34#include "i915_trace.h" 34#include "i915_trace.h"
35#include "intel_drv.h"
36
37static inline int ring_space(struct intel_ring_buffer *ring)
38{
39 int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
40 if (space < 0)
41 space += ring->size;
42 return space;
43}
35 44
36static u32 i915_gem_get_seqno(struct drm_device *dev) 45static u32 i915_gem_get_seqno(struct drm_device *dev)
37{ 46{
@@ -47,537 +56,718 @@ static u32 i915_gem_get_seqno(struct drm_device *dev)
47 return seqno; 56 return seqno;
48} 57}
49 58
50static void 59static int
51render_ring_flush(struct drm_device *dev, 60render_ring_flush(struct intel_ring_buffer *ring,
52 struct intel_ring_buffer *ring, 61 u32 invalidate_domains,
53 u32 invalidate_domains, 62 u32 flush_domains)
54 u32 flush_domains)
55{ 63{
56 drm_i915_private_t *dev_priv = dev->dev_private; 64 struct drm_device *dev = ring->dev;
57 u32 cmd; 65 u32 cmd;
66 int ret;
58 67
59#if WATCH_EXEC 68 /*
60 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__, 69 * read/write caches:
61 invalidate_domains, flush_domains); 70 *
62#endif 71 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
63 72 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
64 trace_i915_gem_request_flush(dev, dev_priv->next_seqno, 73 * also flushed at 2d versus 3d pipeline switches.
65 invalidate_domains, flush_domains); 74 *
66 75 * read-only caches:
67 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) { 76 *
77 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
78 * MI_READ_FLUSH is set, and is always flushed on 965.
79 *
80 * I915_GEM_DOMAIN_COMMAND may not exist?
81 *
82 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
83 * invalidated when MI_EXE_FLUSH is set.
84 *
85 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
86 * invalidated with every MI_FLUSH.
87 *
88 * TLBs:
89 *
90 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
91 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
92 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
93 * are flushed at any MI_FLUSH.
94 */
95
96 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
97 if ((invalidate_domains|flush_domains) &
98 I915_GEM_DOMAIN_RENDER)
99 cmd &= ~MI_NO_WRITE_FLUSH;
100 if (INTEL_INFO(dev)->gen < 4) {
68 /* 101 /*
69 * read/write caches: 102 * On the 965, the sampler cache always gets flushed
70 * 103 * and this bit is reserved.
71 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
72 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
73 * also flushed at 2d versus 3d pipeline switches.
74 *
75 * read-only caches:
76 *
77 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
78 * MI_READ_FLUSH is set, and is always flushed on 965.
79 *
80 * I915_GEM_DOMAIN_COMMAND may not exist?
81 *
82 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
83 * invalidated when MI_EXE_FLUSH is set.
84 *
85 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
86 * invalidated with every MI_FLUSH.
87 *
88 * TLBs:
89 *
90 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
91 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
92 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
93 * are flushed at any MI_FLUSH.
94 */ 104 */
95 105 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
96 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH; 106 cmd |= MI_READ_FLUSH;
97 if ((invalidate_domains|flush_domains) &
98 I915_GEM_DOMAIN_RENDER)
99 cmd &= ~MI_NO_WRITE_FLUSH;
100 if (!IS_I965G(dev)) {
101 /*
102 * On the 965, the sampler cache always gets flushed
103 * and this bit is reserved.
104 */
105 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
106 cmd |= MI_READ_FLUSH;
107 }
108 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
109 cmd |= MI_EXE_FLUSH;
110
111#if WATCH_EXEC
112 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
113#endif
114 intel_ring_begin(dev, ring, 2);
115 intel_ring_emit(dev, ring, cmd);
116 intel_ring_emit(dev, ring, MI_NOOP);
117 intel_ring_advance(dev, ring);
118 } 107 }
119} 108 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
109 cmd |= MI_EXE_FLUSH;
120 110
121static unsigned int render_ring_get_head(struct drm_device *dev, 111 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
122 struct intel_ring_buffer *ring) 112 (IS_G4X(dev) || IS_GEN5(dev)))
123{ 113 cmd |= MI_INVALIDATE_ISP;
124 drm_i915_private_t *dev_priv = dev->dev_private; 114
125 return I915_READ(PRB0_HEAD) & HEAD_ADDR; 115 ret = intel_ring_begin(ring, 2);
116 if (ret)
117 return ret;
118
119 intel_ring_emit(ring, cmd);
120 intel_ring_emit(ring, MI_NOOP);
121 intel_ring_advance(ring);
122
123 return 0;
126} 124}
127 125
128static unsigned int render_ring_get_tail(struct drm_device *dev, 126static void ring_write_tail(struct intel_ring_buffer *ring,
129 struct intel_ring_buffer *ring) 127 u32 value)
130{ 128{
131 drm_i915_private_t *dev_priv = dev->dev_private; 129 drm_i915_private_t *dev_priv = ring->dev->dev_private;
132 return I915_READ(PRB0_TAIL) & TAIL_ADDR; 130 I915_WRITE_TAIL(ring, value);
133} 131}
134 132
135static unsigned int render_ring_get_active_head(struct drm_device *dev, 133u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
136 struct intel_ring_buffer *ring)
137{ 134{
138 drm_i915_private_t *dev_priv = dev->dev_private; 135 drm_i915_private_t *dev_priv = ring->dev->dev_private;
139 u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD; 136 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
137 RING_ACTHD(ring->mmio_base) : ACTHD;
140 138
141 return I915_READ(acthd_reg); 139 return I915_READ(acthd_reg);
142} 140}
143 141
144static void render_ring_advance_ring(struct drm_device *dev, 142static int init_ring_common(struct intel_ring_buffer *ring)
145 struct intel_ring_buffer *ring)
146{
147 drm_i915_private_t *dev_priv = dev->dev_private;
148 I915_WRITE(PRB0_TAIL, ring->tail);
149}
150
151static int init_ring_common(struct drm_device *dev,
152 struct intel_ring_buffer *ring)
153{ 143{
144 drm_i915_private_t *dev_priv = ring->dev->dev_private;
145 struct drm_i915_gem_object *obj = ring->obj;
154 u32 head; 146 u32 head;
155 drm_i915_private_t *dev_priv = dev->dev_private;
156 struct drm_i915_gem_object *obj_priv;
157 obj_priv = to_intel_bo(ring->gem_object);
158 147
159 /* Stop the ring if it's running. */ 148 /* Stop the ring if it's running. */
160 I915_WRITE(ring->regs.ctl, 0); 149 I915_WRITE_CTL(ring, 0);
161 I915_WRITE(ring->regs.head, 0); 150 I915_WRITE_HEAD(ring, 0);
162 I915_WRITE(ring->regs.tail, 0); 151 ring->write_tail(ring, 0);
163 152
164 /* Initialize the ring. */ 153 /* Initialize the ring. */
165 I915_WRITE(ring->regs.start, obj_priv->gtt_offset); 154 I915_WRITE_START(ring, obj->gtt_offset);
166 head = ring->get_head(dev, ring); 155 head = I915_READ_HEAD(ring) & HEAD_ADDR;
167 156
168 /* G45 ring initialization fails to reset head to zero */ 157 /* G45 ring initialization fails to reset head to zero */
169 if (head != 0) { 158 if (head != 0) {
170 DRM_ERROR("%s head not reset to zero " 159 DRM_DEBUG_KMS("%s head not reset to zero "
171 "ctl %08x head %08x tail %08x start %08x\n", 160 "ctl %08x head %08x tail %08x start %08x\n",
172 ring->name, 161 ring->name,
173 I915_READ(ring->regs.ctl), 162 I915_READ_CTL(ring),
174 I915_READ(ring->regs.head), 163 I915_READ_HEAD(ring),
175 I915_READ(ring->regs.tail), 164 I915_READ_TAIL(ring),
176 I915_READ(ring->regs.start)); 165 I915_READ_START(ring));
177 166
178 I915_WRITE(ring->regs.head, 0); 167 I915_WRITE_HEAD(ring, 0);
179 168
180 DRM_ERROR("%s head forced to zero " 169 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
181 "ctl %08x head %08x tail %08x start %08x\n", 170 DRM_ERROR("failed to set %s head to zero "
182 ring->name, 171 "ctl %08x head %08x tail %08x start %08x\n",
183 I915_READ(ring->regs.ctl), 172 ring->name,
184 I915_READ(ring->regs.head), 173 I915_READ_CTL(ring),
185 I915_READ(ring->regs.tail), 174 I915_READ_HEAD(ring),
186 I915_READ(ring->regs.start)); 175 I915_READ_TAIL(ring),
176 I915_READ_START(ring));
177 }
187 } 178 }
188 179
189 I915_WRITE(ring->regs.ctl, 180 I915_WRITE_CTL(ring,
190 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES) 181 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
191 | RING_NO_REPORT | RING_VALID); 182 | RING_REPORT_64K | RING_VALID);
192 183
193 head = I915_READ(ring->regs.head) & HEAD_ADDR;
194 /* If the head is still not zero, the ring is dead */ 184 /* If the head is still not zero, the ring is dead */
195 if (head != 0) { 185 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
186 I915_READ_START(ring) != obj->gtt_offset ||
187 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
196 DRM_ERROR("%s initialization failed " 188 DRM_ERROR("%s initialization failed "
197 "ctl %08x head %08x tail %08x start %08x\n", 189 "ctl %08x head %08x tail %08x start %08x\n",
198 ring->name, 190 ring->name,
199 I915_READ(ring->regs.ctl), 191 I915_READ_CTL(ring),
200 I915_READ(ring->regs.head), 192 I915_READ_HEAD(ring),
201 I915_READ(ring->regs.tail), 193 I915_READ_TAIL(ring),
202 I915_READ(ring->regs.start)); 194 I915_READ_START(ring));
203 return -EIO; 195 return -EIO;
204 } 196 }
205 197
206 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 198 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
207 i915_kernel_lost_context(dev); 199 i915_kernel_lost_context(ring->dev);
208 else { 200 else {
209 ring->head = ring->get_head(dev, ring); 201 ring->head = I915_READ_HEAD(ring);
210 ring->tail = ring->get_tail(dev, ring); 202 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
211 ring->space = ring->head - (ring->tail + 8); 203 ring->space = ring_space(ring);
212 if (ring->space < 0)
213 ring->space += ring->size;
214 } 204 }
205
215 return 0; 206 return 0;
216} 207}
217 208
218static int init_render_ring(struct drm_device *dev, 209/*
219 struct intel_ring_buffer *ring) 210 * 965+ support PIPE_CONTROL commands, which provide finer grained control
211 * over cache flushing.
212 */
213struct pipe_control {
214 struct drm_i915_gem_object *obj;
215 volatile u32 *cpu_page;
216 u32 gtt_offset;
217};
218
219static int
220init_pipe_control(struct intel_ring_buffer *ring)
220{ 221{
221 drm_i915_private_t *dev_priv = dev->dev_private; 222 struct pipe_control *pc;
222 int ret = init_ring_common(dev, ring); 223 struct drm_i915_gem_object *obj;
223 int mode; 224 int ret;
225
226 if (ring->private)
227 return 0;
228
229 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
230 if (!pc)
231 return -ENOMEM;
232
233 obj = i915_gem_alloc_object(ring->dev, 4096);
234 if (obj == NULL) {
235 DRM_ERROR("Failed to allocate seqno page\n");
236 ret = -ENOMEM;
237 goto err;
238 }
239 obj->cache_level = I915_CACHE_LLC;
240
241 ret = i915_gem_object_pin(obj, 4096, true);
242 if (ret)
243 goto err_unref;
244
245 pc->gtt_offset = obj->gtt_offset;
246 pc->cpu_page = kmap(obj->pages[0]);
247 if (pc->cpu_page == NULL)
248 goto err_unpin;
249
250 pc->obj = obj;
251 ring->private = pc;
252 return 0;
253
254err_unpin:
255 i915_gem_object_unpin(obj);
256err_unref:
257 drm_gem_object_unreference(&obj->base);
258err:
259 kfree(pc);
260 return ret;
261}
262
263static void
264cleanup_pipe_control(struct intel_ring_buffer *ring)
265{
266 struct pipe_control *pc = ring->private;
267 struct drm_i915_gem_object *obj;
268
269 if (!ring->private)
270 return;
271
272 obj = pc->obj;
273 kunmap(obj->pages[0]);
274 i915_gem_object_unpin(obj);
275 drm_gem_object_unreference(&obj->base);
276
277 kfree(pc);
278 ring->private = NULL;
279}
224 280
225 if (IS_I9XX(dev) && !IS_GEN3(dev)) { 281static int init_render_ring(struct intel_ring_buffer *ring)
226 mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; 282{
227 if (IS_GEN6(dev)) 283 struct drm_device *dev = ring->dev;
284 struct drm_i915_private *dev_priv = dev->dev_private;
285 int ret = init_ring_common(ring);
286
287 if (INTEL_INFO(dev)->gen > 3) {
288 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
289 if (IS_GEN6(dev) || IS_GEN7(dev))
228 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE; 290 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
229 I915_WRITE(MI_MODE, mode); 291 I915_WRITE(MI_MODE, mode);
230 } 292 }
293
294 if (INTEL_INFO(dev)->gen >= 6) {
295 } else if (IS_GEN5(dev)) {
296 ret = init_pipe_control(ring);
297 if (ret)
298 return ret;
299 }
300
231 return ret; 301 return ret;
232} 302}
233 303
234#define PIPE_CONTROL_FLUSH(addr) \ 304static void render_ring_cleanup(struct intel_ring_buffer *ring)
305{
306 if (!ring->private)
307 return;
308
309 cleanup_pipe_control(ring);
310}
311
312static void
313update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
314{
315 struct drm_device *dev = ring->dev;
316 struct drm_i915_private *dev_priv = dev->dev_private;
317 int id;
318
319 /*
320 * cs -> 1 = vcs, 0 = bcs
321 * vcs -> 1 = bcs, 0 = cs,
322 * bcs -> 1 = cs, 0 = vcs.
323 */
324 id = ring - dev_priv->ring;
325 id += 2 - i;
326 id %= 3;
327
328 intel_ring_emit(ring,
329 MI_SEMAPHORE_MBOX |
330 MI_SEMAPHORE_REGISTER |
331 MI_SEMAPHORE_UPDATE);
332 intel_ring_emit(ring, seqno);
333 intel_ring_emit(ring,
334 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
335}
336
337static int
338gen6_add_request(struct intel_ring_buffer *ring,
339 u32 *result)
340{
341 u32 seqno;
342 int ret;
343
344 ret = intel_ring_begin(ring, 10);
345 if (ret)
346 return ret;
347
348 seqno = i915_gem_get_seqno(ring->dev);
349 update_semaphore(ring, 0, seqno);
350 update_semaphore(ring, 1, seqno);
351
352 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
353 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
354 intel_ring_emit(ring, seqno);
355 intel_ring_emit(ring, MI_USER_INTERRUPT);
356 intel_ring_advance(ring);
357
358 *result = seqno;
359 return 0;
360}
361
362int
363intel_ring_sync(struct intel_ring_buffer *ring,
364 struct intel_ring_buffer *to,
365 u32 seqno)
366{
367 int ret;
368
369 ret = intel_ring_begin(ring, 4);
370 if (ret)
371 return ret;
372
373 intel_ring_emit(ring,
374 MI_SEMAPHORE_MBOX |
375 MI_SEMAPHORE_REGISTER |
376 intel_ring_sync_index(ring, to) << 17 |
377 MI_SEMAPHORE_COMPARE);
378 intel_ring_emit(ring, seqno);
379 intel_ring_emit(ring, 0);
380 intel_ring_emit(ring, MI_NOOP);
381 intel_ring_advance(ring);
382
383 return 0;
384}
385
386#define PIPE_CONTROL_FLUSH(ring__, addr__) \
235do { \ 387do { \
236 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ 388 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
237 PIPE_CONTROL_DEPTH_STALL | 2); \ 389 PIPE_CONTROL_DEPTH_STALL | 2); \
238 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ 390 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
239 OUT_RING(0); \ 391 intel_ring_emit(ring__, 0); \
240 OUT_RING(0); \ 392 intel_ring_emit(ring__, 0); \
241} while (0) 393} while (0)
242 394
243/** 395static int
244 * Creates a new sequence number, emitting a write of it to the status page 396pc_render_add_request(struct intel_ring_buffer *ring,
245 * plus an interrupt, which will trigger i915_user_interrupt_handler. 397 u32 *result)
246 *
247 * Must be called with struct_lock held.
248 *
249 * Returned sequence numbers are nonzero on success.
250 */
251static u32
252render_ring_add_request(struct drm_device *dev,
253 struct intel_ring_buffer *ring,
254 struct drm_file *file_priv,
255 u32 flush_domains)
256{ 398{
257 drm_i915_private_t *dev_priv = dev->dev_private; 399 struct drm_device *dev = ring->dev;
258 u32 seqno; 400 u32 seqno = i915_gem_get_seqno(dev);
401 struct pipe_control *pc = ring->private;
402 u32 scratch_addr = pc->gtt_offset + 128;
403 int ret;
259 404
260 seqno = i915_gem_get_seqno(dev); 405 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
261 406 * incoherent with writes to memory, i.e. completely fubar,
262 if (IS_GEN6(dev)) { 407 * so we need to use PIPE_NOTIFY instead.
263 BEGIN_LP_RING(6); 408 *
264 OUT_RING(GFX_OP_PIPE_CONTROL | 3); 409 * However, we also need to workaround the qword write
265 OUT_RING(PIPE_CONTROL_QW_WRITE | 410 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
266 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH | 411 * memory before requesting an interrupt.
267 PIPE_CONTROL_NOTIFY); 412 */
268 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); 413 ret = intel_ring_begin(ring, 32);
269 OUT_RING(seqno); 414 if (ret)
270 OUT_RING(0); 415 return ret;
271 OUT_RING(0); 416
272 ADVANCE_LP_RING(); 417 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
273 } else if (HAS_PIPE_CONTROL(dev)) { 418 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
274 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128; 419 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
420 intel_ring_emit(ring, seqno);
421 intel_ring_emit(ring, 0);
422 PIPE_CONTROL_FLUSH(ring, scratch_addr);
423 scratch_addr += 128; /* write to separate cachelines */
424 PIPE_CONTROL_FLUSH(ring, scratch_addr);
425 scratch_addr += 128;
426 PIPE_CONTROL_FLUSH(ring, scratch_addr);
427 scratch_addr += 128;
428 PIPE_CONTROL_FLUSH(ring, scratch_addr);
429 scratch_addr += 128;
430 PIPE_CONTROL_FLUSH(ring, scratch_addr);
431 scratch_addr += 128;
432 PIPE_CONTROL_FLUSH(ring, scratch_addr);
433 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
434 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
435 PIPE_CONTROL_NOTIFY);
436 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
437 intel_ring_emit(ring, seqno);
438 intel_ring_emit(ring, 0);
439 intel_ring_advance(ring);
440
441 *result = seqno;
442 return 0;
443}
275 444
276 /* 445static int
277 * Workaround qword write incoherence by flushing the 446render_ring_add_request(struct intel_ring_buffer *ring,
278 * PIPE_NOTIFY buffers out to memory before requesting 447 u32 *result)
279 * an interrupt. 448{
280 */ 449 struct drm_device *dev = ring->dev;
281 BEGIN_LP_RING(32); 450 u32 seqno = i915_gem_get_seqno(dev);
282 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | 451 int ret;
283 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
284 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
285 OUT_RING(seqno);
286 OUT_RING(0);
287 PIPE_CONTROL_FLUSH(scratch_addr);
288 scratch_addr += 128; /* write to separate cachelines */
289 PIPE_CONTROL_FLUSH(scratch_addr);
290 scratch_addr += 128;
291 PIPE_CONTROL_FLUSH(scratch_addr);
292 scratch_addr += 128;
293 PIPE_CONTROL_FLUSH(scratch_addr);
294 scratch_addr += 128;
295 PIPE_CONTROL_FLUSH(scratch_addr);
296 scratch_addr += 128;
297 PIPE_CONTROL_FLUSH(scratch_addr);
298 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
299 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
300 PIPE_CONTROL_NOTIFY);
301 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
302 OUT_RING(seqno);
303 OUT_RING(0);
304 ADVANCE_LP_RING();
305 } else {
306 BEGIN_LP_RING(4);
307 OUT_RING(MI_STORE_DWORD_INDEX);
308 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
309 OUT_RING(seqno);
310 452
311 OUT_RING(MI_USER_INTERRUPT); 453 ret = intel_ring_begin(ring, 4);
312 ADVANCE_LP_RING(); 454 if (ret)
313 } 455 return ret;
314 return seqno; 456
457 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
458 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
459 intel_ring_emit(ring, seqno);
460 intel_ring_emit(ring, MI_USER_INTERRUPT);
461 intel_ring_advance(ring);
462
463 *result = seqno;
464 return 0;
315} 465}
316 466
317static u32 467static u32
318render_ring_get_gem_seqno(struct drm_device *dev, 468ring_get_seqno(struct intel_ring_buffer *ring)
319 struct intel_ring_buffer *ring)
320{ 469{
321 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 470 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
322 if (HAS_PIPE_CONTROL(dev)) 471}
323 return ((volatile u32 *)(dev_priv->seqno_page))[0]; 472
324 else 473static u32
325 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 474pc_render_get_seqno(struct intel_ring_buffer *ring)
475{
476 struct pipe_control *pc = ring->private;
477 return pc->cpu_page[0];
478}
479
480static void
481ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
482{
483 dev_priv->gt_irq_mask &= ~mask;
484 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
485 POSTING_READ(GTIMR);
486}
487
488static void
489ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
490{
491 dev_priv->gt_irq_mask |= mask;
492 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
493 POSTING_READ(GTIMR);
494}
495
496static void
497i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
498{
499 dev_priv->irq_mask &= ~mask;
500 I915_WRITE(IMR, dev_priv->irq_mask);
501 POSTING_READ(IMR);
326} 502}
327 503
328static void 504static void
329render_ring_get_user_irq(struct drm_device *dev, 505i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
330 struct intel_ring_buffer *ring) 506{
507 dev_priv->irq_mask |= mask;
508 I915_WRITE(IMR, dev_priv->irq_mask);
509 POSTING_READ(IMR);
510}
511
512static bool
513render_ring_get_irq(struct intel_ring_buffer *ring)
331{ 514{
332 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 515 struct drm_device *dev = ring->dev;
333 unsigned long irqflags; 516 drm_i915_private_t *dev_priv = dev->dev_private;
334 517
335 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 518 if (!dev->irq_enabled)
336 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) { 519 return false;
520
521 spin_lock(&ring->irq_lock);
522 if (ring->irq_refcount++ == 0) {
337 if (HAS_PCH_SPLIT(dev)) 523 if (HAS_PCH_SPLIT(dev))
338 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 524 ironlake_enable_irq(dev_priv,
525 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
339 else 526 else
340 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 527 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
341 } 528 }
342 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 529 spin_unlock(&ring->irq_lock);
530
531 return true;
343} 532}
344 533
345static void 534static void
346render_ring_put_user_irq(struct drm_device *dev, 535render_ring_put_irq(struct intel_ring_buffer *ring)
347 struct intel_ring_buffer *ring)
348{ 536{
349 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 537 struct drm_device *dev = ring->dev;
350 unsigned long irqflags; 538 drm_i915_private_t *dev_priv = dev->dev_private;
351 539
352 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); 540 spin_lock(&ring->irq_lock);
353 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0); 541 if (--ring->irq_refcount == 0) {
354 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
355 if (HAS_PCH_SPLIT(dev)) 542 if (HAS_PCH_SPLIT(dev))
356 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); 543 ironlake_disable_irq(dev_priv,
544 GT_USER_INTERRUPT |
545 GT_PIPE_NOTIFY);
357 else 546 else
358 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 547 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
359 } 548 }
360 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); 549 spin_unlock(&ring->irq_lock);
361} 550}
362 551
363static void render_setup_status_page(struct drm_device *dev, 552void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
364 struct intel_ring_buffer *ring)
365{ 553{
366 drm_i915_private_t *dev_priv = dev->dev_private; 554 struct drm_device *dev = ring->dev;
367 if (IS_GEN6(dev)) { 555 drm_i915_private_t *dev_priv = ring->dev->dev_private;
368 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr); 556 u32 mmio = 0;
369 I915_READ(HWS_PGA_GEN6); /* posting read */ 557
558 /* The ring status page addresses are no longer next to the rest of
559 * the ring registers as of gen7.
560 */
561 if (IS_GEN7(dev)) {
562 switch (ring->id) {
563 case RING_RENDER:
564 mmio = RENDER_HWS_PGA_GEN7;
565 break;
566 case RING_BLT:
567 mmio = BLT_HWS_PGA_GEN7;
568 break;
569 case RING_BSD:
570 mmio = BSD_HWS_PGA_GEN7;
571 break;
572 }
573 } else if (IS_GEN6(ring->dev)) {
574 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
370 } else { 575 } else {
371 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr); 576 mmio = RING_HWS_PGA(ring->mmio_base);
372 I915_READ(HWS_PGA); /* posting read */
373 } 577 }
374 578
579 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
580 POSTING_READ(mmio);
375} 581}
376 582
377void 583static int
378bsd_ring_flush(struct drm_device *dev, 584bsd_ring_flush(struct intel_ring_buffer *ring,
379 struct intel_ring_buffer *ring, 585 u32 invalidate_domains,
380 u32 invalidate_domains, 586 u32 flush_domains)
381 u32 flush_domains)
382{ 587{
383 intel_ring_begin(dev, ring, 2); 588 int ret;
384 intel_ring_emit(dev, ring, MI_FLUSH);
385 intel_ring_emit(dev, ring, MI_NOOP);
386 intel_ring_advance(dev, ring);
387}
388 589
389static inline unsigned int bsd_ring_get_head(struct drm_device *dev, 590 ret = intel_ring_begin(ring, 2);
390 struct intel_ring_buffer *ring) 591 if (ret)
391{ 592 return ret;
392 drm_i915_private_t *dev_priv = dev->dev_private;
393 return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
394}
395 593
396static inline unsigned int bsd_ring_get_tail(struct drm_device *dev, 594 intel_ring_emit(ring, MI_FLUSH);
397 struct intel_ring_buffer *ring) 595 intel_ring_emit(ring, MI_NOOP);
398{ 596 intel_ring_advance(ring);
399 drm_i915_private_t *dev_priv = dev->dev_private; 597 return 0;
400 return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
401} 598}
402 599
403static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev, 600static int
404 struct intel_ring_buffer *ring) 601ring_add_request(struct intel_ring_buffer *ring,
602 u32 *result)
405{ 603{
406 drm_i915_private_t *dev_priv = dev->dev_private; 604 u32 seqno;
407 return I915_READ(BSD_RING_ACTHD); 605 int ret;
408}
409 606
410static inline void bsd_ring_advance_ring(struct drm_device *dev, 607 ret = intel_ring_begin(ring, 4);
411 struct intel_ring_buffer *ring) 608 if (ret)
412{ 609 return ret;
413 drm_i915_private_t *dev_priv = dev->dev_private;
414 I915_WRITE(BSD_RING_TAIL, ring->tail);
415}
416 610
417static int init_bsd_ring(struct drm_device *dev, 611 seqno = i915_gem_get_seqno(ring->dev);
418 struct intel_ring_buffer *ring) 612
419{ 613 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
420 return init_ring_common(dev, ring); 614 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
615 intel_ring_emit(ring, seqno);
616 intel_ring_emit(ring, MI_USER_INTERRUPT);
617 intel_ring_advance(ring);
618
619 *result = seqno;
620 return 0;
421} 621}
422 622
423static u32 623static bool
424bsd_ring_add_request(struct drm_device *dev, 624gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
425 struct intel_ring_buffer *ring,
426 struct drm_file *file_priv,
427 u32 flush_domains)
428{ 625{
429 u32 seqno; 626 struct drm_device *dev = ring->dev;
430 627 drm_i915_private_t *dev_priv = dev->dev_private;
431 seqno = i915_gem_get_seqno(dev);
432 628
433 intel_ring_begin(dev, ring, 4); 629 if (!dev->irq_enabled)
434 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); 630 return false;
435 intel_ring_emit(dev, ring,
436 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
437 intel_ring_emit(dev, ring, seqno);
438 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
439 intel_ring_advance(dev, ring);
440 631
441 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); 632 spin_lock(&ring->irq_lock);
633 if (ring->irq_refcount++ == 0) {
634 ring->irq_mask &= ~rflag;
635 I915_WRITE_IMR(ring, ring->irq_mask);
636 ironlake_enable_irq(dev_priv, gflag);
637 }
638 spin_unlock(&ring->irq_lock);
442 639
443 return seqno; 640 return true;
444} 641}
445 642
446static void bsd_setup_status_page(struct drm_device *dev, 643static void
447 struct intel_ring_buffer *ring) 644gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
448{ 645{
646 struct drm_device *dev = ring->dev;
449 drm_i915_private_t *dev_priv = dev->dev_private; 647 drm_i915_private_t *dev_priv = dev->dev_private;
450 I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr); 648
451 I915_READ(BSD_HWS_PGA); 649 spin_lock(&ring->irq_lock);
650 if (--ring->irq_refcount == 0) {
651 ring->irq_mask |= rflag;
652 I915_WRITE_IMR(ring, ring->irq_mask);
653 ironlake_disable_irq(dev_priv, gflag);
654 }
655 spin_unlock(&ring->irq_lock);
452} 656}
453 657
454static void 658static bool
455bsd_ring_get_user_irq(struct drm_device *dev, 659bsd_ring_get_irq(struct intel_ring_buffer *ring)
456 struct intel_ring_buffer *ring)
457{ 660{
458 /* do nothing */ 661 struct drm_device *dev = ring->dev;
662 drm_i915_private_t *dev_priv = dev->dev_private;
663
664 if (!dev->irq_enabled)
665 return false;
666
667 spin_lock(&ring->irq_lock);
668 if (ring->irq_refcount++ == 0) {
669 if (IS_G4X(dev))
670 i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
671 else
672 ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
673 }
674 spin_unlock(&ring->irq_lock);
675
676 return true;
459} 677}
460static void 678static void
461bsd_ring_put_user_irq(struct drm_device *dev, 679bsd_ring_put_irq(struct intel_ring_buffer *ring)
462 struct intel_ring_buffer *ring)
463{ 680{
464 /* do nothing */ 681 struct drm_device *dev = ring->dev;
465} 682 drm_i915_private_t *dev_priv = dev->dev_private;
466 683
467static u32 684 spin_lock(&ring->irq_lock);
468bsd_ring_get_gem_seqno(struct drm_device *dev, 685 if (--ring->irq_refcount == 0) {
469 struct intel_ring_buffer *ring) 686 if (IS_G4X(dev))
470{ 687 i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
471 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 688 else
689 ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
690 }
691 spin_unlock(&ring->irq_lock);
472} 692}
473 693
474static int 694static int
475bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev, 695ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
476 struct intel_ring_buffer *ring, 696{
477 struct drm_i915_gem_execbuffer2 *exec, 697 int ret;
478 struct drm_clip_rect *cliprects, 698
479 uint64_t exec_offset) 699 ret = intel_ring_begin(ring, 2);
480{ 700 if (ret)
481 uint32_t exec_start; 701 return ret;
482 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 702
483 intel_ring_begin(dev, ring, 2); 703 intel_ring_emit(ring,
484 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START | 704 MI_BATCH_BUFFER_START | (2 << 6) |
485 (2 << 6) | MI_BATCH_NON_SECURE_I965); 705 MI_BATCH_NON_SECURE_I965);
486 intel_ring_emit(dev, ring, exec_start); 706 intel_ring_emit(ring, offset);
487 intel_ring_advance(dev, ring); 707 intel_ring_advance(ring);
708
488 return 0; 709 return 0;
489} 710}
490 711
491
492static int 712static int
493render_ring_dispatch_gem_execbuffer(struct drm_device *dev, 713render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
494 struct intel_ring_buffer *ring, 714 u32 offset, u32 len)
495 struct drm_i915_gem_execbuffer2 *exec,
496 struct drm_clip_rect *cliprects,
497 uint64_t exec_offset)
498{ 715{
499 drm_i915_private_t *dev_priv = dev->dev_private; 716 struct drm_device *dev = ring->dev;
500 int nbox = exec->num_cliprects; 717 int ret;
501 int i = 0, count; 718
502 uint32_t exec_start, exec_len; 719 if (IS_I830(dev) || IS_845G(dev)) {
503 exec_start = (uint32_t) exec_offset + exec->batch_start_offset; 720 ret = intel_ring_begin(ring, 4);
504 exec_len = (uint32_t) exec->batch_len; 721 if (ret)
505 722 return ret;
506 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1); 723
507 724 intel_ring_emit(ring, MI_BATCH_BUFFER);
508 count = nbox ? nbox : 1; 725 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
509 726 intel_ring_emit(ring, offset + len - 8);
510 for (i = 0; i < count; i++) { 727 intel_ring_emit(ring, 0);
511 if (i < nbox) { 728 } else {
512 int ret = i915_emit_box(dev, cliprects, i, 729 ret = intel_ring_begin(ring, 2);
513 exec->DR1, exec->DR4); 730 if (ret)
514 if (ret) 731 return ret;
515 return ret;
516 }
517 732
518 if (IS_I830(dev) || IS_845G(dev)) { 733 if (INTEL_INFO(dev)->gen >= 4) {
519 intel_ring_begin(dev, ring, 4); 734 intel_ring_emit(ring,
520 intel_ring_emit(dev, ring, MI_BATCH_BUFFER); 735 MI_BATCH_BUFFER_START | (2 << 6) |
521 intel_ring_emit(dev, ring, 736 MI_BATCH_NON_SECURE_I965);
522 exec_start | MI_BATCH_NON_SECURE); 737 intel_ring_emit(ring, offset);
523 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
524 intel_ring_emit(dev, ring, 0);
525 } else { 738 } else {
526 intel_ring_begin(dev, ring, 4); 739 intel_ring_emit(ring,
527 if (IS_I965G(dev)) { 740 MI_BATCH_BUFFER_START | (2 << 6));
528 intel_ring_emit(dev, ring, 741 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
529 MI_BATCH_BUFFER_START | (2 << 6)
530 | MI_BATCH_NON_SECURE_I965);
531 intel_ring_emit(dev, ring, exec_start);
532 } else {
533 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
534 | (2 << 6));
535 intel_ring_emit(dev, ring, exec_start |
536 MI_BATCH_NON_SECURE);
537 }
538 } 742 }
539 intel_ring_advance(dev, ring);
540 }
541
542 if (IS_G4X(dev) || IS_IRONLAKE(dev)) {
543 intel_ring_begin(dev, ring, 2);
544 intel_ring_emit(dev, ring, MI_FLUSH |
545 MI_NO_WRITE_FLUSH |
546 MI_INVALIDATE_ISP );
547 intel_ring_emit(dev, ring, MI_NOOP);
548 intel_ring_advance(dev, ring);
549 } 743 }
550 /* XXX breadcrumb */ 744 intel_ring_advance(ring);
551 745
552 return 0; 746 return 0;
553} 747}
554 748
555static void cleanup_status_page(struct drm_device *dev, 749static void cleanup_status_page(struct intel_ring_buffer *ring)
556 struct intel_ring_buffer *ring)
557{ 750{
558 drm_i915_private_t *dev_priv = dev->dev_private; 751 drm_i915_private_t *dev_priv = ring->dev->dev_private;
559 struct drm_gem_object *obj; 752 struct drm_i915_gem_object *obj;
560 struct drm_i915_gem_object *obj_priv;
561 753
562 obj = ring->status_page.obj; 754 obj = ring->status_page.obj;
563 if (obj == NULL) 755 if (obj == NULL)
564 return; 756 return;
565 obj_priv = to_intel_bo(obj);
566 757
567 kunmap(obj_priv->pages[0]); 758 kunmap(obj->pages[0]);
568 i915_gem_object_unpin(obj); 759 i915_gem_object_unpin(obj);
569 drm_gem_object_unreference(obj); 760 drm_gem_object_unreference(&obj->base);
570 ring->status_page.obj = NULL; 761 ring->status_page.obj = NULL;
571 762
572 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 763 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
573} 764}
574 765
575static int init_status_page(struct drm_device *dev, 766static int init_status_page(struct intel_ring_buffer *ring)
576 struct intel_ring_buffer *ring)
577{ 767{
768 struct drm_device *dev = ring->dev;
578 drm_i915_private_t *dev_priv = dev->dev_private; 769 drm_i915_private_t *dev_priv = dev->dev_private;
579 struct drm_gem_object *obj; 770 struct drm_i915_gem_object *obj;
580 struct drm_i915_gem_object *obj_priv;
581 int ret; 771 int ret;
582 772
583 obj = i915_gem_alloc_object(dev, 4096); 773 obj = i915_gem_alloc_object(dev, 4096);
@@ -586,16 +776,15 @@ static int init_status_page(struct drm_device *dev,
586 ret = -ENOMEM; 776 ret = -ENOMEM;
587 goto err; 777 goto err;
588 } 778 }
589 obj_priv = to_intel_bo(obj); 779 obj->cache_level = I915_CACHE_LLC;
590 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
591 780
592 ret = i915_gem_object_pin(obj, 4096); 781 ret = i915_gem_object_pin(obj, 4096, true);
593 if (ret != 0) { 782 if (ret != 0) {
594 goto err_unref; 783 goto err_unref;
595 } 784 }
596 785
597 ring->status_page.gfx_addr = obj_priv->gtt_offset; 786 ring->status_page.gfx_addr = obj->gtt_offset;
598 ring->status_page.page_addr = kmap(obj_priv->pages[0]); 787 ring->status_page.page_addr = kmap(obj->pages[0]);
599 if (ring->status_page.page_addr == NULL) { 788 if (ring->status_page.page_addr == NULL) {
600 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); 789 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
601 goto err_unpin; 790 goto err_unpin;
@@ -603,7 +792,7 @@ static int init_status_page(struct drm_device *dev,
603 ring->status_page.obj = obj; 792 ring->status_page.obj = obj;
604 memset(ring->status_page.page_addr, 0, PAGE_SIZE); 793 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
605 794
606 ring->setup_status_page(dev, ring); 795 intel_ring_setup_status_page(ring);
607 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", 796 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
608 ring->name, ring->status_page.gfx_addr); 797 ring->name, ring->status_page.gfx_addr);
609 798
@@ -612,23 +801,28 @@ static int init_status_page(struct drm_device *dev,
612err_unpin: 801err_unpin:
613 i915_gem_object_unpin(obj); 802 i915_gem_object_unpin(obj);
614err_unref: 803err_unref:
615 drm_gem_object_unreference(obj); 804 drm_gem_object_unreference(&obj->base);
616err: 805err:
617 return ret; 806 return ret;
618} 807}
619 808
620
621int intel_init_ring_buffer(struct drm_device *dev, 809int intel_init_ring_buffer(struct drm_device *dev,
622 struct intel_ring_buffer *ring) 810 struct intel_ring_buffer *ring)
623{ 811{
624 struct drm_i915_gem_object *obj_priv; 812 struct drm_i915_gem_object *obj;
625 struct drm_gem_object *obj;
626 int ret; 813 int ret;
627 814
628 ring->dev = dev; 815 ring->dev = dev;
816 INIT_LIST_HEAD(&ring->active_list);
817 INIT_LIST_HEAD(&ring->request_list);
818 INIT_LIST_HEAD(&ring->gpu_write_list);
819
820 init_waitqueue_head(&ring->irq_queue);
821 spin_lock_init(&ring->irq_lock);
822 ring->irq_mask = ~0;
629 823
630 if (I915_NEED_GFX_HWS(dev)) { 824 if (I915_NEED_GFX_HWS(dev)) {
631 ret = init_status_page(dev, ring); 825 ret = init_status_page(ring);
632 if (ret) 826 if (ret)
633 return ret; 827 return ret;
634 } 828 }
@@ -640,15 +834,14 @@ int intel_init_ring_buffer(struct drm_device *dev,
640 goto err_hws; 834 goto err_hws;
641 } 835 }
642 836
643 ring->gem_object = obj; 837 ring->obj = obj;
644 838
645 ret = i915_gem_object_pin(obj, ring->alignment); 839 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
646 if (ret) 840 if (ret)
647 goto err_unref; 841 goto err_unref;
648 842
649 obj_priv = to_intel_bo(obj);
650 ring->map.size = ring->size; 843 ring->map.size = ring->size;
651 ring->map.offset = dev->agp->base + obj_priv->gtt_offset; 844 ring->map.offset = dev->agp->base + obj->gtt_offset;
652 ring->map.type = 0; 845 ring->map.type = 0;
653 ring->map.flags = 0; 846 ring->map.flags = 0;
654 ring->map.mtrr = 0; 847 ring->map.mtrr = 0;
@@ -661,58 +854,68 @@ int intel_init_ring_buffer(struct drm_device *dev,
661 } 854 }
662 855
663 ring->virtual_start = ring->map.handle; 856 ring->virtual_start = ring->map.handle;
664 ret = ring->init(dev, ring); 857 ret = ring->init(ring);
665 if (ret) 858 if (ret)
666 goto err_unmap; 859 goto err_unmap;
667 860
668 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 861 /* Workaround an erratum on the i830 which causes a hang if
669 i915_kernel_lost_context(dev); 862 * the TAIL pointer points to within the last 2 cachelines
670 else { 863 * of the buffer.
671 ring->head = ring->get_head(dev, ring); 864 */
672 ring->tail = ring->get_tail(dev, ring); 865 ring->effective_size = ring->size;
673 ring->space = ring->head - (ring->tail + 8); 866 if (IS_I830(ring->dev))
674 if (ring->space < 0) 867 ring->effective_size -= 128;
675 ring->space += ring->size; 868
676 } 869 return 0;
677 INIT_LIST_HEAD(&ring->active_list);
678 INIT_LIST_HEAD(&ring->request_list);
679 return ret;
680 870
681err_unmap: 871err_unmap:
682 drm_core_ioremapfree(&ring->map, dev); 872 drm_core_ioremapfree(&ring->map, dev);
683err_unpin: 873err_unpin:
684 i915_gem_object_unpin(obj); 874 i915_gem_object_unpin(obj);
685err_unref: 875err_unref:
686 drm_gem_object_unreference(obj); 876 drm_gem_object_unreference(&obj->base);
687 ring->gem_object = NULL; 877 ring->obj = NULL;
688err_hws: 878err_hws:
689 cleanup_status_page(dev, ring); 879 cleanup_status_page(ring);
690 return ret; 880 return ret;
691} 881}
692 882
693void intel_cleanup_ring_buffer(struct drm_device *dev, 883void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
694 struct intel_ring_buffer *ring)
695{ 884{
696 if (ring->gem_object == NULL) 885 struct drm_i915_private *dev_priv;
886 int ret;
887
888 if (ring->obj == NULL)
697 return; 889 return;
698 890
699 drm_core_ioremapfree(&ring->map, dev); 891 /* Disable the ring buffer. The ring must be idle at this point */
892 dev_priv = ring->dev->dev_private;
893 ret = intel_wait_ring_idle(ring);
894 if (ret)
895 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
896 ring->name, ret);
897
898 I915_WRITE_CTL(ring, 0);
700 899
701 i915_gem_object_unpin(ring->gem_object); 900 drm_core_ioremapfree(&ring->map, ring->dev);
702 drm_gem_object_unreference(ring->gem_object); 901
703 ring->gem_object = NULL; 902 i915_gem_object_unpin(ring->obj);
704 cleanup_status_page(dev, ring); 903 drm_gem_object_unreference(&ring->obj->base);
904 ring->obj = NULL;
905
906 if (ring->cleanup)
907 ring->cleanup(ring);
908
909 cleanup_status_page(ring);
705} 910}
706 911
707int intel_wrap_ring_buffer(struct drm_device *dev, 912static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
708 struct intel_ring_buffer *ring)
709{ 913{
710 unsigned int *virt; 914 unsigned int *virt;
711 int rem; 915 int rem = ring->size - ring->tail;
712 rem = ring->size - ring->tail;
713 916
714 if (ring->space < rem) { 917 if (ring->space < rem) {
715 int ret = intel_wait_ring_buffer(dev, ring, rem); 918 int ret = intel_wait_ring_buffer(ring, rem);
716 if (ret) 919 if (ret)
717 return ret; 920 return ret;
718 } 921 }
@@ -725,25 +928,36 @@ int intel_wrap_ring_buffer(struct drm_device *dev,
725 } 928 }
726 929
727 ring->tail = 0; 930 ring->tail = 0;
728 ring->space = ring->head - 8; 931 ring->space = ring_space(ring);
729 932
730 return 0; 933 return 0;
731} 934}
732 935
733int intel_wait_ring_buffer(struct drm_device *dev, 936int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
734 struct intel_ring_buffer *ring, int n)
735{ 937{
938 struct drm_device *dev = ring->dev;
939 struct drm_i915_private *dev_priv = dev->dev_private;
736 unsigned long end; 940 unsigned long end;
941 u32 head;
942
943 /* If the reported head position has wrapped or hasn't advanced,
944 * fallback to the slow and accurate path.
945 */
946 head = intel_read_status_page(ring, 4);
947 if (head > ring->head) {
948 ring->head = head;
949 ring->space = ring_space(ring);
950 if (ring->space >= n)
951 return 0;
952 }
737 953
738 trace_i915_ring_wait_begin (dev); 954 trace_i915_ring_wait_begin(ring);
739 end = jiffies + 3 * HZ; 955 end = jiffies + 3 * HZ;
740 do { 956 do {
741 ring->head = ring->get_head(dev, ring); 957 ring->head = I915_READ_HEAD(ring);
742 ring->space = ring->head - (ring->tail + 8); 958 ring->space = ring_space(ring);
743 if (ring->space < 0)
744 ring->space += ring->size;
745 if (ring->space >= n) { 959 if (ring->space >= n) {
746 trace_i915_ring_wait_end (dev); 960 trace_i915_ring_wait_end(ring);
747 return 0; 961 return 0;
748 } 962 }
749 963
@@ -753,116 +967,404 @@ int intel_wait_ring_buffer(struct drm_device *dev,
753 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 967 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
754 } 968 }
755 969
756 yield(); 970 msleep(1);
971 if (atomic_read(&dev_priv->mm.wedged))
972 return -EAGAIN;
757 } while (!time_after(jiffies, end)); 973 } while (!time_after(jiffies, end));
758 trace_i915_ring_wait_end (dev); 974 trace_i915_ring_wait_end(ring);
759 return -EBUSY; 975 return -EBUSY;
760} 976}
761 977
762void intel_ring_begin(struct drm_device *dev, 978int intel_ring_begin(struct intel_ring_buffer *ring,
763 struct intel_ring_buffer *ring, int num_dwords) 979 int num_dwords)
764{ 980{
981 struct drm_i915_private *dev_priv = ring->dev->dev_private;
765 int n = 4*num_dwords; 982 int n = 4*num_dwords;
766 if (unlikely(ring->tail + n > ring->size)) 983 int ret;
767 intel_wrap_ring_buffer(dev, ring);
768 if (unlikely(ring->space < n))
769 intel_wait_ring_buffer(dev, ring, n);
770 984
771 ring->space -= n; 985 if (unlikely(atomic_read(&dev_priv->mm.wedged)))
772} 986 return -EIO;
773 987
774void intel_ring_advance(struct drm_device *dev, 988 if (unlikely(ring->tail + n > ring->effective_size)) {
775 struct intel_ring_buffer *ring) 989 ret = intel_wrap_ring_buffer(ring);
776{ 990 if (unlikely(ret))
777 ring->tail &= ring->size - 1; 991 return ret;
778 ring->advance_ring(dev, ring); 992 }
993
994 if (unlikely(ring->space < n)) {
995 ret = intel_wait_ring_buffer(ring, n);
996 if (unlikely(ret))
997 return ret;
998 }
999
1000 ring->space -= n;
1001 return 0;
779} 1002}
780 1003
781void intel_fill_struct(struct drm_device *dev, 1004void intel_ring_advance(struct intel_ring_buffer *ring)
782 struct intel_ring_buffer *ring,
783 void *data,
784 unsigned int len)
785{ 1005{
786 unsigned int *virt = ring->virtual_start + ring->tail;
787 BUG_ON((len&~(4-1)) != 0);
788 intel_ring_begin(dev, ring, len/4);
789 memcpy(virt, data, len);
790 ring->tail += len;
791 ring->tail &= ring->size - 1; 1006 ring->tail &= ring->size - 1;
792 ring->space -= len; 1007 ring->write_tail(ring, ring->tail);
793 intel_ring_advance(dev, ring);
794} 1008}
795 1009
796struct intel_ring_buffer render_ring = { 1010static const struct intel_ring_buffer render_ring = {
797 .name = "render ring", 1011 .name = "render ring",
798 .regs = { 1012 .id = RING_RENDER,
799 .ctl = PRB0_CTL, 1013 .mmio_base = RENDER_RING_BASE,
800 .head = PRB0_HEAD,
801 .tail = PRB0_TAIL,
802 .start = PRB0_START
803 },
804 .ring_flag = I915_EXEC_RENDER,
805 .size = 32 * PAGE_SIZE, 1014 .size = 32 * PAGE_SIZE,
806 .alignment = PAGE_SIZE,
807 .virtual_start = NULL,
808 .dev = NULL,
809 .gem_object = NULL,
810 .head = 0,
811 .tail = 0,
812 .space = 0,
813 .user_irq_refcount = 0,
814 .irq_gem_seqno = 0,
815 .waiting_gem_seqno = 0,
816 .setup_status_page = render_setup_status_page,
817 .init = init_render_ring, 1015 .init = init_render_ring,
818 .get_head = render_ring_get_head, 1016 .write_tail = ring_write_tail,
819 .get_tail = render_ring_get_tail,
820 .get_active_head = render_ring_get_active_head,
821 .advance_ring = render_ring_advance_ring,
822 .flush = render_ring_flush, 1017 .flush = render_ring_flush,
823 .add_request = render_ring_add_request, 1018 .add_request = render_ring_add_request,
824 .get_gem_seqno = render_ring_get_gem_seqno, 1019 .get_seqno = ring_get_seqno,
825 .user_irq_get = render_ring_get_user_irq, 1020 .irq_get = render_ring_get_irq,
826 .user_irq_put = render_ring_put_user_irq, 1021 .irq_put = render_ring_put_irq,
827 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer, 1022 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
828 .status_page = {NULL, 0, NULL}, 1023 .cleanup = render_ring_cleanup,
829 .map = {0,}
830}; 1024};
831 1025
832/* ring buffer for bit-stream decoder */ 1026/* ring buffer for bit-stream decoder */
833 1027
834struct intel_ring_buffer bsd_ring = { 1028static const struct intel_ring_buffer bsd_ring = {
835 .name = "bsd ring", 1029 .name = "bsd ring",
836 .regs = { 1030 .id = RING_BSD,
837 .ctl = BSD_RING_CTL, 1031 .mmio_base = BSD_RING_BASE,
838 .head = BSD_RING_HEAD,
839 .tail = BSD_RING_TAIL,
840 .start = BSD_RING_START
841 },
842 .ring_flag = I915_EXEC_BSD,
843 .size = 32 * PAGE_SIZE, 1032 .size = 32 * PAGE_SIZE,
844 .alignment = PAGE_SIZE, 1033 .init = init_ring_common,
845 .virtual_start = NULL, 1034 .write_tail = ring_write_tail,
846 .dev = NULL,
847 .gem_object = NULL,
848 .head = 0,
849 .tail = 0,
850 .space = 0,
851 .user_irq_refcount = 0,
852 .irq_gem_seqno = 0,
853 .waiting_gem_seqno = 0,
854 .setup_status_page = bsd_setup_status_page,
855 .init = init_bsd_ring,
856 .get_head = bsd_ring_get_head,
857 .get_tail = bsd_ring_get_tail,
858 .get_active_head = bsd_ring_get_active_head,
859 .advance_ring = bsd_ring_advance_ring,
860 .flush = bsd_ring_flush, 1035 .flush = bsd_ring_flush,
861 .add_request = bsd_ring_add_request, 1036 .add_request = ring_add_request,
862 .get_gem_seqno = bsd_ring_get_gem_seqno, 1037 .get_seqno = ring_get_seqno,
863 .user_irq_get = bsd_ring_get_user_irq, 1038 .irq_get = bsd_ring_get_irq,
864 .user_irq_put = bsd_ring_put_user_irq, 1039 .irq_put = bsd_ring_put_irq,
865 .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer, 1040 .dispatch_execbuffer = ring_dispatch_execbuffer,
866 .status_page = {NULL, 0, NULL}, 1041};
867 .map = {0,} 1042
1043
1044static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1045 u32 value)
1046{
1047 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1048
1049 /* Every tail move must follow the sequence below */
1050 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1051 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1052 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1053 I915_WRITE(GEN6_BSD_RNCID, 0x0);
1054
1055 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1056 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1057 50))
1058 DRM_ERROR("timed out waiting for IDLE Indicator\n");
1059
1060 I915_WRITE_TAIL(ring, value);
1061 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1062 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1063 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1064}
1065
1066static int gen6_ring_flush(struct intel_ring_buffer *ring,
1067 u32 invalidate, u32 flush)
1068{
1069 uint32_t cmd;
1070 int ret;
1071
1072 ret = intel_ring_begin(ring, 4);
1073 if (ret)
1074 return ret;
1075
1076 cmd = MI_FLUSH_DW;
1077 if (invalidate & I915_GEM_GPU_DOMAINS)
1078 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1079 intel_ring_emit(ring, cmd);
1080 intel_ring_emit(ring, 0);
1081 intel_ring_emit(ring, 0);
1082 intel_ring_emit(ring, MI_NOOP);
1083 intel_ring_advance(ring);
1084 return 0;
1085}
1086
1087static int
1088gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1089 u32 offset, u32 len)
1090{
1091 int ret;
1092
1093 ret = intel_ring_begin(ring, 2);
1094 if (ret)
1095 return ret;
1096
1097 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1098 /* bit0-7 is the length on GEN6+ */
1099 intel_ring_emit(ring, offset);
1100 intel_ring_advance(ring);
1101
1102 return 0;
1103}
1104
1105static bool
1106gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1107{
1108 return gen6_ring_get_irq(ring,
1109 GT_USER_INTERRUPT,
1110 GEN6_RENDER_USER_INTERRUPT);
1111}
1112
1113static void
1114gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1115{
1116 return gen6_ring_put_irq(ring,
1117 GT_USER_INTERRUPT,
1118 GEN6_RENDER_USER_INTERRUPT);
1119}
1120
1121static bool
1122gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1123{
1124 return gen6_ring_get_irq(ring,
1125 GT_GEN6_BSD_USER_INTERRUPT,
1126 GEN6_BSD_USER_INTERRUPT);
1127}
1128
1129static void
1130gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1131{
1132 return gen6_ring_put_irq(ring,
1133 GT_GEN6_BSD_USER_INTERRUPT,
1134 GEN6_BSD_USER_INTERRUPT);
1135}
1136
1137/* ring buffer for Video Codec for Gen6+ */
1138static const struct intel_ring_buffer gen6_bsd_ring = {
1139 .name = "gen6 bsd ring",
1140 .id = RING_BSD,
1141 .mmio_base = GEN6_BSD_RING_BASE,
1142 .size = 32 * PAGE_SIZE,
1143 .init = init_ring_common,
1144 .write_tail = gen6_bsd_ring_write_tail,
1145 .flush = gen6_ring_flush,
1146 .add_request = gen6_add_request,
1147 .get_seqno = ring_get_seqno,
1148 .irq_get = gen6_bsd_ring_get_irq,
1149 .irq_put = gen6_bsd_ring_put_irq,
1150 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1151};
1152
1153/* Blitter support (SandyBridge+) */
1154
1155static bool
1156blt_ring_get_irq(struct intel_ring_buffer *ring)
1157{
1158 return gen6_ring_get_irq(ring,
1159 GT_BLT_USER_INTERRUPT,
1160 GEN6_BLITTER_USER_INTERRUPT);
1161}
1162
1163static void
1164blt_ring_put_irq(struct intel_ring_buffer *ring)
1165{
1166 gen6_ring_put_irq(ring,
1167 GT_BLT_USER_INTERRUPT,
1168 GEN6_BLITTER_USER_INTERRUPT);
1169}
1170
1171
1172/* Workaround for some stepping of SNB,
1173 * each time when BLT engine ring tail moved,
1174 * the first command in the ring to be parsed
1175 * should be MI_BATCH_BUFFER_START
1176 */
1177#define NEED_BLT_WORKAROUND(dev) \
1178 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1179
1180static inline struct drm_i915_gem_object *
1181to_blt_workaround(struct intel_ring_buffer *ring)
1182{
1183 return ring->private;
1184}
1185
1186static int blt_ring_init(struct intel_ring_buffer *ring)
1187{
1188 if (NEED_BLT_WORKAROUND(ring->dev)) {
1189 struct drm_i915_gem_object *obj;
1190 u32 *ptr;
1191 int ret;
1192
1193 obj = i915_gem_alloc_object(ring->dev, 4096);
1194 if (obj == NULL)
1195 return -ENOMEM;
1196
1197 ret = i915_gem_object_pin(obj, 4096, true);
1198 if (ret) {
1199 drm_gem_object_unreference(&obj->base);
1200 return ret;
1201 }
1202
1203 ptr = kmap(obj->pages[0]);
1204 *ptr++ = MI_BATCH_BUFFER_END;
1205 *ptr++ = MI_NOOP;
1206 kunmap(obj->pages[0]);
1207
1208 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1209 if (ret) {
1210 i915_gem_object_unpin(obj);
1211 drm_gem_object_unreference(&obj->base);
1212 return ret;
1213 }
1214
1215 ring->private = obj;
1216 }
1217
1218 return init_ring_common(ring);
1219}
1220
1221static int blt_ring_begin(struct intel_ring_buffer *ring,
1222 int num_dwords)
1223{
1224 if (ring->private) {
1225 int ret = intel_ring_begin(ring, num_dwords+2);
1226 if (ret)
1227 return ret;
1228
1229 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1230 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1231
1232 return 0;
1233 } else
1234 return intel_ring_begin(ring, 4);
1235}
1236
1237static int blt_ring_flush(struct intel_ring_buffer *ring,
1238 u32 invalidate, u32 flush)
1239{
1240 uint32_t cmd;
1241 int ret;
1242
1243 ret = blt_ring_begin(ring, 4);
1244 if (ret)
1245 return ret;
1246
1247 cmd = MI_FLUSH_DW;
1248 if (invalidate & I915_GEM_DOMAIN_RENDER)
1249 cmd |= MI_INVALIDATE_TLB;
1250 intel_ring_emit(ring, cmd);
1251 intel_ring_emit(ring, 0);
1252 intel_ring_emit(ring, 0);
1253 intel_ring_emit(ring, MI_NOOP);
1254 intel_ring_advance(ring);
1255 return 0;
1256}
1257
1258static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1259{
1260 if (!ring->private)
1261 return;
1262
1263 i915_gem_object_unpin(ring->private);
1264 drm_gem_object_unreference(ring->private);
1265 ring->private = NULL;
1266}
1267
1268static const struct intel_ring_buffer gen6_blt_ring = {
1269 .name = "blt ring",
1270 .id = RING_BLT,
1271 .mmio_base = BLT_RING_BASE,
1272 .size = 32 * PAGE_SIZE,
1273 .init = blt_ring_init,
1274 .write_tail = ring_write_tail,
1275 .flush = blt_ring_flush,
1276 .add_request = gen6_add_request,
1277 .get_seqno = ring_get_seqno,
1278 .irq_get = blt_ring_get_irq,
1279 .irq_put = blt_ring_put_irq,
1280 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
1281 .cleanup = blt_ring_cleanup,
868}; 1282};
1283
1284int intel_init_render_ring_buffer(struct drm_device *dev)
1285{
1286 drm_i915_private_t *dev_priv = dev->dev_private;
1287 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1288
1289 *ring = render_ring;
1290 if (INTEL_INFO(dev)->gen >= 6) {
1291 ring->add_request = gen6_add_request;
1292 ring->irq_get = gen6_render_ring_get_irq;
1293 ring->irq_put = gen6_render_ring_put_irq;
1294 } else if (IS_GEN5(dev)) {
1295 ring->add_request = pc_render_add_request;
1296 ring->get_seqno = pc_render_get_seqno;
1297 }
1298
1299 if (!I915_NEED_GFX_HWS(dev)) {
1300 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1301 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1302 }
1303
1304 return intel_init_ring_buffer(dev, ring);
1305}
1306
1307int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1308{
1309 drm_i915_private_t *dev_priv = dev->dev_private;
1310 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1311
1312 *ring = render_ring;
1313 if (INTEL_INFO(dev)->gen >= 6) {
1314 ring->add_request = gen6_add_request;
1315 ring->irq_get = gen6_render_ring_get_irq;
1316 ring->irq_put = gen6_render_ring_put_irq;
1317 } else if (IS_GEN5(dev)) {
1318 ring->add_request = pc_render_add_request;
1319 ring->get_seqno = pc_render_get_seqno;
1320 }
1321
1322 ring->dev = dev;
1323 INIT_LIST_HEAD(&ring->active_list);
1324 INIT_LIST_HEAD(&ring->request_list);
1325 INIT_LIST_HEAD(&ring->gpu_write_list);
1326
1327 ring->size = size;
1328 ring->effective_size = ring->size;
1329 if (IS_I830(ring->dev))
1330 ring->effective_size -= 128;
1331
1332 ring->map.offset = start;
1333 ring->map.size = size;
1334 ring->map.type = 0;
1335 ring->map.flags = 0;
1336 ring->map.mtrr = 0;
1337
1338 drm_core_ioremap_wc(&ring->map, dev);
1339 if (ring->map.handle == NULL) {
1340 DRM_ERROR("can not ioremap virtual address for"
1341 " ring buffer\n");
1342 return -ENOMEM;
1343 }
1344
1345 ring->virtual_start = (void __force __iomem *)ring->map.handle;
1346 return 0;
1347}
1348
1349int intel_init_bsd_ring_buffer(struct drm_device *dev)
1350{
1351 drm_i915_private_t *dev_priv = dev->dev_private;
1352 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1353
1354 if (IS_GEN6(dev) || IS_GEN7(dev))
1355 *ring = gen6_bsd_ring;
1356 else
1357 *ring = bsd_ring;
1358
1359 return intel_init_ring_buffer(dev, ring);
1360}
1361
1362int intel_init_blt_ring_buffer(struct drm_device *dev)
1363{
1364 drm_i915_private_t *dev_priv = dev->dev_private;
1365 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1366
1367 *ring = gen6_blt_ring;
1368
1369 return intel_init_ring_buffer(dev, ring);
1370}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-03 21:27:29 -0400