drm/i915: Move ringbuffer-related code to intel_ringbuffer.c.

This is preparation for supporting multiple ringbuffers on Ironlake.
The non-copy-and-paste changes are:
- de-staticing functions
- I915_GEM_GPU_DOMAINS moving to i915_drv.h to be used by both files.
- i915_gem_add_request had only half its implementation
  copy-and-pasted out of the middle of it.

1 files changed, 568 insertions, 0 deletions

diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
new file mode 100644
index 000000000000..13a796fafaee
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -0,0 +1,568 @@
+/*
+ * Copyright © 2008-2010 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *    Zou Nan hai <nanhai.zou@intel.com>
+ *    Xiang Hai hao<haihao.xiang@intel.com>
+ *
+ */
+
+#include "drmP.h"
+#include "drm.h"
+#include "i915_drm.h"
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "intel_drv.h"
+
+void
+i915_gem_flush(struct drm_device *dev,
+               uint32_t invalidate_domains,
+               uint32_t flush_domains)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        uint32_t cmd;
+        RING_LOCALS;
+
+#if WATCH_EXEC
+        DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
+                  invalidate_domains, flush_domains);
+#endif
+        trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
+                                     invalidate_domains, flush_domains);
+
+        if (flush_domains & I915_GEM_DOMAIN_CPU)
+                drm_agp_chipset_flush(dev);
+
+        if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
+                /*
+                 * read/write caches:
+                 *
+                 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
+                 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
+                 * also flushed at 2d versus 3d pipeline switches.
+                 *
+                 * read-only caches:
+                 *
+                 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
+                 * MI_READ_FLUSH is set, and is always flushed on 965.
+                 *
+                 * I915_GEM_DOMAIN_COMMAND may not exist?
+                 *
+                 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
+                 * invalidated when MI_EXE_FLUSH is set.
+                 *
+                 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
+                 * invalidated with every MI_FLUSH.
+                 *
+                 * TLBs:
+                 *
+                 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
+                 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
+                 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
+                 * are flushed at any MI_FLUSH.
+                 */
+
+                cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+                if ((invalidate_domains|flush_domains) &
+                    I915_GEM_DOMAIN_RENDER)
+                        cmd &= ~MI_NO_WRITE_FLUSH;
+                if (!IS_I965G(dev)) {
+                        /*
+                         * On the 965, the sampler cache always gets flushed
+                         * and this bit is reserved.
+                         */
+                        if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+                                cmd |= MI_READ_FLUSH;
+                }
+                if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+                        cmd |= MI_EXE_FLUSH;
+
+#if WATCH_EXEC
+                DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
+#endif
+                BEGIN_LP_RING(2);
+                OUT_RING(cmd);
+                OUT_RING(MI_NOOP);
+                ADVANCE_LP_RING();
+        }
+
+}
+#define PIPE_CONTROL_FLUSH(addr)                                        \
+        OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
+                 PIPE_CONTROL_DEPTH_STALL);                             \
+        OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
+        OUT_RING(0);                                                    \
+        OUT_RING(0);                                                    \
+
+/**
+ * Creates a new sequence number, emitting a write of it to the status page
+ * plus an interrupt, which will trigger i915_user_interrupt_handler.
+ *
+ * Must be called with struct_lock held.
+ *
+ * Returned sequence numbers are nonzero on success.
+ */
+uint32_t
+i915_ring_add_request(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        uint32_t seqno;
+        RING_LOCALS;
+
+        /* Grab the seqno we're going to make this request be, and bump the
+         * next (skipping 0 so it can be the reserved no-seqno value).
+         */
+        seqno = dev_priv->mm.next_gem_seqno;
+        dev_priv->mm.next_gem_seqno++;
+        if (dev_priv->mm.next_gem_seqno == 0)
+                dev_priv->mm.next_gem_seqno++;
+
+        if (HAS_PIPE_CONTROL(dev)) {
+                u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
+
+                /*
+                 * Workaround qword write incoherence by flushing the
+                 * PIPE_NOTIFY buffers out to memory before requesting
+                 * an interrupt.
+                 */
+                BEGIN_LP_RING(32);
+                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
+                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
+                OUT_RING(seqno);
+                OUT_RING(0);
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                scratch_addr += 128; /* write to separate cachelines */
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                scratch_addr += 128;
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                scratch_addr += 128;
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                scratch_addr += 128;
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                scratch_addr += 128;
+                PIPE_CONTROL_FLUSH(scratch_addr);
+                OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
+                         PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
+                         PIPE_CONTROL_NOTIFY);
+                OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
+                OUT_RING(seqno);
+                OUT_RING(0);
+                ADVANCE_LP_RING();
+        } else {
+                BEGIN_LP_RING(4);
+                OUT_RING(MI_STORE_DWORD_INDEX);
+                OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+                OUT_RING(seqno);
+
+                OUT_RING(MI_USER_INTERRUPT);
+                ADVANCE_LP_RING();
+        }
+        return seqno;
+}
+
+void i915_user_irq_get(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
+
+        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+        if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)) {
+                if (HAS_PCH_SPLIT(dev))
+                        ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                else
+                        i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
+        }
+        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+}
+
+void i915_user_irq_put(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        unsigned long irqflags;
+
+        spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
+        BUG_ON(dev->irq_enabled && dev_priv->user_irq_refcount <= 0);
+        if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
+                if (HAS_PCH_SPLIT(dev))
+                        ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
+                else
+                        i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
+        }
+        spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
+}
+
+/** Dispatch a batchbuffer to the ring
+ */
+int
+i915_dispatch_gem_execbuffer(struct drm_device *dev,
+                              struct drm_i915_gem_execbuffer2 *exec,
+                              struct drm_clip_rect *cliprects,
+                              uint64_t exec_offset)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        int nbox = exec->num_cliprects;
+        int i = 0, count;
+        uint32_t exec_start, exec_len;
+        RING_LOCALS;
+
+        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
+        exec_len = (uint32_t) exec->batch_len;
+
+        trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
+
+        count = nbox ? nbox : 1;
+
+        for (i = 0; i < count; i++) {
+                if (i < nbox) {
+                        int ret = i915_emit_box(dev, cliprects, i,
+                                                exec->DR1, exec->DR4);
+                        if (ret)
+                                return ret;
+                }
+
+                if (IS_I830(dev) || IS_845G(dev)) {
+                        BEGIN_LP_RING(4);
+                        OUT_RING(MI_BATCH_BUFFER);
+                        OUT_RING(exec_start | MI_BATCH_NON_SECURE);
+                        OUT_RING(exec_start + exec_len - 4);
+                        OUT_RING(0);
+                        ADVANCE_LP_RING();
+                } else {
+                        BEGIN_LP_RING(2);
+                        if (IS_I965G(dev)) {
+                                OUT_RING(MI_BATCH_BUFFER_START |
+                                         (2 << 6) |
+                                         MI_BATCH_NON_SECURE_I965);
+                                OUT_RING(exec_start);
+                        } else {
+                                OUT_RING(MI_BATCH_BUFFER_START |
+                                         (2 << 6));
+                                OUT_RING(exec_start | MI_BATCH_NON_SECURE);
+                        }
+                        ADVANCE_LP_RING();
+                }
+        }
+
+        /* XXX breadcrumb */
+        return 0;
+}
+
+static void
+i915_gem_cleanup_hws(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj_priv;
+
+        if (dev_priv->hws_obj == NULL)
+                return;
+
+        obj = dev_priv->hws_obj;
+        obj_priv = to_intel_bo(obj);
+
+        kunmap(obj_priv->pages[0]);
+        i915_gem_object_unpin(obj);
+        drm_gem_object_unreference(obj);
+        dev_priv->hws_obj = NULL;
+
+        memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
+        dev_priv->hw_status_page = NULL;
+
+        if (HAS_PIPE_CONTROL(dev))
+                i915_gem_cleanup_pipe_control(dev);
+
+        /* Write high address into HWS_PGA when disabling. */
+        I915_WRITE(HWS_PGA, 0x1ffff000);
+}
+
+static int
+i915_gem_init_hws(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj_priv;
+        int ret;
+
+        /* If we need a physical address for the status page, it's already
+         * initialized at driver load time.
+         */
+        if (!I915_NEED_GFX_HWS(dev))
+                return 0;
+
+        obj = i915_gem_alloc_object(dev, 4096);
+        if (obj == NULL) {
+                DRM_ERROR("Failed to allocate status page\n");
+                ret = -ENOMEM;
+                goto err;
+        }
+        obj_priv = to_intel_bo(obj);
+        obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
+
+        ret = i915_gem_object_pin(obj, 4096);
+        if (ret != 0) {
+                drm_gem_object_unreference(obj);
+                goto err_unref;
+        }
+
+        dev_priv->status_gfx_addr = obj_priv->gtt_offset;
+
+        dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
+        if (dev_priv->hw_status_page == NULL) {
+                DRM_ERROR("Failed to map status page.\n");
+                memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
+                ret = -EINVAL;
+                goto err_unpin;
+        }
+
+        if (HAS_PIPE_CONTROL(dev)) {
+                ret = i915_gem_init_pipe_control(dev);
+                if (ret)
+                        goto err_unpin;
+        }
+
+        dev_priv->hws_obj = obj;
+        memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
+        if (IS_GEN6(dev)) {
+                I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
+                I915_READ(HWS_PGA_GEN6); /* posting read */
+        } else {
+                I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
+                I915_READ(HWS_PGA); /* posting read */
+        }
+        DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
+
+        return 0;
+
+err_unpin:
+        i915_gem_object_unpin(obj);
+err_unref:
+        drm_gem_object_unreference(obj);
+err:
+        return 0;
+}
+
+int
+i915_gem_init_ringbuffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_gem_object *obj;
+        struct drm_i915_gem_object *obj_priv;
+        drm_i915_ring_buffer_t *ring = &dev_priv->ring;
+        int ret;
+        u32 head;
+
+        ret = i915_gem_init_hws(dev);
+        if (ret != 0)
+                return ret;
+
+        obj = i915_gem_alloc_object(dev, 128 * 1024);
+        if (obj == NULL) {
+                DRM_ERROR("Failed to allocate ringbuffer\n");
+                i915_gem_cleanup_hws(dev);
+                return -ENOMEM;
+        }
+        obj_priv = to_intel_bo(obj);
+
+        ret = i915_gem_object_pin(obj, 4096);
+        if (ret != 0) {
+                drm_gem_object_unreference(obj);
+                i915_gem_cleanup_hws(dev);
+                return ret;
+        }
+
+        /* Set up the kernel mapping for the ring. */
+        ring->Size = obj->size;
+
+        ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
+        ring->map.size = obj->size;
+        ring->map.type = 0;
+        ring->map.flags = 0;
+        ring->map.mtrr = 0;
+
+        drm_core_ioremap_wc(&ring->map, dev);
+        if (ring->map.handle == NULL) {
+                DRM_ERROR("Failed to map ringbuffer.\n");
+                memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
+                i915_gem_object_unpin(obj);
+                drm_gem_object_unreference(obj);
+                i915_gem_cleanup_hws(dev);
+                return -EINVAL;
+        }
+        ring->ring_obj = obj;
+        ring->virtual_start = ring->map.handle;
+
+        /* Stop the ring if it's running. */
+        I915_WRITE(PRB0_CTL, 0);
+        I915_WRITE(PRB0_TAIL, 0);
+        I915_WRITE(PRB0_HEAD, 0);
+
+        /* Initialize the ring. */
+        I915_WRITE(PRB0_START, obj_priv->gtt_offset);
+        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+
+        /* G45 ring initialization fails to reset head to zero */
+        if (head != 0) {
+                DRM_ERROR("Ring head not reset to zero "
+                          "ctl %08x head %08x tail %08x start %08x\n",
+                          I915_READ(PRB0_CTL),
+                          I915_READ(PRB0_HEAD),
+                          I915_READ(PRB0_TAIL),
+                          I915_READ(PRB0_START));
+                I915_WRITE(PRB0_HEAD, 0);
+
+                DRM_ERROR("Ring head forced to zero "
+                          "ctl %08x head %08x tail %08x start %08x\n",
+                          I915_READ(PRB0_CTL),
+                          I915_READ(PRB0_HEAD),
+                          I915_READ(PRB0_TAIL),
+                          I915_READ(PRB0_START));
+        }
+
+        I915_WRITE(PRB0_CTL,
+                   ((obj->size - 4096) & RING_NR_PAGES) |
+                   RING_NO_REPORT |
+                   RING_VALID);
+
+        head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+
+        /* If the head is still not zero, the ring is dead */
+        if (head != 0) {
+                DRM_ERROR("Ring initialization failed "
+                          "ctl %08x head %08x tail %08x start %08x\n",
+                          I915_READ(PRB0_CTL),
+                          I915_READ(PRB0_HEAD),
+                          I915_READ(PRB0_TAIL),
+                          I915_READ(PRB0_START));
+                return -EIO;
+        }
+
+        /* Update our cache of the ring state */
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                i915_kernel_lost_context(dev);
+        else {
+                ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+                ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
+                ring->space = ring->head - (ring->tail + 8);
+                if (ring->space < 0)
+                        ring->space += ring->Size;
+        }
+
+        if (IS_I9XX(dev) && !IS_GEN3(dev)) {
+                I915_WRITE(MI_MODE,
+                           (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
+        }
+
+        return 0;
+}
+
+void
+i915_gem_cleanup_ringbuffer(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+
+        if (dev_priv->ring.ring_obj == NULL)
+                return;
+
+        drm_core_ioremapfree(&dev_priv->ring.map, dev);
+
+        i915_gem_object_unpin(dev_priv->ring.ring_obj);
+        drm_gem_object_unreference(dev_priv->ring.ring_obj);
+        dev_priv->ring.ring_obj = NULL;
+        memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
+
+        i915_gem_cleanup_hws(dev);
+}
+
+/* As a ringbuffer is only allowed to wrap between instructions, fill
+ * the tail with NOOPs.
+ */
+int i915_wrap_ring(struct drm_device *dev)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        volatile unsigned int *virt;
+        int rem;
+
+        rem = dev_priv->ring.Size - dev_priv->ring.tail;
+        if (dev_priv->ring.space < rem) {
+                int ret = i915_wait_ring(dev, rem, __func__);
+                if (ret)
+                        return ret;
+        }
+        dev_priv->ring.space -= rem;
+
+        virt = (unsigned int *)
+                (dev_priv->ring.virtual_start + dev_priv->ring.tail);
+        rem /= 4;
+        while (rem--)
+                *virt++ = MI_NOOP;
+
+        dev_priv->ring.tail = 0;
+
+        return 0;
+}
+
+int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
+{
+        drm_i915_private_t *dev_priv = dev->dev_private;
+        drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
+        u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
+        u32 last_acthd = I915_READ(acthd_reg);
+        u32 acthd;
+        u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+        int i;
+
+        trace_i915_ring_wait_begin (dev);
+
+        for (i = 0; i < 100000; i++) {
+                ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+                acthd = I915_READ(acthd_reg);
+                ring->space = ring->head - (ring->tail + 8);
+                if (ring->space < 0)
+                        ring->space += ring->Size;
+                if (ring->space >= n) {
+                        trace_i915_ring_wait_end (dev);
+                        return 0;
+                }
+
+                if (dev->primary->master) {
+                        struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
+                        if (master_priv->sarea_priv)
+                                master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
+                }
+
+
+                if (ring->head != last_head)
+                        i = 0;
+                if (acthd != last_acthd)
+                        i = 0;
+
+                last_head = ring->head;
+                last_acthd = acthd;
+                msleep_interruptible(10);
+
+        }
+
+        trace_i915_ring_wait_end (dev);
+        return -EBUSY;
+}