1 files changed, 224 insertions, 69 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 9e35dafc5807..4200c32407ec 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -176,7 +176,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 
         pinned = 0;
         mutex_lock(&dev->struct_mutex);
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
                 if (obj->pin_count)
                         pinned += obj->gtt_space->size;
         mutex_unlock(&dev->struct_mutex);
@@ -956,7 +956,7 @@ i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
 
         ret = 0;
         if (seqno == ring->outstanding_lazy_request)
-                ret = i915_add_request(ring, NULL, NULL);
+                ret = i915_add_request(ring, NULL);
 
         return ret;
 }
@@ -1087,6 +1087,25 @@ i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
                             interruptible, NULL);
 }
 
+static int
+i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
+                                     struct intel_ring_buffer *ring)
+{
+        i915_gem_retire_requests_ring(ring);
+
+        /* Manually manage the write flush as we may have not yet
+         * retired the buffer.
+         *
+         * Note that the last_write_seqno is always the earlier of
+         * the two (read/write) seqno, so if we haved successfully waited,
+         * we know we have passed the last write.
+         */
+        obj->last_write_seqno = 0;
+        obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
+
+        return 0;
+}
+
 /**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
@@ -1107,18 +1126,7 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
         if (ret)
                 return ret;
 
-        i915_gem_retire_requests_ring(ring);
-
-        /* Manually manage the write flush as we may have not yet
-         * retired the buffer.
-         */
-        if (obj->last_write_seqno &&
-            i915_seqno_passed(seqno, obj->last_write_seqno)) {
-                obj->last_write_seqno = 0;
-                obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-        }
-
-        return 0;
+        return i915_gem_object_wait_rendering__tail(obj, ring);
 }
 
 /* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1154,19 +1162,10 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
         mutex_unlock(&dev->struct_mutex);
         ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
         mutex_lock(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-        i915_gem_retire_requests_ring(ring);
-
-        /* Manually manage the write flush as we may have not yet
-         * retired the buffer.
-         */
-        if (obj->last_write_seqno &&
-            i915_seqno_passed(seqno, obj->last_write_seqno)) {
-                obj->last_write_seqno = 0;
-                obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
-        }
-
-        return ret;
+        return i915_gem_object_wait_rendering__tail(obj, ring);
 }
 
 /**
@@ -1676,7 +1675,7 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
         /* ->put_pages might need to allocate memory for the bit17 swizzle
          * array, hence protect them from being reaped by removing them from gtt
          * lists early. */
-        list_del(&obj->gtt_list);
+        list_del(&obj->global_list);
 
         ops->put_pages(obj);
         obj->pages = NULL;
@@ -1696,7 +1695,7 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
 
         list_for_each_entry_safe(obj, next,
                                  &dev_priv->mm.unbound_list,
-                                 gtt_list) {
+                                 global_list) {
                 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
                     i915_gem_object_put_pages(obj) == 0) {
                         count += obj->base.size >> PAGE_SHIFT;
@@ -1733,7 +1732,8 @@ i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 
         i915_gem_evict_everything(dev_priv->dev);
 
-        list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
+        list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
+                                 global_list)
                 i915_gem_object_put_pages(obj);
 }
 
@@ -1867,7 +1867,7 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
         if (ret)
                 return ret;
 
-        list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+        list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
         return 0;
 }
 
@@ -2005,17 +2005,18 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
         return 0;
 }
 
-int
-i915_add_request(struct intel_ring_buffer *ring,
-                 struct drm_file *file,
-                 u32 *out_seqno)
+int __i915_add_request(struct intel_ring_buffer *ring,
+                       struct drm_file *file,
+                       struct drm_i915_gem_object *obj,
+                       u32 *out_seqno)
 {
         drm_i915_private_t *dev_priv = ring->dev->dev_private;
         struct drm_i915_gem_request *request;
-        u32 request_ring_position;
+        u32 request_ring_position, request_start;
         int was_empty;
         int ret;
 
+        request_start = intel_ring_get_tail(ring);
         /*
          * Emit any outstanding flushes - execbuf can fail to emit the flush
          * after having emitted the batchbuffer command. Hence we need to fix
@@ -2047,7 +2048,21 @@ i915_add_request(struct intel_ring_buffer *ring,
 
         request->seqno = intel_ring_get_seqno(ring);
         request->ring = ring;
+        request->head = request_start;
         request->tail = request_ring_position;
+        request->ctx = ring->last_context;
+        request->batch_obj = obj;
+
+        /* Whilst this request exists, batch_obj will be on the
+         * active_list, and so will hold the active reference. Only when this
+         * request is retired will the the batch_obj be moved onto the
+         * inactive_list and lose its active reference. Hence we do not need
+         * to explicitly hold another reference here.
+         */
+
+        if (request->ctx)
+                i915_gem_context_reference(request->ctx);
+
         request->emitted_jiffies = jiffies;
         was_empty = list_empty(&ring->request_list);
         list_add_tail(&request->list, &ring->request_list);
@@ -2100,9 +2115,114 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
         spin_unlock(&file_priv->mm.lock);
 }
 
+static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
+{
+        if (acthd >= obj->gtt_offset &&
+            acthd < obj->gtt_offset + obj->base.size)
+                return true;
+
+        return false;
+}
+
+static bool i915_head_inside_request(const u32 acthd_unmasked,
+                                     const u32 request_start,
+                                     const u32 request_end)
+{
+        const u32 acthd = acthd_unmasked & HEAD_ADDR;
+
+        if (request_start < request_end) {
+                if (acthd >= request_start && acthd < request_end)
+                        return true;
+        } else if (request_start > request_end) {
+                if (acthd >= request_start || acthd < request_end)
+                        return true;
+        }
+
+        return false;
+}
+
+static bool i915_request_guilty(struct drm_i915_gem_request *request,
+                                const u32 acthd, bool *inside)
+{
+        /* There is a possibility that unmasked head address
+         * pointing inside the ring, matches the batch_obj address range.
+         * However this is extremely unlikely.
+         */
+
+        if (request->batch_obj) {
+                if (i915_head_inside_object(acthd, request->batch_obj)) {
+                        *inside = true;
+                        return true;
+                }
+        }
+
+        if (i915_head_inside_request(acthd, request->head, request->tail)) {
+                *inside = false;
+                return true;
+        }
+
+        return false;
+}
+
+static void i915_set_reset_status(struct intel_ring_buffer *ring,
+                                  struct drm_i915_gem_request *request,
+                                  u32 acthd)
+{
+        struct i915_ctx_hang_stats *hs = NULL;
+        bool inside, guilty;
+
+        /* Innocent until proven guilty */
+        guilty = false;
+
+        if (ring->hangcheck.action != wait &&
+            i915_request_guilty(request, acthd, &inside)) {
+                DRM_ERROR("%s hung %s bo (0x%x ctx %d) at 0x%x\n",
+                          ring->name,
+                          inside ? "inside" : "flushing",
+                          request->batch_obj ?
+                          request->batch_obj->gtt_offset : 0,
+                          request->ctx ? request->ctx->id : 0,
+                          acthd);
+
+                guilty = true;
+        }
+
+        /* If contexts are disabled or this is the default context, use
+         * file_priv->reset_state
+         */
+        if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
+                hs = &request->ctx->hang_stats;
+        else if (request->file_priv)
+                hs = &request->file_priv->hang_stats;
+
+        if (hs) {
+                if (guilty)
+                        hs->batch_active++;
+                else
+                        hs->batch_pending++;
+        }
+}
+
+static void i915_gem_free_request(struct drm_i915_gem_request *request)
+{
+        list_del(&request->list);
+        i915_gem_request_remove_from_client(request);
+
+        if (request->ctx)
+                i915_gem_context_unreference(request->ctx);
+
+        kfree(request);
+}
+
 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
                                       struct intel_ring_buffer *ring)
 {
+        u32 completed_seqno;
+        u32 acthd;
+
+        acthd = intel_ring_get_active_head(ring);
+        completed_seqno = ring->get_seqno(ring, false);
+
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
 
@@ -2110,9 +2230,10 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
                                            struct drm_i915_gem_request,
                                            list);
 
-                list_del(&request->list);
-                i915_gem_request_remove_from_client(request);
-                kfree(request);
+                if (request->seqno > completed_seqno)
+                        i915_set_reset_status(ring, request, acthd);
+
+                i915_gem_free_request(request);
         }
 
         while (!list_empty(&ring->active_list)) {
@@ -2193,9 +2314,7 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
                  */
                 ring->last_retired_head = request->tail;
 
-                list_del(&request->list);
-                i915_gem_request_remove_from_client(request);
-                kfree(request);
+                i915_gem_free_request(request);
         }
 
         /* Move any buffers on the active list that are no longer referenced
@@ -2262,7 +2381,7 @@ i915_gem_retire_work_handler(struct work_struct *work)
         idle = true;
         for_each_ring(ring, dev_priv, i) {
                 if (ring->gpu_caches_dirty)
-                        i915_add_request(ring, NULL, NULL);
+                        i915_add_request(ring, NULL);
 
                 idle &= list_empty(&ring->request_list);
         }
@@ -2494,9 +2613,10 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
                 obj->has_aliasing_ppgtt_mapping = 0;
         }
         i915_gem_gtt_finish_object(obj);
+        i915_gem_object_unpin_pages(obj);
 
         list_del(&obj->mm_list);
-        list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+        list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
         /* Avoid an unnecessary call to unbind on rebind. */
         obj->map_and_fenceable = true;
 
@@ -2676,18 +2796,33 @@ static inline int fence_number(struct drm_i915_private *dev_priv,
         return fence - dev_priv->fence_regs;
 }
 
+struct write_fence {
+        struct drm_device *dev;
+        struct drm_i915_gem_object *obj;
+        int fence;
+};
+
 static void i915_gem_write_fence__ipi(void *data)
 {
+        struct write_fence *args = data;
+
+        /* Required for SNB+ with LLC */
         wbinvd();
+
+        /* Required for VLV */
+        i915_gem_write_fence(args->dev, args->fence, args->obj);
 }
 
 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
                                          struct drm_i915_fence_reg *fence,
                                          bool enable)
 {
-        struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int fence_reg = fence_number(dev_priv, fence);
+        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct write_fence args = {
+                .dev = obj->base.dev,
+                .fence = fence_number(dev_priv, fence),
+                .obj = enable ? obj : NULL,
+        };
 
         /* In order to fully serialize access to the fenced region and
          * the update to the fence register we need to take extreme
@@ -2698,13 +2833,19 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
          * SNB+ we need to take a step further and emit an explicit wbinvd()
          * on each processor in order to manually flush all memory
          * transactions before updating the fence register.
+         *
+         * However, Valleyview complicates matter. There the wbinvd is
+         * insufficient and unlike SNB/IVB requires the serialising
+         * register write. (Note that that register write by itself is
+         * conversely not sufficient for SNB+.) To compromise, we do both.
          */
-        if (HAS_LLC(obj->base.dev))
-                on_each_cpu(i915_gem_write_fence__ipi, NULL, 1);
-        i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL);
+        if (INTEL_INFO(args.dev)->gen >= 6)
+                on_each_cpu(i915_gem_write_fence__ipi, &args, 1);
+        else
+                i915_gem_write_fence(args.dev, args.fence, args.obj);
 
         if (enable) {
-                obj->fence_reg = fence_reg;
+                obj->fence_reg = args.fence;
                 fence->obj = obj;
                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
         } else {
@@ -2883,7 +3024,7 @@ static void i915_gem_verify_gtt(struct drm_device *dev)
         struct drm_i915_gem_object *obj;
         int err = 0;
 
-        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
                 if (obj->gtt_space == NULL) {
                         printk(KERN_ERR "object found on GTT list with no space reserved\n");
                         err++;
@@ -2930,6 +3071,8 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
         struct drm_mm_node *node;
         u32 size, fence_size, fence_alignment, unfenced_alignment;
         bool mappable, fenceable;
+        size_t gtt_max = map_and_fenceable ?
+                dev_priv->gtt.mappable_end : dev_priv->gtt.total;
         int ret;
 
         fence_size = i915_gem_get_gtt_size(dev,
@@ -2956,9 +3099,11 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
         /* If the object is bigger than the entire aperture, reject it early
          * before evicting everything in a vain attempt to find space.
          */
-        if (obj->base.size >
-            (map_and_fenceable ? dev_priv->gtt.mappable_end : dev_priv->gtt.total)) {
-                DRM_ERROR("Attempting to bind an object larger than the aperture\n");
+        if (obj->base.size > gtt_max) {
+                DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
+                          obj->base.size,
+                          map_and_fenceable ? "mappable" : "total",
+                          gtt_max);
                 return -E2BIG;
         }
 
@@ -2974,14 +3119,10 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 return -ENOMEM;
         }
 
- search_free:
-        if (map_and_fenceable)
-                ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
-                                                          size, alignment, obj->cache_level,
-                                                          0, dev_priv->gtt.mappable_end);
-        else
-                ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node,
-                                                 size, alignment, obj->cache_level);
+search_free:
+        ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
+                                                  size, alignment,
+                                                  obj->cache_level, 0, gtt_max);
         if (ret) {
                 ret = i915_gem_evict_something(dev, size, alignment,
                                                obj->cache_level,
@@ -3007,7 +3148,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 return ret;
         }
 
-        list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
+        list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
         obj->gtt_space = node;
@@ -3022,7 +3163,6 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 
         obj->map_and_fenceable = mappable && fenceable;
 
-        i915_gem_object_unpin_pages(obj);
         trace_i915_gem_object_bind(obj, map_and_fenceable);
         i915_gem_verify_gtt(dev);
         return 0;
@@ -3722,7 +3862,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
                           const struct drm_i915_gem_object_ops *ops)
 {
         INIT_LIST_HEAD(&obj->mm_list);
-        INIT_LIST_HEAD(&obj->gtt_list);
+        INIT_LIST_HEAD(&obj->global_list);
         INIT_LIST_HEAD(&obj->ring_list);
         INIT_LIST_HEAD(&obj->exec_list);
 
@@ -3822,7 +3962,13 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
                 dev_priv->mm.interruptible = was_interruptible;
         }
 
-        obj->pages_pin_count = 0;
+        /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
+         * before progressing. */
+        if (obj->stolen)
+                i915_gem_object_unpin_pages(obj);
+
+        if (WARN_ON(obj->pages_pin_count))
+                obj->pages_pin_count = 0;
         i915_gem_object_put_pages(obj);
         i915_gem_object_free_mmap_offset(obj);
         i915_gem_object_release_stolen(obj);
@@ -3973,12 +4119,21 @@ static int i915_gem_init_rings(struct drm_device *dev)
                         goto cleanup_bsd_ring;
         }
 
+        if (HAS_VEBOX(dev)) {
+                ret = intel_init_vebox_ring_buffer(dev);
+                if (ret)
+                        goto cleanup_blt_ring;
+        }
+
+
         ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
         if (ret)
-                goto cleanup_blt_ring;
+                goto cleanup_vebox_ring;
 
         return 0;
 
+cleanup_vebox_ring:
+        intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
 cleanup_blt_ring:
         intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
 cleanup_bsd_ring:
@@ -4453,10 +4608,10 @@ i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
         }
 
         cnt = 0;
-        list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
                 if (obj->pages_pin_count == 0)
                         cnt += obj->base.size >> PAGE_SHIFT;
-        list_for_each_entry(obj, &dev_priv->mm.inactive_list, gtt_list)
+        list_for_each_entry(obj, &dev_priv->mm.inactive_list, global_list)
                 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
                         cnt += obj->base.size >> PAGE_SHIFT;