1 files changed, 110 insertions, 46 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index c79c0b62ef60..3dfc848ff755 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -35,18 +35,18 @@
 #include <linux/swap.h>
 #include <linux/pci.h>
 
-static void i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
-static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
-                                             bool write);
-static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
-                                                     uint64_t offset,
-                                                     uint64_t size);
+static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
+                                                          bool write);
+static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
+                                                                  uint64_t offset,
+                                                                  uint64_t size);
 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
-static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
-                                       unsigned alignment,
-                                       bool map_and_fenceable);
+static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+                                                    unsigned alignment,
+                                                    bool map_and_fenceable);
 static void i915_gem_clear_fence_reg(struct drm_device *dev,
                                      struct drm_i915_fence_reg *reg);
 static int i915_gem_phys_pwrite(struct drm_device *dev,
@@ -1935,6 +1935,8 @@ i915_gem_retire_work_handler(struct work_struct *work)
 {
         drm_i915_private_t *dev_priv;
         struct drm_device *dev;
+        bool idle;
+        int i;
 
         dev_priv = container_of(work, drm_i915_private_t,
                                 mm.retire_work.work);
@@ -1948,11 +1950,31 @@ i915_gem_retire_work_handler(struct work_struct *work)
 
         i915_gem_retire_requests(dev);
 
-        if (!dev_priv->mm.suspended &&
-                (!list_empty(&dev_priv->ring[RCS].request_list) ||
-                 !list_empty(&dev_priv->ring[VCS].request_list) ||
-                 !list_empty(&dev_priv->ring[BCS].request_list)))
+        /* Send a periodic flush down the ring so we don't hold onto GEM
+         * objects indefinitely.
+         */
+        idle = true;
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct intel_ring_buffer *ring = &dev_priv->ring[i];
+
+                if (!list_empty(&ring->gpu_write_list)) {
+                        struct drm_i915_gem_request *request;
+                        int ret;
+
+                        ret = i915_gem_flush_ring(dev, ring, 0,
+                                                  I915_GEM_GPU_DOMAINS);
+                        request = kzalloc(sizeof(*request), GFP_KERNEL);
+                        if (ret || request == NULL ||
+                            i915_add_request(dev, NULL, request, ring))
+                            kfree(request);
+                }
+
+                idle &= list_empty(&ring->request_list);
+        }
+
+        if (!dev_priv->mm.suspended && !idle)
                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -2142,25 +2164,37 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
         return ret;
 }
 
-void
+int
 i915_gem_flush_ring(struct drm_device *dev,
                     struct intel_ring_buffer *ring,
                     uint32_t invalidate_domains,
                     uint32_t flush_domains)
 {
-        ring->flush(ring, invalidate_domains, flush_domains);
+        int ret;
+
+        ret = ring->flush(ring, invalidate_domains, flush_domains);
+        if (ret)
+                return ret;
+
         i915_gem_process_flushing_list(dev, flush_domains, ring);
+        return 0;
 }
 
 static int i915_ring_idle(struct drm_device *dev,
                           struct intel_ring_buffer *ring)
 {
+        int ret;
+
         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
                 return 0;
 
-        if (!list_empty(&ring->gpu_write_list))
-                i915_gem_flush_ring(dev, ring,
+        if (!list_empty(&ring->gpu_write_list)) {
+                ret = i915_gem_flush_ring(dev, ring,
                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+                if (ret)
+                        return ret;
+        }
+
         return i915_wait_request(dev,
                                  i915_gem_next_request_seqno(dev, ring),
                                  ring);
@@ -2370,10 +2404,13 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
         int ret;
 
         if (obj->fenced_gpu_access) {
-                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
-                        i915_gem_flush_ring(obj->base.dev,
-                                            obj->last_fenced_ring,
-                                            0, obj->base.write_domain);
+                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                        ret = i915_gem_flush_ring(obj->base.dev,
+                                                  obj->last_fenced_ring,
+                                                  0, obj->base.write_domain);
+                        if (ret)
+                                return ret;
+                }
 
                 obj->fenced_gpu_access = false;
         }
@@ -2393,6 +2430,12 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
                 obj->last_fenced_ring = NULL;
         }
 
+        /* Ensure that all CPU reads are completed before installing a fence
+         * and all writes before removing the fence.
+         */
+        if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
+                mb();
+
         return 0;
 }
 
@@ -2523,9 +2566,12 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
                                 return ret;
                 } else if (obj->tiling_changed) {
                         if (obj->fenced_gpu_access) {
-                                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
-                                        i915_gem_flush_ring(obj->base.dev, obj->ring,
-                                                            0, obj->base.write_domain);
+                                if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                                        ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+                                                                  0, obj->base.write_domain);
+                                        if (ret)
+                                                return ret;
+                                }
 
                                 obj->fenced_gpu_access = false;
                         }
@@ -2736,10 +2782,8 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 obj->gtt_space = NULL;
 
                 if (ret == -ENOMEM) {
-                        /* first try to clear up some space from the GTT */
-                        ret = i915_gem_evict_something(dev, size,
-                                                       alignment,
-                                                       map_and_fenceable);
+                        /* first try to reclaim some memory by clearing the GTT */
+                        ret = i915_gem_evict_everything(dev, false);
                         if (ret) {
                                 /* now try to shrink everyone else */
                                 if (gfpmask) {
@@ -2747,7 +2791,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                                         goto search_free;
                                 }
 
-                                return ret;
+                                return -ENOMEM;
                         }
 
                         goto search_free;
@@ -2762,9 +2806,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
                 drm_mm_put_block(obj->gtt_space);
                 obj->gtt_space = NULL;
 
-                ret = i915_gem_evict_something(dev, size,
-                                               alignment, map_and_fenceable);
-                if (ret)
+                if (i915_gem_evict_everything(dev, false))
                         return ret;
 
                 goto search_free;
@@ -2811,17 +2853,16 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj)
 }
 
 /** Flushes any GPU write domain for the object if it's dirty. */
-static void
+static int
 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
 {
         struct drm_device *dev = obj->base.dev;
 
         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
-                return;
+                return 0;
 
         /* Queue the GPU write cache flushing we need. */
-        i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
-        BUG_ON(obj->base.write_domain);
+        return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
@@ -2833,10 +2874,16 @@ i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
                 return;
 
-        /* No actual flushing is required for the GTT write domain.   Writes
+        /* No actual flushing is required for the GTT write domain.  Writes
          * to it immediately go to main memory as far as we know, so there's
          * no chipset flush.  It also doesn't land in render cache.
+         *
+         * However, we do have to enforce the order so that all writes through
+         * the GTT land before any writes to the device, such as updates to
+         * the GATT itself.
          */
+        wmb();
+
         i915_gem_release_mmap(obj);
 
         old_write_domain = obj->base.write_domain;
@@ -2882,7 +2929,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
         if (obj->gtt_space == NULL)
                 return -EINVAL;
 
-        i915_gem_object_flush_gpu_write_domain(obj);
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
+                return ret;
+
         if (obj->pending_gpu_write || write) {
                 ret = i915_gem_object_wait_rendering(obj, true);
                 if (ret)
@@ -2927,7 +2977,10 @@ i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
         if (obj->gtt_space == NULL)
                 return -EINVAL;
 
-        i915_gem_object_flush_gpu_write_domain(obj);
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
+                return ret;
+
 
         /* Currently, we are always called from an non-interruptible context. */
         if (pipelined != obj->ring) {
@@ -2952,12 +3005,17 @@ int
 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
                           bool interruptible)
 {
+        int ret;
+
         if (!obj->active)
                 return 0;
 
-        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
-                i915_gem_flush_ring(obj->base.dev, obj->ring,
-                                    0, obj->base.write_domain);
+        if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+                ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+                                          0, obj->base.write_domain);
+                if (ret)
+                        return ret;
+        }
 
         return i915_gem_object_wait_rendering(obj, interruptible);
 }
@@ -2974,7 +3032,10 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
-        i915_gem_object_flush_gpu_write_domain(obj);
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
+                return ret;
+
         ret = i915_gem_object_wait_rendering(obj, true);
         if (ret)
                 return ret;
@@ -3069,7 +3130,10 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
         if (offset == 0 && size == obj->base.size)
                 return i915_gem_object_set_to_cpu_domain(obj, 0);
 
-        i915_gem_object_flush_gpu_write_domain(obj);
+        ret = i915_gem_object_flush_gpu_write_domain(obj);
+        if (ret)
+                return ret;
+
         ret = i915_gem_object_wait_rendering(obj, true);
         if (ret)
                 return ret;
@@ -3362,8 +3426,8 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
                  * flush earlier is beneficial.
                  */
                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
-                        i915_gem_flush_ring(dev, obj->ring,
-                                            0, obj->base.write_domain);
+                        ret = i915_gem_flush_ring(dev, obj->ring,
+                                                  0, obj->base.write_domain);
                 } else if (obj->ring->outstanding_lazy_request ==
                            obj->last_rendering_seqno) {
                         struct drm_i915_gem_request *request;