drm/i915: Segregate memory domains in the GTT using coloring

Several functions of the GPU have the restriction that differing memory
domains cannot be placed next to each other (as the GPU may prefetch
beyond the end of one domain and hang as it crosses into the other
domain). We use the facility of the drm_mm to mark ranges with a
particular color that corresponds to the cache attributes of those pages
in order to prevent allocating adjacent blocks of differing memory
types.

v2: Rebase ontop of drm_mm coloring v2.
v3: Fix rebinding existing gtt_space and add a verification routine.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

1 files changed, 101 insertions, 10 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index b274810eaeab..19bdc245a87a 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2586,6 +2586,76 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+static bool i915_gem_valid_gtt_space(struct drm_device *dev,
+                                     struct drm_mm_node *gtt_space,
+                                     unsigned long cache_level)
+{
+        struct drm_mm_node *other;
+
+        /* On non-LLC machines we have to be careful when putting differing
+         * types of snoopable memory together to avoid the prefetcher
+         * crossing memory domains and dieing.
+         */
+        if (HAS_LLC(dev))
+                return true;
+
+        if (gtt_space == NULL)
+                return true;
+
+        if (list_empty(&gtt_space->node_list))
+                return true;
+
+        other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
+        if (other->allocated && !other->hole_follows && other->color != cache_level)
+                return false;
+
+        other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
+        if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
+                return false;
+
+        return true;
+}
+
+static void i915_gem_verify_gtt(struct drm_device *dev)
+{
+#if WATCH_GTT
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *obj;
+        int err = 0;
+
+        list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+                if (obj->gtt_space == NULL) {
+                        printk(KERN_ERR "object found on GTT list with no space reserved\n");
+                        err++;
+                        continue;
+                }
+
+                if (obj->cache_level != obj->gtt_space->color) {
+                        printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
+                               obj->gtt_space->start,
+                               obj->gtt_space->start + obj->gtt_space->size,
+                               obj->cache_level,
+                               obj->gtt_space->color);
+                        err++;
+                        continue;
+                }
+
+                if (!i915_gem_valid_gtt_space(dev,
+                                              obj->gtt_space,
+                                              obj->cache_level)) {
+                        printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
+                               obj->gtt_space->start,
+                               obj->gtt_space->start + obj->gtt_space->size,
+                               obj->cache_level);
+                        err++;
+                        continue;
+                }
+        }
+
+        WARN_ON(err);
+#endif
+}
+
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
@@ -2640,36 +2710,47 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  search_free:
         if (map_and_fenceable)
                 free_space =
-                        drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
-                                                    size, alignment,
-                                                    0, dev_priv->mm.gtt_mappable_end,
-                                                    0);
+                        drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
+                                                          size, alignment, obj->cache_level,
+                                                          0, dev_priv->mm.gtt_mappable_end,
+                                                          false);
         else
-                free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
-                                                size, alignment, 0);
+                free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
+                                                      size, alignment, obj->cache_level,
+                                                      false);
 
         if (free_space != NULL) {
                 if (map_and_fenceable)
                         obj->gtt_space =
                                 drm_mm_get_block_range_generic(free_space,
-                                                               size, alignment, 0,
+                                                               size, alignment, obj->cache_level,
                                                                0, dev_priv->mm.gtt_mappable_end,
-                                                               0);
+                                                               false);
                 else
                         obj->gtt_space =
-                                drm_mm_get_block(free_space, size, alignment);
+                                drm_mm_get_block_generic(free_space,
+                                                         size, alignment, obj->cache_level,
+                                                         false);
         }
         if (obj->gtt_space == NULL) {
                 /* If the gtt is empty and we're still having trouble
                  * fitting our object in, we're out of memory.
                  */
                 ret = i915_gem_evict_something(dev, size, alignment,
+                                               obj->cache_level,
                                                map_and_fenceable);
                 if (ret)
                         return ret;
 
                 goto search_free;
         }
+        if (WARN_ON(!i915_gem_valid_gtt_space(dev,
+                                              obj->gtt_space,
+                                              obj->cache_level))) {
+                drm_mm_put_block(obj->gtt_space);
+                obj->gtt_space = NULL;
+                return -EINVAL;
+        }
 
         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
         if (ret) {
@@ -2732,6 +2813,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
         obj->map_and_fenceable = mappable && fenceable;
 
         trace_i915_gem_object_bind(obj, map_and_fenceable);
+        i915_gem_verify_gtt(dev);
         return 0;
 }
 
@@ -2873,6 +2955,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                 return -EBUSY;
         }
 
+        if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
+                ret = i915_gem_object_unbind(obj);
+                if (ret)
+                        return ret;
+        }
+
         if (obj->gtt_space) {
                 ret = i915_gem_object_finish_gpu(obj);
                 if (ret)
@@ -2884,7 +2972,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                  * registers with snooped memory, so relinquish any fences
                  * currently pointing to our region in the aperture.
                  */
-                if (INTEL_INFO(obj->base.dev)->gen < 6) {
+                if (INTEL_INFO(dev)->gen < 6) {
                         ret = i915_gem_object_put_fence(obj);
                         if (ret)
                                 return ret;
@@ -2895,6 +2983,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                 if (obj->has_aliasing_ppgtt_mapping)
                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
                                                obj, cache_level);
+
+                obj->gtt_space->color = cache_level;
         }
 
         if (cache_level == I915_CACHE_NONE) {
@@ -2921,6 +3011,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
         }
 
         obj->cache_level = cache_level;
+        i915_gem_verify_gtt(dev);
         return 0;
 }