1 files changed, 93 insertions, 261 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 0758c7802e6..df5a7135c26 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -35,6 +35,7 @@
 #include <linux/swap.h>
 #include <linux/pci.h>
 
+static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
@@ -48,8 +49,6 @@ static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
                                            unsigned alignment);
 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
-static int i915_gem_evict_something(struct drm_device *dev, int min_size);
-static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
                                 struct drm_i915_gem_pwrite *args,
                                 struct drm_file *file_priv);
@@ -58,6 +57,14 @@ static void i915_gem_free_object_tail(struct drm_gem_object *obj);
 static LIST_HEAD(shrink_list);
 static DEFINE_SPINLOCK(shrink_list_lock);
 
+static inline bool
+i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
+{
+        return obj_priv->gtt_space &&
+                !obj_priv->active &&
+                obj_priv->pin_count == 0;
+}
+
 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
                      unsigned long end)
 {
@@ -313,7 +320,8 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
         if (ret == -ENOMEM) {
                 struct drm_device *dev = obj->dev;
 
-                ret = i915_gem_evict_something(dev, obj->size);
+                ret = i915_gem_evict_something(dev, obj->size,
+                                               i915_gem_get_gtt_alignment(obj));
                 if (ret)
                         return ret;
 
@@ -1036,6 +1044,11 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
         }
 
+        
+        /* Maintain LRU order of "inactive" objects */
+        if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
+                list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
         drm_gem_object_unreference(obj);
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -1137,7 +1150,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
         struct drm_gem_object *obj = vma->vm_private_data;
         struct drm_device *dev = obj->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         pgoff_t page_offset;
         unsigned long pfn;
@@ -1155,8 +1168,6 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                 if (ret)
                         goto unlock;
 
-                list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
-
                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
                 if (ret)
                         goto unlock;
@@ -1169,6 +1180,9 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                         goto unlock;
         }
 
+        if (i915_gem_object_is_inactive(obj_priv))
+                list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
                 page_offset;
 
@@ -1363,7 +1377,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *file_priv)
 {
         struct drm_i915_gem_mmap_gtt *args = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_gem_object *obj;
         struct drm_i915_gem_object *obj_priv;
         int ret;
@@ -1409,7 +1422,6 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
                         mutex_unlock(&dev->struct_mutex);
                         return ret;
                 }
-                list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
         }
 
         drm_gem_object_unreference(obj);
@@ -1493,9 +1505,16 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
         struct inode *inode;
 
+        /* Our goal here is to return as much of the memory as
+         * is possible back to the system as we are called from OOM.
+         * To do this we must instruct the shmfs to drop all of its
+         * backing pages, *now*. Here we mirror the actions taken
+         * when by shmem_delete_inode() to release the backing store.
+         */
         inode = obj->filp->f_path.dentry->d_inode;
-        if (inode->i_op->truncate)
-                inode->i_op->truncate (inode);
+        truncate_inode_pages(inode->i_mapping, 0);
+        if (inode->i_op->truncate_range)
+                inode->i_op->truncate_range(inode, 0, (loff_t)-1);
 
         obj_priv->madv = __I915_MADV_PURGED;
 }
@@ -1887,19 +1906,6 @@ i915_gem_flush(struct drm_device *dev,
                                 flush_domains);
 }
 
-static void
-i915_gem_flush_ring(struct drm_device *dev,
-               uint32_t invalidate_domains,
-               uint32_t flush_domains,
-               struct intel_ring_buffer *ring)
-{
-        if (flush_domains & I915_GEM_DOMAIN_CPU)
-                drm_agp_chipset_flush(dev);
-        ring->flush(dev, ring,
-                        invalidate_domains,
-                        flush_domains);
-}
-
 /**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
@@ -1973,8 +1979,6 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
          * cause memory corruption through use-after-free.
          */
 
-        BUG_ON(obj_priv->active);
-
         /* release the fence reg _after_ flushing */
         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
                 i915_gem_clear_fence_reg(obj);
@@ -2010,34 +2014,7 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
         return ret;
 }
 
-static struct drm_gem_object *
-i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj_priv;
-        struct drm_gem_object *best = NULL;
-        struct drm_gem_object *first = NULL;
-
-        /* Try to find the smallest clean object */
-        list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
-                struct drm_gem_object *obj = &obj_priv->base;
-                if (obj->size >= min_size) {
-                        if ((!obj_priv->dirty ||
-                             i915_gem_object_is_purgeable(obj_priv)) &&
-                            (!best || obj->size < best->size)) {
-                                best = obj;
-                                if (best->size == min_size)
-                                        return best;
-                        }
-                        if (!first)
-                            first = obj;
-                }
-        }
-
-        return best ? best : first;
-}
-
-static int
+int
 i915_gpu_idle(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
@@ -2078,155 +2055,6 @@ i915_gpu_idle(struct drm_device *dev)
         return ret;
 }
 
-static int
-i915_gem_evict_everything(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
-        bool lists_empty;
-
-        spin_lock(&dev_priv->mm.active_list_lock);
-        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-                       list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->render_ring.active_list) &&
-                       (!HAS_BSD(dev)
-                        || list_empty(&dev_priv->bsd_ring.active_list)));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-
-        if (lists_empty)
-                return -ENOSPC;
-
-        /* Flush everything (on to the inactive lists) and evict */
-        ret = i915_gpu_idle(dev);
-        if (ret)
-                return ret;
-
-        BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
-
-        ret = i915_gem_evict_from_inactive_list(dev);
-        if (ret)
-                return ret;
-
-        spin_lock(&dev_priv->mm.active_list_lock);
-        lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-                       list_empty(&dev_priv->mm.flushing_list) &&
-                       list_empty(&dev_priv->render_ring.active_list) &&
-                       (!HAS_BSD(dev)
-                        || list_empty(&dev_priv->bsd_ring.active_list)));
-        spin_unlock(&dev_priv->mm.active_list_lock);
-        BUG_ON(!lists_empty);
-
-        return 0;
-}
-
-static int
-i915_gem_evict_something(struct drm_device *dev, int min_size)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct drm_gem_object *obj;
-        int ret;
-
-        struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
-        struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
-        for (;;) {
-                i915_gem_retire_requests(dev);
-
-                /* If there's an inactive buffer available now, grab it
-                 * and be done.
-                 */
-                obj = i915_gem_find_inactive_object(dev, min_size);
-                if (obj) {
-                        struct drm_i915_gem_object *obj_priv;
-
-#if WATCH_LRU
-                        DRM_INFO("%s: evicting %p\n", __func__, obj);
-#endif
-                        obj_priv = to_intel_bo(obj);
-                        BUG_ON(obj_priv->pin_count != 0);
-                        BUG_ON(obj_priv->active);
-
-                        /* Wait on the rendering and unbind the buffer. */
-                        return i915_gem_object_unbind(obj);
-                }
-
-                /* If we didn't get anything, but the ring is still processing
-                 * things, wait for the next to finish and hopefully leave us
-                 * a buffer to evict.
-                 */
-                if (!list_empty(&render_ring->request_list)) {
-                        struct drm_i915_gem_request *request;
-
-                        request = list_first_entry(&render_ring->request_list,
-                                                   struct drm_i915_gem_request,
-                                                   list);
-
-                        ret = i915_wait_request(dev,
-                                        request->seqno, request->ring);
-                        if (ret)
-                                return ret;
-
-                        continue;
-                }
-
-                if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
-                        struct drm_i915_gem_request *request;
-
-                        request = list_first_entry(&bsd_ring->request_list,
-                                                   struct drm_i915_gem_request,
-                                                   list);
-
-                        ret = i915_wait_request(dev,
-                                        request->seqno, request->ring);
-                        if (ret)
-                                return ret;
-
-                        continue;
-                }
-
-                /* If we didn't have anything on the request list but there
-                 * are buffers awaiting a flush, emit one and try again.
-                 * When we wait on it, those buffers waiting for that flush
-                 * will get moved to inactive.
-                 */
-                if (!list_empty(&dev_priv->mm.flushing_list)) {
-                        struct drm_i915_gem_object *obj_priv;
-
-                        /* Find an object that we can immediately reuse */
-                        list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
-                                obj = &obj_priv->base;
-                                if (obj->size >= min_size)
-                                        break;
-
-                                obj = NULL;
-                        }
-
-                        if (obj != NULL) {
-                                uint32_t seqno;
-
-                                i915_gem_flush_ring(dev,
-                                               obj->write_domain,
-                                               obj->write_domain,
-                                               obj_priv->ring);
-                                seqno = i915_add_request(dev, NULL,
-                                                obj->write_domain,
-                                                obj_priv->ring);
-                                if (seqno == 0)
-                                        return -ENOMEM;
-                                continue;
-                        }
-                }
-
-                /* If we didn't do any of the above, there's no single buffer
-                 * large enough to swap out for the new one, so just evict
-                 * everything and start again. (This should be rare.)
-                 */
-                if (!list_empty (&dev_priv->mm.inactive_list))
-                        return i915_gem_evict_from_inactive_list(dev);
-                else
-                        return i915_gem_evict_everything(dev);
-        }
-}
-
 int
 i915_gem_object_get_pages(struct drm_gem_object *obj,
                           gfp_t gfpmask)
@@ -2666,7 +2494,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
 #if WATCH_LRU
                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
 #endif
-                ret = i915_gem_evict_something(dev, obj->size);
+                ret = i915_gem_evict_something(dev, obj->size, alignment);
                 if (ret)
                         return ret;
 
@@ -2684,7 +2512,8 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
 
                 if (ret == -ENOMEM) {
                         /* first try to clear up some space from the GTT */
-                        ret = i915_gem_evict_something(dev, obj->size);
+                        ret = i915_gem_evict_something(dev, obj->size,
+                                                       alignment);
                         if (ret) {
                                 /* now try to shrink everyone else */
                                 if (gfpmask) {
@@ -2714,7 +2543,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
                 drm_mm_put_block(obj_priv->gtt_space);
                 obj_priv->gtt_space = NULL;
 
-                ret = i915_gem_evict_something(dev, obj->size);
+                ret = i915_gem_evict_something(dev, obj->size, alignment);
                 if (ret)
                         return ret;
 
@@ -2723,6 +2552,9 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
         atomic_inc(&dev->gtt_count);
         atomic_add(obj->size, &dev->gtt_memory);
 
+        /* keep track of bounds object by adding it to the inactive list */
+        list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
+
         /* Assert that the object is not currently in any GPU domain. As it
          * wasn't in the GTT, there shouldn't be any way it could have been in
          * a GPU cache
@@ -3117,6 +2949,7 @@ static void
 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
 {
         struct drm_device               *dev = obj->dev;
+        drm_i915_private_t              *dev_priv = dev->dev_private;
         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
         uint32_t                        invalidate_domains = 0;
         uint32_t                        flush_domains = 0;
@@ -3179,6 +3012,13 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
                 obj->pending_write_domain = obj->write_domain;
         obj->read_domains = obj->pending_read_domains;
 
+        if (flush_domains & I915_GEM_GPU_DOMAINS) {
+                if (obj_priv->ring == &dev_priv->render_ring)
+                        dev_priv->flush_rings |= FLUSH_RENDER_RING;
+                else if (obj_priv->ring == &dev_priv->bsd_ring)
+                        dev_priv->flush_rings |= FLUSH_BSD_RING;
+        }
+
         dev->invalidate_domains |= invalidate_domains;
         dev->flush_domains |= flush_domains;
 #if WATCH_BUF
@@ -3718,7 +3558,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                 ring = &dev_priv->render_ring;
         }
 
-
         if (args->buffer_count < 1) {
                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
                 return -EINVAL;
@@ -3892,6 +3731,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
          */
         dev->invalidate_domains = 0;
         dev->flush_domains = 0;
+        dev_priv->flush_rings = 0;
 
         for (i = 0; i < args->buffer_count; i++) {
                 struct drm_gem_object *obj = object_list[i];
@@ -3912,16 +3752,14 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                 i915_gem_flush(dev,
                                dev->invalidate_domains,
                                dev->flush_domains);
-                if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
+                if (dev_priv->flush_rings & FLUSH_RENDER_RING)
                         (void)i915_add_request(dev, file_priv,
-                                        dev->flush_domains,
-                                        &dev_priv->render_ring);
-
-                        if (HAS_BSD(dev))
-                                (void)i915_add_request(dev, file_priv,
-                                                dev->flush_domains,
-                                                &dev_priv->bsd_ring);
-                }
+                                               dev->flush_domains,
+                                               &dev_priv->render_ring);
+                if (dev_priv->flush_rings & FLUSH_BSD_RING)
+                        (void)i915_add_request(dev, file_priv,
+                                               dev->flush_domains,
+                                               &dev_priv->bsd_ring);
         }
 
         for (i = 0; i < args->buffer_count; i++) {
@@ -4192,6 +4030,10 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
                 if (alignment == 0)
                         alignment = i915_gem_get_gtt_alignment(obj);
                 if (obj_priv->gtt_offset & (alignment - 1)) {
+                        WARN(obj_priv->pin_count,
+                             "bo is already pinned with incorrect alignment:"
+                             " offset=%x, req.alignment=%x\n",
+                             obj_priv->gtt_offset, alignment);
                         ret = i915_gem_object_unbind(obj);
                         if (ret)
                                 return ret;
@@ -4213,8 +4055,7 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
                 atomic_inc(&dev->pin_count);
                 atomic_add(obj->size, &dev->pin_memory);
                 if (!obj_priv->active &&
-                    (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
-                    !list_empty(&obj_priv->list))
+                    (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
                         list_del_init(&obj_priv->list);
         }
         i915_verify_inactive(dev, __FILE__, __LINE__);
@@ -4359,22 +4200,34 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
         }
 
         mutex_lock(&dev->struct_mutex);
-        /* Update the active list for the hardware's current position.
-         * Otherwise this only updates on a delayed timer or when irqs are
-         * actually unmasked, and our working set ends up being larger than
-         * required.
-         */
-        i915_gem_retire_requests(dev);
 
-        obj_priv = to_intel_bo(obj);
-        /* Don't count being on the flushing list against the object being
-         * done.  Otherwise, a buffer left on the flushing list but not getting
-         * flushed (because nobody's flushing that domain) won't ever return
-         * unbusy and get reused by libdrm's bo cache.  The other expected
-         * consumer of this interface, OpenGL's occlusion queries, also specs
-         * that the objects get unbusy "eventually" without any interference.
+        /* Count all active objects as busy, even if they are currently not used
+         * by the gpu. Users of this interface expect objects to eventually
+         * become non-busy without any further actions, therefore emit any
+         * necessary flushes here.
          */
-        args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
+        obj_priv = to_intel_bo(obj);
+        args->busy = obj_priv->active;
+        if (args->busy) {
+                /* Unconditionally flush objects, even when the gpu still uses this
+                 * object. Userspace calling this function indicates that it wants to
+                 * use this buffer rather sooner than later, so issuing the required
+                 * flush earlier is beneficial.
+                 */
+                if (obj->write_domain) {
+                        i915_gem_flush(dev, 0, obj->write_domain);
+                        (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
+                }
+
+                /* Update the active list for the hardware's current position.
+                 * Otherwise this only updates on a delayed timer or when irqs
+                 * are actually unmasked, and our working set ends up being
+                 * larger than required.
+                 */
+                i915_gem_retire_requests_ring(dev, obj_priv->ring);
+
+                args->busy = obj_priv->active;
+        }
 
         drm_gem_object_unreference(obj);
         mutex_unlock(&dev->struct_mutex);
@@ -4514,30 +4367,6 @@ void i915_gem_free_object(struct drm_gem_object *obj)
         i915_gem_free_object_tail(obj);
 }
 
-/** Unbinds all inactive objects. */
-static int
-i915_gem_evict_from_inactive_list(struct drm_device *dev)
-{
-        drm_i915_private_t *dev_priv = dev->dev_private;
-
-        while (!list_empty(&dev_priv->mm.inactive_list)) {
-                struct drm_gem_object *obj;
-                int ret;
-
-                obj = &list_first_entry(&dev_priv->mm.inactive_list,
-                                        struct drm_i915_gem_object,
-                                        list)->base;
-
-                ret = i915_gem_object_unbind(obj);
-                if (ret != 0) {
-                        DRM_ERROR("Error unbinding object: %d\n", ret);
-                        return ret;
-                }
-        }
-
-        return 0;
-}
-
 int
 i915_gem_idle(struct drm_device *dev)
 {
@@ -4562,7 +4391,7 @@ i915_gem_idle(struct drm_device *dev)
 
         /* Under UMS, be paranoid and evict. */
         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = i915_gem_evict_from_inactive_list(dev);
+                ret = i915_gem_evict_inactive(dev);
                 if (ret) {
                         mutex_unlock(&dev->struct_mutex);
                         return ret;
@@ -4680,6 +4509,8 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
                         goto cleanup_render_ring;
         }
 
+        dev_priv->next_seqno = 1;
+
         return 0;
 
 cleanup_render_ring:
@@ -4841,7 +4672,7 @@ i915_gem_load(struct drm_device *dev)
  * e.g. for cursor + overlay regs
  */
 int i915_gem_init_phys_object(struct drm_device *dev,
-                              int id, int size)
+                              int id, int size, int align)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_phys_object *phys_obj;
@@ -4856,7 +4687,7 @@ int i915_gem_init_phys_object(struct drm_device *dev,
 
         phys_obj->id = id;
 
-        phys_obj->handle = drm_pci_alloc(dev, size, 0);
+        phys_obj->handle = drm_pci_alloc(dev, size, align);
         if (!phys_obj->handle) {
                 ret = -ENOMEM;
                 goto kfree_obj;
@@ -4938,7 +4769,9 @@ out:
 
 int
 i915_gem_attach_phys_object(struct drm_device *dev,
-                            struct drm_gem_object *obj, int id)
+                            struct drm_gem_object *obj,
+                            int id,
+                            int align)
 {
         drm_i915_private_t *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj_priv;
@@ -4957,11 +4790,10 @@ i915_gem_attach_phys_object(struct drm_device *dev,
                 i915_gem_detach_phys_object(dev, obj);
         }
 
-
         /* create a new object */
         if (!dev_priv->mm.phys_objs[id - 1]) {
                 ret = i915_gem_init_phys_object(dev, id,
-                                                obj->size);
+                                                obj->size, align);
                 if (ret) {
                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
                         goto out;