1 files changed, 336 insertions, 309 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 28f91df2604d..4a9faea626db 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -160,33 +160,6 @@ i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 }
 
 int
-i915_gem_init_ioctl(struct drm_device *dev, void *data,
-                    struct drm_file *file)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_init *args = data;
-
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                return -ENODEV;
-
-        if (args->gtt_start >= args->gtt_end ||
-            (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
-                return -EINVAL;
-
-        /* GEM with user mode setting was never supported on ilk and later. */
-        if (INTEL_INFO(dev)->gen >= 5)
-                return -ENODEV;
-
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
-                                  args->gtt_end);
-        dev_priv->gtt.mappable_end = args->gtt_end;
-        mutex_unlock(&dev->struct_mutex);
-
-        return 0;
-}
-
-int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
                             struct drm_file *file)
 {
@@ -208,40 +181,137 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
         return 0;
 }
 
-static void i915_gem_object_detach_phys(struct drm_i915_gem_object *obj)
+static int
+i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 {
-        drm_dma_handle_t *phys = obj->phys_handle;
+        struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
+        char *vaddr = obj->phys_handle->vaddr;
+        struct sg_table *st;
+        struct scatterlist *sg;
+        int i;
 
-        if (!phys)
-                return;
+        if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
+                return -EINVAL;
 
-        if (obj->madv == I915_MADV_WILLNEED) {
+        for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
+                struct page *page;
+                char *src;
+
+                page = shmem_read_mapping_page(mapping, i);
+                if (IS_ERR(page))
+                        return PTR_ERR(page);
+
+                src = kmap_atomic(page);
+                memcpy(vaddr, src, PAGE_SIZE);
+                drm_clflush_virt_range(vaddr, PAGE_SIZE);
+                kunmap_atomic(src);
+
+                page_cache_release(page);
+                vaddr += PAGE_SIZE;
+        }
+
+        i915_gem_chipset_flush(obj->base.dev);
+
+        st = kmalloc(sizeof(*st), GFP_KERNEL);
+        if (st == NULL)
+                return -ENOMEM;
+
+        if (sg_alloc_table(st, 1, GFP_KERNEL)) {
+                kfree(st);
+                return -ENOMEM;
+        }
+
+        sg = st->sgl;
+        sg->offset = 0;
+        sg->length = obj->base.size;
+
+        sg_dma_address(sg) = obj->phys_handle->busaddr;
+        sg_dma_len(sg) = obj->base.size;
+
+        obj->pages = st;
+        obj->has_dma_mapping = true;
+        return 0;
+}
+
+static void
+i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
+{
+        int ret;
+
+        BUG_ON(obj->madv == __I915_MADV_PURGED);
+
+        ret = i915_gem_object_set_to_cpu_domain(obj, true);
+        if (ret) {
+                /* In the event of a disaster, abandon all caches and
+                 * hope for the best.
+                 */
+                WARN_ON(ret != -EIO);
+                obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+        }
+
+        if (obj->madv == I915_MADV_DONTNEED)
+                obj->dirty = 0;
+
+        if (obj->dirty) {
                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
-                char *vaddr = phys->vaddr;
+                char *vaddr = obj->phys_handle->vaddr;
                 int i;
 
                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
-                        struct page *page = shmem_read_mapping_page(mapping, i);
-                        if (!IS_ERR(page)) {
-                                char *dst = kmap_atomic(page);
-                                memcpy(dst, vaddr, PAGE_SIZE);
-                                drm_clflush_virt_range(dst, PAGE_SIZE);
-                                kunmap_atomic(dst);
-
-                                set_page_dirty(page);
+                        struct page *page;
+                        char *dst;
+
+                        page = shmem_read_mapping_page(mapping, i);
+                        if (IS_ERR(page))
+                                continue;
+
+                        dst = kmap_atomic(page);
+                        drm_clflush_virt_range(vaddr, PAGE_SIZE);
+                        memcpy(dst, vaddr, PAGE_SIZE);
+                        kunmap_atomic(dst);
+
+                        set_page_dirty(page);
+                        if (obj->madv == I915_MADV_WILLNEED)
                                 mark_page_accessed(page);
-                                page_cache_release(page);
-                        }
+                        page_cache_release(page);
                         vaddr += PAGE_SIZE;
                 }
-                i915_gem_chipset_flush(obj->base.dev);
+                obj->dirty = 0;
         }
 
-#ifdef CONFIG_X86
-        set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE);
-#endif
-        drm_pci_free(obj->base.dev, phys);
-        obj->phys_handle = NULL;
+        sg_free_table(obj->pages);
+        kfree(obj->pages);
+
+        obj->has_dma_mapping = false;
+}
+
+static void
+i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
+{
+        drm_pci_free(obj->base.dev, obj->phys_handle);
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
+        .get_pages = i915_gem_object_get_pages_phys,
+        .put_pages = i915_gem_object_put_pages_phys,
+        .release = i915_gem_object_release_phys,
+};
+
+static int
+drop_pages(struct drm_i915_gem_object *obj)
+{
+        struct i915_vma *vma, *next;
+        int ret;
+
+        drm_gem_object_reference(&obj->base);
+        list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
+                if (i915_vma_unbind(vma))
+                        break;
+
+        ret = i915_gem_object_put_pages(obj);
+        drm_gem_object_unreference(&obj->base);
+
+        return ret;
 }
 
 int
@@ -249,9 +319,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
                             int align)
 {
         drm_dma_handle_t *phys;
-        struct address_space *mapping;
-        char *vaddr;
-        int i;
+        int ret;
 
         if (obj->phys_handle) {
                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
@@ -266,41 +334,19 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
         if (obj->base.filp == NULL)
                 return -EINVAL;
 
+        ret = drop_pages(obj);
+        if (ret)
+                return ret;
+
         /* create a new object */
         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
         if (!phys)
                 return -ENOMEM;
 
-        vaddr = phys->vaddr;
-#ifdef CONFIG_X86
-        set_memory_wc((unsigned long)vaddr, phys->size / PAGE_SIZE);
-#endif
-        mapping = file_inode(obj->base.filp)->i_mapping;
-        for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
-                struct page *page;
-                char *src;
-
-                page = shmem_read_mapping_page(mapping, i);
-                if (IS_ERR(page)) {
-#ifdef CONFIG_X86
-                        set_memory_wb((unsigned long)phys->vaddr, phys->size / PAGE_SIZE);
-#endif
-                        drm_pci_free(obj->base.dev, phys);
-                        return PTR_ERR(page);
-                }
-
-                src = kmap_atomic(page);
-                memcpy(vaddr, src, PAGE_SIZE);
-                kunmap_atomic(src);
-
-                mark_page_accessed(page);
-                page_cache_release(page);
-
-                vaddr += PAGE_SIZE;
-        }
-
         obj->phys_handle = phys;
-        return 0;
+        obj->ops = &i915_gem_phys_ops;
+
+        return i915_gem_object_get_pages(obj);
 }
 
 static int
@@ -311,6 +357,14 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
         struct drm_device *dev = obj->base.dev;
         void *vaddr = obj->phys_handle->vaddr + args->offset;
         char __user *user_data = to_user_ptr(args->data_ptr);
+        int ret;
+
+        /* We manually control the domain here and pretend that it
+         * remains coherent i.e. in the GTT domain, like shmem_pwrite.
+         */
+        ret = i915_gem_object_wait_rendering(obj, false);
+        if (ret)
+                return ret;
 
         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
                 unsigned long unwritten;
@@ -326,6 +380,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
                         return -EFAULT;
         }
 
+        drm_clflush_virt_range(vaddr, args->size);
         i915_gem_chipset_flush(dev);
         return 0;
 }
@@ -346,6 +401,7 @@ static int
 i915_gem_create(struct drm_file *file,
                 struct drm_device *dev,
                 uint64_t size,
+                bool dumb,
                 uint32_t *handle_p)
 {
         struct drm_i915_gem_object *obj;
@@ -361,6 +417,7 @@ i915_gem_create(struct drm_file *file,
         if (obj == NULL)
                 return -ENOMEM;
 
+        obj->base.dumb = dumb;
         ret = drm_gem_handle_create(file, &obj->base, &handle);
         /* drop reference from allocate - handle holds it now */
         drm_gem_object_unreference_unlocked(&obj->base);
@@ -380,7 +437,7 @@ i915_gem_dumb_create(struct drm_file *file,
         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
         args->size = args->pitch * args->height;
         return i915_gem_create(file, dev,
-                               args->size, &args->handle);
+                               args->size, true, &args->handle);
 }
 
 /**
@@ -393,7 +450,7 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
         struct drm_i915_gem_create *args = data;
 
         return i915_gem_create(file, dev,
-                               args->size, &args->handle);
+                               args->size, false, &args->handle);
 }
 
 static inline int
@@ -1046,11 +1103,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
          * pread/pwrite currently are reading and writing from the CPU
          * perspective, requiring manual detiling by the client.
          */
-        if (obj->phys_handle) {
-                ret = i915_gem_phys_pwrite(obj, args, file);
-                goto out;
-        }
-
         if (obj->tiling_mode == I915_TILING_NONE &&
             obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
             cpu_write_needs_clflush(obj)) {
@@ -1060,8 +1112,12 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                  * textures). Fallback to the shmem path in that case. */
         }
 
-        if (ret == -EFAULT || ret == -ENOSPC)
-                ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+        if (ret == -EFAULT || ret == -ENOSPC) {
+                if (obj->phys_handle)
+                        ret = i915_gem_phys_pwrite(obj, args, file);
+                else
+                        ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+        }
 
 out:
         drm_gem_object_unreference(&obj->base);
@@ -1134,7 +1190,7 @@ static bool can_wait_boost(struct drm_i915_file_private *file_priv)
 }
 
 /**
- * __wait_seqno - wait until execution of seqno has finished
+ * __i915_wait_seqno - wait until execution of seqno has finished
  * @ring: the ring expected to report seqno
  * @seqno: duh!
  * @reset_counter: reset sequence associated with the given seqno
@@ -1151,7 +1207,7 @@ static bool can_wait_boost(struct drm_i915_file_private *file_priv)
  * Returns 0 if the seqno was found within the alloted time. Else returns the
  * errno with remaining time filled in timeout argument.
  */
-static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
+int __i915_wait_seqno(struct intel_engine_cs *ring, u32 seqno,
                         unsigned reset_counter,
                         bool interruptible,
                         s64 *timeout,
@@ -1171,7 +1227,8 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
                 return 0;
 
-        timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
+        timeout_expire = timeout ?
+                jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
 
         if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
                 gen6_rps_boost(dev_priv);
@@ -1247,6 +1304,16 @@ static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
                 s64 tres = *timeout - (now - before);
 
                 *timeout = tres < 0 ? 0 : tres;
+
+                /*
+                 * Apparently ktime isn't accurate enough and occasionally has a
+                 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
+                 * things up to make the test happy. We allow up to 1 jiffy.
+                 *
+                 * This is a regrssion from the timespec->ktime conversion.
+                 */
+                if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
+                        *timeout = 0;
         }
 
         return ret;
@@ -1262,6 +1329,7 @@ i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
         struct drm_device *dev = ring->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         bool interruptible = dev_priv->mm.interruptible;
+        unsigned reset_counter;
         int ret;
 
         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -1275,14 +1343,13 @@ i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
         if (ret)
                 return ret;
 
-        return __wait_seqno(ring, seqno,
-                            atomic_read(&dev_priv->gpu_error.reset_counter),
-                            interruptible, NULL, NULL);
+        reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
+        return __i915_wait_seqno(ring, seqno, reset_counter, interruptible,
+                                 NULL, NULL);
 }
 
 static int
-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
-                                     struct intel_engine_cs *ring)
+i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
 {
         if (!obj->active)
                 return 0;
@@ -1319,7 +1386,7 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
         if (ret)
                 return ret;
 
-        return i915_gem_object_wait_rendering__tail(obj, ring);
+        return i915_gem_object_wait_rendering__tail(obj);
 }
 
 /* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1354,12 +1421,13 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
 
         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
         mutex_unlock(&dev->struct_mutex);
-        ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
+        ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL,
+                                file_priv);
         mutex_lock(&dev->struct_mutex);
         if (ret)
                 return ret;
 
-        return i915_gem_object_wait_rendering__tail(obj, ring);
+        return i915_gem_object_wait_rendering__tail(obj);
 }
 
 /**
@@ -1466,6 +1534,16 @@ unlock:
  *
  * While the mapping holds a reference on the contents of the object, it doesn't
  * imply a ref on the object itself.
+ *
+ * IMPORTANT:
+ *
+ * DRM driver writers who look a this function as an example for how to do GEM
+ * mmap support, please don't implement mmap support like here. The modern way
+ * to implement DRM mmap support is with an mmap offset ioctl (like
+ * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
+ * That way debug tooling like valgrind will understand what's going on, hiding
+ * the mmap call in a driver private ioctl will break that. The i915 driver only
+ * does cpu mmaps this way because we didn't know better.
  */
 int
 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
@@ -1762,10 +1840,10 @@ static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
         drm_gem_free_mmap_offset(&obj->base);
 }
 
-int
+static int
 i915_gem_mmap_gtt(struct drm_file *file,
                   struct drm_device *dev,
-                  uint32_t handle,
+                  uint32_t handle, bool dumb,
                   uint64_t *offset)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1782,6 +1860,13 @@ i915_gem_mmap_gtt(struct drm_file *file,
                 goto unlock;
         }
 
+        /*
+         * We don't allow dumb mmaps on objects created using another
+         * interface.
+         */
+        WARN_ONCE(dumb && !(obj->base.dumb || obj->base.import_attach),
+                  "Illegal dumb map of accelerated buffer.\n");
+
         if (obj->base.size > dev_priv->gtt.mappable_end) {
                 ret = -E2BIG;
                 goto out;
@@ -1806,6 +1891,15 @@ unlock:
         return ret;
 }
 
+int
+i915_gem_dumb_map_offset(struct drm_file *file,
+                         struct drm_device *dev,
+                         uint32_t handle,
+                         uint64_t *offset)
+{
+        return i915_gem_mmap_gtt(file, dev, handle, true, offset);
+}
+
 /**
  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
  * @dev: DRM device
@@ -1827,7 +1921,7 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_i915_gem_mmap_gtt *args = data;
 
-        return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
+        return i915_gem_mmap_gtt(file, dev, args->handle, false, &args->offset);
 }
 
 static inline int
@@ -1945,7 +2039,14 @@ unsigned long
 i915_gem_shrink(struct drm_i915_private *dev_priv,
                 long target, unsigned flags)
 {
-        const bool purgeable_only = flags & I915_SHRINK_PURGEABLE;
+        const struct {
+                struct list_head *list;
+                unsigned int bit;
+        } phases[] = {
+                { &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
+                { &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
+                { NULL, 0 },
+        }, *phase;
         unsigned long count = 0;
 
         /*
@@ -1967,48 +2068,30 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
          * dev->struct_mutex and so we won't ever be able to observe an
          * object on the bound_list with a reference count equals 0.
          */
-        if (flags & I915_SHRINK_UNBOUND) {
+        for (phase = phases; phase->list; phase++) {
                 struct list_head still_in_list;
 
-                INIT_LIST_HEAD(&still_in_list);
-                while (count < target && !list_empty(&dev_priv->mm.unbound_list)) {
-                        struct drm_i915_gem_object *obj;
-
-                        obj = list_first_entry(&dev_priv->mm.unbound_list,
-                                               typeof(*obj), global_list);
-                        list_move_tail(&obj->global_list, &still_in_list);
-
-                        if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
-                                continue;
-
-                        drm_gem_object_reference(&obj->base);
-
-                        if (i915_gem_object_put_pages(obj) == 0)
-                                count += obj->base.size >> PAGE_SHIFT;
-
-                        drm_gem_object_unreference(&obj->base);
-                }
-                list_splice(&still_in_list, &dev_priv->mm.unbound_list);
-        }
-
-        if (flags & I915_SHRINK_BOUND) {
-                struct list_head still_in_list;
+                if ((flags & phase->bit) == 0)
+                        continue;
 
                 INIT_LIST_HEAD(&still_in_list);
-                while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
+                while (count < target && !list_empty(phase->list)) {
                         struct drm_i915_gem_object *obj;
                         struct i915_vma *vma, *v;
 
-                        obj = list_first_entry(&dev_priv->mm.bound_list,
+                        obj = list_first_entry(phase->list,
                                                typeof(*obj), global_list);
                         list_move_tail(&obj->global_list, &still_in_list);
 
-                        if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
+                        if (flags & I915_SHRINK_PURGEABLE &&
+                            !i915_gem_object_is_purgeable(obj))
                                 continue;
 
                         drm_gem_object_reference(&obj->base);
 
-                        list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
+                        /* For the unbound phase, this should be a no-op! */
+                        list_for_each_entry_safe(vma, v,
+                                                 &obj->vma_list, vma_link)
                                 if (i915_vma_unbind(vma))
                                         break;
 
@@ -2017,7 +2100,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
                         drm_gem_object_unreference(&obj->base);
                 }
-                list_splice(&still_in_list, &dev_priv->mm.bound_list);
+                list_splice(&still_in_list, phase->list);
         }
 
         return count;
@@ -2122,6 +2205,10 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
         if (i915_gem_object_needs_bit17_swizzle(obj))
                 i915_gem_object_do_bit_17_swizzle(obj);
 
+        if (obj->tiling_mode != I915_TILING_NONE &&
+            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+                i915_gem_object_pin_pages(obj);
+
         return 0;
 
 err_pages:
@@ -2420,15 +2507,13 @@ int __i915_add_request(struct intel_engine_cs *ring,
         ring->outstanding_lazy_seqno = 0;
         ring->preallocated_lazy_request = NULL;
 
-        if (!dev_priv->ums.mm_suspended) {
-                i915_queue_hangcheck(ring->dev);
+        i915_queue_hangcheck(ring->dev);
 
-                cancel_delayed_work_sync(&dev_priv->mm.idle_work);
-                queue_delayed_work(dev_priv->wq,
-                                   &dev_priv->mm.retire_work,
-                                   round_jiffies_up_relative(HZ));
-                intel_mark_busy(dev_priv->dev);
-        }
+        cancel_delayed_work_sync(&dev_priv->mm.idle_work);
+        queue_delayed_work(dev_priv->wq,
+                           &dev_priv->mm.retire_work,
+                           round_jiffies_up_relative(HZ));
+        intel_mark_busy(dev_priv->dev);
 
         if (out_seqno)
                 *out_seqno = request->seqno;
@@ -2495,12 +2580,20 @@ static void i915_set_reset_status(struct drm_i915_private *dev_priv,
 
 static void i915_gem_free_request(struct drm_i915_gem_request *request)
 {
+        struct intel_context *ctx = request->ctx;
+
         list_del(&request->list);
         i915_gem_request_remove_from_client(request);
 
-        if (request->ctx)
-                i915_gem_context_unreference(request->ctx);
+        if (ctx) {
+                if (i915.enable_execlists) {
+                        struct intel_engine_cs *ring = request->ring;
 
+                        if (ctx != ring->default_context)
+                                intel_lr_context_unpin(ring, ctx);
+                }
+                i915_gem_context_unreference(ctx);
+        }
         kfree(request);
 }
 
@@ -2555,6 +2648,23 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
         }
 
         /*
+         * Clear the execlists queue up before freeing the requests, as those
+         * are the ones that keep the context and ringbuffer backing objects
+         * pinned in place.
+         */
+        while (!list_empty(&ring->execlist_queue)) {
+                struct intel_ctx_submit_request *submit_req;
+
+                submit_req = list_first_entry(&ring->execlist_queue,
+                                struct intel_ctx_submit_request,
+                                execlist_link);
+                list_del(&submit_req->execlist_link);
+                intel_runtime_pm_put(dev_priv);
+                i915_gem_context_unreference(submit_req->ctx);
+                kfree(submit_req);
+        }
+
+        /*
          * We must free the requests after all the corresponding objects have
          * been moved off active lists. Which is the same order as the normal
          * retire_requests function does. This is important if object hold
@@ -2571,18 +2681,6 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
                 i915_gem_free_request(request);
         }
 
-        while (!list_empty(&ring->execlist_queue)) {
-                struct intel_ctx_submit_request *submit_req;
-
-                submit_req = list_first_entry(&ring->execlist_queue,
-                                struct intel_ctx_submit_request,
-                                execlist_link);
-                list_del(&submit_req->execlist_link);
-                intel_runtime_pm_put(dev_priv);
-                i915_gem_context_unreference(submit_req->ctx);
-                kfree(submit_req);
-        }
-
         /* These may not have been flush before the reset, do so now */
         kfree(ring->preallocated_lazy_request);
         ring->preallocated_lazy_request = NULL;
@@ -2719,6 +2817,15 @@ i915_gem_retire_requests(struct drm_device *dev)
         for_each_ring(ring, dev_priv, i) {
                 i915_gem_retire_requests_ring(ring);
                 idle &= list_empty(&ring->request_list);
+                if (i915.enable_execlists) {
+                        unsigned long flags;
+
+                        spin_lock_irqsave(&ring->execlist_lock, flags);
+                        idle &= list_empty(&ring->execlist_queue);
+                        spin_unlock_irqrestore(&ring->execlist_lock, flags);
+
+                        intel_execlists_retire_requests(ring);
+                }
         }
 
         if (idle)
@@ -2811,6 +2918,9 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
         u32 seqno = 0;
         int ret = 0;
 
+        if (args->flags != 0)
+                return -EINVAL;
+
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
@@ -2846,8 +2956,8 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
         mutex_unlock(&dev->struct_mutex);
 
-        return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns,
-                            file->driver_priv);
+        return __i915_wait_seqno(ring, seqno, reset_counter, true,
+                                 &args->timeout_ns, file->driver_priv);
 
 out:
         drm_gem_object_unreference(&obj->base);
@@ -3166,6 +3276,7 @@ static void i915_gem_write_fence(struct drm_device *dev, int reg,
              obj->stride, obj->tiling_mode);
 
         switch (INTEL_INFO(dev)->gen) {
+        case 9:
         case 8:
         case 7:
         case 6:
@@ -3384,46 +3495,6 @@ static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
         return true;
 }
 
-static void i915_gem_verify_gtt(struct drm_device *dev)
-{
-#if WATCH_GTT
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj;
-        int err = 0;
-
-        list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
-                if (obj->gtt_space == NULL) {
-                        printk(KERN_ERR "object found on GTT list with no space reserved\n");
-                        err++;
-                        continue;
-                }
-
-                if (obj->cache_level != obj->gtt_space->color) {
-                        printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
-                               i915_gem_obj_ggtt_offset(obj),
-                               i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
-                               obj->cache_level,
-                               obj->gtt_space->color);
-                        err++;
-                        continue;
-                }
-
-                if (!i915_gem_valid_gtt_space(dev,
-                                              obj->gtt_space,
-                                              obj->cache_level)) {
-                        printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
-                               i915_gem_obj_ggtt_offset(obj),
-                               i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
-                               obj->cache_level);
-                        err++;
-                        continue;
-                }
-        }
-
-        WARN_ON(err);
-#endif
-}
-
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
@@ -3514,25 +3585,10 @@ search_free:
         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
         list_add_tail(&vma->mm_list, &vm->inactive_list);
 
-        if (i915_is_ggtt(vm)) {
-                bool mappable, fenceable;
-
-                fenceable = (vma->node.size == fence_size &&
-                             (vma->node.start & (fence_alignment - 1)) == 0);
-
-                mappable = (vma->node.start + obj->base.size <=
-                            dev_priv->gtt.mappable_end);
-
-                obj->map_and_fenceable = mappable && fenceable;
-        }
-
-        WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
-
         trace_i915_vma_bind(vma, flags);
         vma->bind_vma(vma, obj->cache_level,
-                      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
+                      flags & PIN_GLOBAL ? GLOBAL_BIND : 0);
 
-        i915_gem_verify_gtt(dev);
         return vma;
 
 err_remove_node:
@@ -3560,7 +3616,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
          * Stolen memory is always coherent with the GPU as it is explicitly
          * marked as wc by the system, or the system is cache-coherent.
          */
-        if (obj->stolen)
+        if (obj->stolen || obj->phys_handle)
                 return false;
 
         /* If the GPU is snooping the contents of the CPU cache,
@@ -3739,7 +3795,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                 list_for_each_entry(vma, &obj->vma_list, vma_link)
                         if (drm_mm_node_allocated(&vma->node))
                                 vma->bind_vma(vma, cache_level,
-                                              obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
+                                                vma->bound & GLOBAL_BIND);
         }
 
         list_for_each_entry(vma, &obj->vma_list, vma_link)
@@ -3769,7 +3825,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                                                     old_write_domain);
         }
 
-        i915_gem_verify_gtt(dev);
         return 0;
 }
 
@@ -4067,7 +4122,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
         if (seqno == 0)
                 return 0;
 
-        ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
+        ret = __i915_wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
         if (ret == 0)
                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
 
@@ -4101,6 +4156,7 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
 {
         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
         struct i915_vma *vma;
+        unsigned bound;
         int ret;
 
         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
@@ -4109,6 +4165,9 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
                 return -EINVAL;
 
+        if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
+                return -EINVAL;
+
         vma = i915_gem_obj_to_vma(obj, vm);
         if (vma) {
                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
@@ -4130,15 +4189,39 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
                 }
         }
 
+        bound = vma ? vma->bound : 0;
         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
                 vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
                 if (IS_ERR(vma))
                         return PTR_ERR(vma);
         }
 
-        if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
+        if (flags & PIN_GLOBAL && !(vma->bound & GLOBAL_BIND))
                 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
 
+        if ((bound ^ vma->bound) & GLOBAL_BIND) {
+                bool mappable, fenceable;
+                u32 fence_size, fence_alignment;
+
+                fence_size = i915_gem_get_gtt_size(obj->base.dev,
+                                                   obj->base.size,
+                                                   obj->tiling_mode);
+                fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
+                                                             obj->base.size,
+                                                             obj->tiling_mode,
+                                                             true);
+
+                fenceable = (vma->node.size == fence_size &&
+                             (vma->node.start & (fence_alignment - 1)) == 0);
+
+                mappable = (vma->node.start + obj->base.size <=
+                            dev_priv->gtt.mappable_end);
+
+                obj->map_and_fenceable = mappable && fenceable;
+        }
+
+        WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+
         vma->pin_count++;
         if (flags & PIN_MAPPABLE)
                 obj->pin_mappable |= true;
@@ -4193,7 +4276,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
         struct drm_i915_gem_object *obj;
         int ret;
 
-        if (INTEL_INFO(dev)->gen >= 6)
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return -ENODEV;
 
         ret = i915_mutex_lock_interruptible(dev);
@@ -4249,6 +4332,9 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
         struct drm_i915_gem_object *obj;
         int ret;
 
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                return -ENODEV;
+
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
@@ -4326,6 +4412,7 @@ int
 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_madvise *args = data;
         struct drm_i915_gem_object *obj;
         int ret;
@@ -4353,6 +4440,15 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                 goto out;
         }
 
+        if (obj->pages &&
+            obj->tiling_mode != I915_TILING_NONE &&
+            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+                if (obj->madv == I915_MADV_WILLNEED)
+                        i915_gem_object_unpin_pages(obj);
+                if (args->madv == I915_MADV_WILLNEED)
+                        i915_gem_object_pin_pages(obj);
+        }
+
         if (obj->madv != __I915_MADV_PURGED)
                 obj->madv = args->madv;
 
@@ -4495,8 +4591,6 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
                 }
         }
 
-        i915_gem_object_detach_phys(obj);
-
         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
          * before progressing. */
         if (obj->stolen)
@@ -4504,6 +4598,11 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 
         WARN_ON(obj->frontbuffer_bits);
 
+        if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
+            dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
+            obj->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_unpin_pages(obj);
+
         if (WARN_ON(obj->pages_pin_count))
                 obj->pages_pin_count = 0;
         if (discard_backing_storage(obj))
@@ -4576,9 +4675,6 @@ i915_gem_suspend(struct drm_device *dev)
         int ret = 0;
 
         mutex_lock(&dev->struct_mutex);
-        if (dev_priv->ums.mm_suspended)
-                goto err;
-
         ret = i915_gpu_idle(dev);
         if (ret)
                 goto err;
@@ -4589,15 +4685,7 @@ i915_gem_suspend(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_gem_evict_everything(dev);
 
-        i915_kernel_lost_context(dev);
         i915_gem_stop_ringbuffers(dev);
-
-        /* Hack!  Don't let anybody do execbuf while we don't control the chip.
-         * We need to replace this with a semaphore, or something.
-         * And not confound ums.mm_suspended!
-         */
-        dev_priv->ums.mm_suspended = !drm_core_check_feature(dev,
-                                                             DRIVER_MODESET);
         mutex_unlock(&dev->struct_mutex);
 
         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
@@ -4888,9 +4976,6 @@ int i915_gem_init(struct drm_device *dev)
         }
         mutex_unlock(&dev->struct_mutex);
 
-        /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                dev_priv->dri1.allow_batchbuffer = 1;
         return ret;
 }
 
@@ -4905,74 +4990,6 @@ i915_gem_cleanup_ringbuffer(struct drm_device *dev)
                 dev_priv->gt.cleanup_ring(ring);
 }
 
-int
-i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
-
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                return 0;
-
-        if (i915_reset_in_progress(&dev_priv->gpu_error)) {
-                DRM_ERROR("Reenabling wedged hardware, good luck\n");
-                atomic_set(&dev_priv->gpu_error.reset_counter, 0);
-        }
-
-        mutex_lock(&dev->struct_mutex);
-        dev_priv->ums.mm_suspended = 0;
-
-        ret = i915_gem_init_hw(dev);
-        if (ret != 0) {
-                mutex_unlock(&dev->struct_mutex);
-                return ret;
-        }
-
-        BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
-
-        ret = drm_irq_install(dev, dev->pdev->irq);
-        if (ret)
-                goto cleanup_ringbuffer;
-        mutex_unlock(&dev->struct_mutex);
-
-        return 0;
-
-cleanup_ringbuffer:
-        i915_gem_cleanup_ringbuffer(dev);
-        dev_priv->ums.mm_suspended = 1;
-        mutex_unlock(&dev->struct_mutex);
-
-        return ret;
-}
-
-int
-i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                return 0;
-
-        mutex_lock(&dev->struct_mutex);
-        drm_irq_uninstall(dev);
-        mutex_unlock(&dev->struct_mutex);
-
-        return i915_gem_suspend(dev);
-}
-
-void
-i915_gem_lastclose(struct drm_device *dev)
-{
-        int ret;
-
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                return;
-
-        ret = i915_gem_suspend(dev);
-        if (ret)
-                DRM_ERROR("failed to idle hardware: %d\n", ret);
-}
-
 static void
 init_ring_lists(struct intel_engine_cs *ring)
 {
@@ -5119,6 +5136,15 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
         return ret;
 }
 
+/**
+ * i915_gem_track_fb - update frontbuffer tracking
+ * old: current GEM buffer for the frontbuffer slots
+ * new: new GEM buffer for the frontbuffer slots
+ * frontbuffer_bits: bitmask of frontbuffer slots
+ *
+ * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
+ * from @old and setting them in @new. Both @old and @new can be NULL.
+ */
 void i915_gem_track_fb(struct drm_i915_gem_object *old,
                        struct drm_i915_gem_object *new,
                        unsigned frontbuffer_bits)
@@ -5302,7 +5328,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
         struct drm_device *dev = dev_priv->dev;
         struct drm_i915_gem_object *obj;
         unsigned long timeout = msecs_to_jiffies(5000) + 1;
-        unsigned long pinned, bound, unbound, freed;
+        unsigned long pinned, bound, unbound, freed_pages;
         bool was_interruptible;
         bool unlock;
 
@@ -5319,7 +5345,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
         was_interruptible = dev_priv->mm.interruptible;
         dev_priv->mm.interruptible = false;
 
-        freed = i915_gem_shrink_all(dev_priv);
+        freed_pages = i915_gem_shrink_all(dev_priv);
 
         dev_priv->mm.interruptible = was_interruptible;
 
@@ -5350,14 +5376,15 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
         if (unlock)
                 mutex_unlock(&dev->struct_mutex);
 
-        pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
-                freed, pinned);
+        if (freed_pages || unbound || bound)
+                pr_info("Purging GPU memory, %lu bytes freed, %lu bytes still pinned.\n",
+                        freed_pages << PAGE_SHIFT, pinned);
         if (unbound || bound)
                 pr_err("%lu and %lu bytes still available in the "
                        "bound and unbound GPU page lists.\n",
                        bound, unbound);
 
-        *(unsigned long *)ptr += freed;
+        *(unsigned long *)ptr += freed_pages;
         return NOTIFY_DONE;
 }