1 files changed, 159 insertions, 32 deletions

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 1449b452cc6..ee896d91c5b 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -43,8 +43,6 @@ static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
                                                      uint64_t offset,
                                                      uint64_t size);
 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
-static int i915_gem_object_get_pages(struct drm_gem_object *obj);
-static void i915_gem_object_put_pages(struct drm_gem_object *obj);
 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
                                            unsigned alignment);
@@ -143,15 +141,27 @@ fast_shmem_read(struct page **pages,
                 int length)
 {
         char __iomem *vaddr;
-        int ret;
+        int unwritten;
 
         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
         if (vaddr == NULL)
                 return -ENOMEM;
-        ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
+        unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
         kunmap_atomic(vaddr, KM_USER0);
 
-        return ret;
+        if (unwritten)
+                return -EFAULT;
+
+        return 0;
+}
+
+static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
+{
+        drm_i915_private_t *dev_priv = obj->dev->dev_private;
+        struct drm_i915_gem_object *obj_priv = obj->driver_private;
+
+        return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
+                obj_priv->tiling_mode != I915_TILING_NONE;
 }
 
 static inline int
@@ -181,6 +191,64 @@ slow_shmem_copy(struct page *dst_page,
         return 0;
 }
 
+static inline int
+slow_shmem_bit17_copy(struct page *gpu_page,
+                      int gpu_offset,
+                      struct page *cpu_page,
+                      int cpu_offset,
+                      int length,
+                      int is_read)
+{
+        char *gpu_vaddr, *cpu_vaddr;
+
+        /* Use the unswizzled path if this page isn't affected. */
+        if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
+                if (is_read)
+                        return slow_shmem_copy(cpu_page, cpu_offset,
+                                               gpu_page, gpu_offset, length);
+                else
+                        return slow_shmem_copy(gpu_page, gpu_offset,
+                                               cpu_page, cpu_offset, length);
+        }
+
+        gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
+        if (gpu_vaddr == NULL)
+                return -ENOMEM;
+
+        cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
+        if (cpu_vaddr == NULL) {
+                kunmap_atomic(gpu_vaddr, KM_USER0);
+                return -ENOMEM;
+        }
+
+        /* Copy the data, XORing A6 with A17 (1). The user already knows he's
+         * XORing with the other bits (A9 for Y, A9 and A10 for X)
+         */
+        while (length > 0) {
+                int cacheline_end = ALIGN(gpu_offset + 1, 64);
+                int this_length = min(cacheline_end - gpu_offset, length);
+                int swizzled_gpu_offset = gpu_offset ^ 64;
+
+                if (is_read) {
+                        memcpy(cpu_vaddr + cpu_offset,
+                               gpu_vaddr + swizzled_gpu_offset,
+                               this_length);
+                } else {
+                        memcpy(gpu_vaddr + swizzled_gpu_offset,
+                               cpu_vaddr + cpu_offset,
+                               this_length);
+                }
+                cpu_offset += this_length;
+                gpu_offset += this_length;
+                length -= this_length;
+        }
+
+        kunmap_atomic(cpu_vaddr, KM_USER1);
+        kunmap_atomic(gpu_vaddr, KM_USER0);
+
+        return 0;
+}
+
 /**
  * This is the fast shmem pread path, which attempts to copy_from_user directly
  * from the backing pages of the object to the user's address space.  On a
@@ -269,6 +337,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
         int page_length;
         int ret;
         uint64_t data_ptr = args->data_ptr;
+        int do_bit17_swizzling;
 
         remain = args->size;
 
@@ -286,13 +355,15 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 
         down_read(&mm->mmap_sem);
         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
-                                      num_pages, 0, 0, user_pages, NULL);
+                                      num_pages, 1, 0, user_pages, NULL);
         up_read(&mm->mmap_sem);
         if (pinned_pages < num_pages) {
                 ret = -EFAULT;
                 goto fail_put_user_pages;
         }
 
+        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+
         mutex_lock(&dev->struct_mutex);
 
         ret = i915_gem_object_get_pages(obj);
@@ -327,11 +398,20 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
-                ret = slow_shmem_copy(user_pages[data_page_index],
-                                      data_page_offset,
-                                      obj_priv->pages[shmem_page_index],
-                                      shmem_page_offset,
-                                      page_length);
+                if (do_bit17_swizzling) {
+                        ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                                                    shmem_page_offset,
+                                                    user_pages[data_page_index],
+                                                    data_page_offset,
+                                                    page_length,
+                                                    1);
+                } else {
+                        ret = slow_shmem_copy(user_pages[data_page_index],
+                                              data_page_offset,
+                                              obj_priv->pages[shmem_page_index],
+                                              shmem_page_offset,
+                                              page_length);
+                }
                 if (ret)
                         goto fail_put_pages;
 
@@ -383,9 +463,14 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
-        if (ret != 0)
+        if (i915_gem_object_needs_bit17_swizzle(obj)) {
                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
+        } else {
+                ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
+                if (ret != 0)
+                        ret = i915_gem_shmem_pread_slow(dev, obj, args,
+                                                        file_priv);
+        }
 
         drm_gem_object_unreference(obj);
 
@@ -727,6 +812,7 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
         int page_length;
         int ret;
         uint64_t data_ptr = args->data_ptr;
+        int do_bit17_swizzling;
 
         remain = args->size;
 
@@ -751,6 +837,8 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 goto fail_put_user_pages;
         }
 
+        do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+
         mutex_lock(&dev->struct_mutex);
 
         ret = i915_gem_object_get_pages(obj);
@@ -785,11 +873,20 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
                 if ((data_page_offset + page_length) > PAGE_SIZE)
                         page_length = PAGE_SIZE - data_page_offset;
 
-                ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
-                                      shmem_page_offset,
-                                      user_pages[data_page_index],
-                                      data_page_offset,
-                                      page_length);
+                if (do_bit17_swizzling) {
+                        ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
+                                                    shmem_page_offset,
+                                                    user_pages[data_page_index],
+                                                    data_page_offset,
+                                                    page_length,
+                                                    0);
+                } else {
+                        ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
+                                              shmem_page_offset,
+                                              user_pages[data_page_index],
+                                              data_page_offset,
+                                              page_length);
+                }
                 if (ret)
                         goto fail_put_pages;
 
@@ -854,6 +951,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
                                                        file_priv);
                 }
+        } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
+                ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
         } else {
                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
                 if (ret == -EFAULT) {
@@ -1285,7 +1384,7 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
         return 0;
 }
 
-static void
+void
 i915_gem_object_put_pages(struct drm_gem_object *obj)
 {
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
@@ -1297,6 +1396,9 @@ i915_gem_object_put_pages(struct drm_gem_object *obj)
         if (--obj_priv->pages_refcount != 0)
                 return;
 
+        if (obj_priv->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_save_bit_17_swizzle(obj);
+
         for (i = 0; i < page_count; i++)
                 if (obj_priv->pages[i] != NULL) {
                         if (obj_priv->dirty)
@@ -1494,8 +1596,19 @@ i915_gem_retire_request(struct drm_device *dev,
 
                 if (obj->write_domain != 0)
                         i915_gem_object_move_to_flushing(obj);
-                else
+                else {
+                        /* Take a reference on the object so it won't be
+                         * freed while the spinlock is held.  The list
+                         * protection for this spinlock is safe when breaking
+                         * the lock like this since the next thing we do
+                         * is just get the head of the list again.
+                         */
+                        drm_gem_object_reference(obj);
                         i915_gem_object_move_to_inactive(obj);
+                        spin_unlock(&dev_priv->mm.active_list_lock);
+                        drm_gem_object_unreference(obj);
+                        spin_lock(&dev_priv->mm.active_list_lock);
+                }
         }
 out:
         spin_unlock(&dev_priv->mm.active_list_lock);
@@ -1884,7 +1997,7 @@ i915_gem_evict_everything(struct drm_device *dev)
         return ret;
 }
 
-static int
+int
 i915_gem_object_get_pages(struct drm_gem_object *obj)
 {
         struct drm_i915_gem_object *obj_priv = obj->driver_private;
@@ -1922,6 +2035,10 @@ i915_gem_object_get_pages(struct drm_gem_object *obj)
                 }
                 obj_priv->pages[i] = page;
         }
+
+        if (obj_priv->tiling_mode != I915_TILING_NONE)
+                i915_gem_object_do_bit_17_swizzle(obj);
+
         return 0;
 }
 
@@ -3002,13 +3119,13 @@ i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
                         drm_free(*relocs, reloc_count * sizeof(**relocs),
                                  DRM_MEM_DRIVER);
                         *relocs = NULL;
-                        return ret;
+                        return -EFAULT;
                 }
 
                 reloc_index += exec_list[i].relocation_count;
         }
 
-        return ret;
+        return 0;
 }
 
 static int
@@ -3017,23 +3134,28 @@ i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
                             struct drm_i915_gem_relocation_entry *relocs)
 {
         uint32_t reloc_count = 0, i;
-        int ret;
+        int ret = 0;
 
         for (i = 0; i < buffer_count; i++) {
                 struct drm_i915_gem_relocation_entry __user *user_relocs;
+                int unwritten;
 
                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
 
-                if (ret == 0) {
-                        ret = copy_to_user(user_relocs,
-                                           &relocs[reloc_count],
-                                           exec_list[i].relocation_count *
-                                           sizeof(*relocs));
+                unwritten = copy_to_user(user_relocs,
+                                         &relocs[reloc_count],
+                                         exec_list[i].relocation_count *
+                                         sizeof(*relocs));
+
+                if (unwritten) {
+                        ret = -EFAULT;
+                        goto err;
                 }
 
                 reloc_count += exec_list[i].relocation_count;
         }
 
+err:
         drm_free(relocs, reloc_count * sizeof(*relocs), DRM_MEM_DRIVER);
 
         return ret;
@@ -3243,7 +3365,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
         exec_offset = exec_list[args->buffer_count - 1].offset;
 
 #if WATCH_EXEC
-        i915_gem_dump_object(object_list[args->buffer_count - 1],
+        i915_gem_dump_object(batch_obj,
                               args->batch_len,
                               __func__,
                               ~0);
@@ -3308,10 +3430,12 @@ err:
                                    (uintptr_t) args->buffers_ptr,
                                    exec_list,
                                    sizeof(*exec_list) * args->buffer_count);
-                if (ret)
+                if (ret) {
+                        ret = -EFAULT;
                         DRM_ERROR("failed to copy %d exec entries "
                                   "back to user (%d)\n",
                                   args->buffer_count, ret);
+                }
         }
 
         /* Copy the updated relocations out regardless of current error
@@ -3593,6 +3717,7 @@ void i915_gem_free_object(struct drm_gem_object *obj)
         i915_gem_free_mmap_offset(obj);
 
         drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
+        kfree(obj_priv->bit_17);
         drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
 }
 
@@ -3962,8 +4087,10 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
         dev_priv->mm.suspended = 0;
 
         ret = i915_gem_init_ringbuffer(dev);
-        if (ret != 0)
+        if (ret != 0) {
+                mutex_unlock(&dev->struct_mutex);
                 return ret;
+        }
 
         spin_lock(&dev_priv->mm.active_list_lock);
         BUG_ON(!list_empty(&dev_priv->mm.active_list));