aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c4788
1 files changed, 1947 insertions, 2841 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 90b1d6753b9d..a087e1bf0c2f 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -31,149 +31,229 @@
31#include "i915_drv.h" 31#include "i915_drv.h"
32#include "i915_trace.h" 32#include "i915_trace.h"
33#include "intel_drv.h" 33#include "intel_drv.h"
34#include <linux/shmem_fs.h>
34#include <linux/slab.h> 35#include <linux/slab.h>
35#include <linux/swap.h> 36#include <linux/swap.h>
36#include <linux/pci.h> 37#include <linux/pci.h>
37#include <linux/intel-gtt.h> 38
38 39static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); 40static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); 41static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
41static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 42static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 43 bool write);
43static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 44static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
44 int write); 45 uint64_t offset,
45static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 46 uint64_t size);
46 uint64_t offset, 47static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
47 uint64_t size); 48static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
48static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 49 unsigned alignment,
49static int i915_gem_object_wait_rendering(struct drm_gem_object *obj); 50 bool map_and_fenceable);
50static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 51static void i915_gem_clear_fence_reg(struct drm_device *dev,
51 unsigned alignment); 52 struct drm_i915_fence_reg *reg);
52static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 53static int i915_gem_phys_pwrite(struct drm_device *dev,
53static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 54 struct drm_i915_gem_object *obj,
54 struct drm_i915_gem_pwrite *args, 55 struct drm_i915_gem_pwrite *args,
55 struct drm_file *file_priv); 56 struct drm_file *file);
56static void i915_gem_free_object_tail(struct drm_gem_object *obj); 57static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
57 58
58static LIST_HEAD(shrink_list); 59static int i915_gem_inactive_shrink(struct shrinker *shrinker,
59static DEFINE_SPINLOCK(shrink_list_lock); 60 struct shrink_control *sc);
60 61
61static inline bool 62/* some bookkeeping */
62i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) 63static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
64 size_t size)
63{ 65{
64 return obj_priv->gtt_space && 66 dev_priv->mm.object_count++;
65 !obj_priv->active && 67 dev_priv->mm.object_memory += size;
66 obj_priv->pin_count == 0;
67} 68}
68 69
69int i915_gem_do_init(struct drm_device *dev, unsigned long start, 70static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
70 unsigned long end) 71 size_t size)
71{ 72{
72 drm_i915_private_t *dev_priv = dev->dev_private; 73 dev_priv->mm.object_count--;
74 dev_priv->mm.object_memory -= size;
75}
73 76
74 if (start >= end || 77static int
75 (start & (PAGE_SIZE - 1)) != 0 || 78i915_gem_wait_for_error(struct drm_device *dev)
76 (end & (PAGE_SIZE - 1)) != 0) { 79{
77 return -EINVAL; 80 struct drm_i915_private *dev_priv = dev->dev_private;
81 struct completion *x = &dev_priv->error_completion;
82 unsigned long flags;
83 int ret;
84
85 if (!atomic_read(&dev_priv->mm.wedged))
86 return 0;
87
88 ret = wait_for_completion_interruptible(x);
89 if (ret)
90 return ret;
91
92 if (atomic_read(&dev_priv->mm.wedged)) {
93 /* GPU is hung, bump the completion count to account for
94 * the token we just consumed so that we never hit zero and
95 * end up waiting upon a subsequent completion event that
96 * will never happen.
97 */
98 spin_lock_irqsave(&x->wait.lock, flags);
99 x->done++;
100 spin_unlock_irqrestore(&x->wait.lock, flags);
78 } 101 }
102 return 0;
103}
79 104
80 drm_mm_init(&dev_priv->mm.gtt_space, start, 105int i915_mutex_lock_interruptible(struct drm_device *dev)
81 end - start); 106{
107 int ret;
82 108
83 dev->gtt_total = (uint32_t) (end - start); 109 ret = i915_gem_wait_for_error(dev);
110 if (ret)
111 return ret;
84 112
113 ret = mutex_lock_interruptible(&dev->struct_mutex);
114 if (ret)
115 return ret;
116
117 WARN_ON(i915_verify_lists(dev));
85 return 0; 118 return 0;
86} 119}
87 120
121static inline bool
122i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
123{
124 return obj->gtt_space && !obj->active && obj->pin_count == 0;
125}
126
127void i915_gem_do_init(struct drm_device *dev,
128 unsigned long start,
129 unsigned long mappable_end,
130 unsigned long end)
131{
132 drm_i915_private_t *dev_priv = dev->dev_private;
133
134 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
135
136 dev_priv->mm.gtt_start = start;
137 dev_priv->mm.gtt_mappable_end = mappable_end;
138 dev_priv->mm.gtt_end = end;
139 dev_priv->mm.gtt_total = end - start;
140 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
141
142 /* Take over this portion of the GTT */
143 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
144}
145
88int 146int
89i915_gem_init_ioctl(struct drm_device *dev, void *data, 147i915_gem_init_ioctl(struct drm_device *dev, void *data,
90 struct drm_file *file_priv) 148 struct drm_file *file)
91{ 149{
92 struct drm_i915_gem_init *args = data; 150 struct drm_i915_gem_init *args = data;
93 int ret; 151
152 if (args->gtt_start >= args->gtt_end ||
153 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
154 return -EINVAL;
94 155
95 mutex_lock(&dev->struct_mutex); 156 mutex_lock(&dev->struct_mutex);
96 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); 157 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
97 mutex_unlock(&dev->struct_mutex); 158 mutex_unlock(&dev->struct_mutex);
98 159
99 return ret; 160 return 0;
100} 161}
101 162
102int 163int
103i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 164i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
104 struct drm_file *file_priv) 165 struct drm_file *file)
105{ 166{
167 struct drm_i915_private *dev_priv = dev->dev_private;
106 struct drm_i915_gem_get_aperture *args = data; 168 struct drm_i915_gem_get_aperture *args = data;
169 struct drm_i915_gem_object *obj;
170 size_t pinned;
107 171
108 if (!(dev->driver->driver_features & DRIVER_GEM)) 172 if (!(dev->driver->driver_features & DRIVER_GEM))
109 return -ENODEV; 173 return -ENODEV;
110 174
111 args->aper_size = dev->gtt_total; 175 pinned = 0;
112 args->aper_available_size = (args->aper_size - 176 mutex_lock(&dev->struct_mutex);
113 atomic_read(&dev->pin_memory)); 177 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
178 pinned += obj->gtt_space->size;
179 mutex_unlock(&dev->struct_mutex);
180
181 args->aper_size = dev_priv->mm.gtt_total;
182 args->aper_available_size = args->aper_size -pinned;
114 183
115 return 0; 184 return 0;
116} 185}
117 186
118 187static int
119/** 188i915_gem_create(struct drm_file *file,
120 * Creates a new mm object and returns a handle to it. 189 struct drm_device *dev,
121 */ 190 uint64_t size,
122int 191 uint32_t *handle_p)
123i915_gem_create_ioctl(struct drm_device *dev, void *data,
124 struct drm_file *file_priv)
125{ 192{
126 struct drm_i915_gem_create *args = data; 193 struct drm_i915_gem_object *obj;
127 struct drm_gem_object *obj;
128 int ret; 194 int ret;
129 u32 handle; 195 u32 handle;
130 196
131 args->size = roundup(args->size, PAGE_SIZE); 197 size = roundup(size, PAGE_SIZE);
132 198
133 /* Allocate the new object */ 199 /* Allocate the new object */
134 obj = i915_gem_alloc_object(dev, args->size); 200 obj = i915_gem_alloc_object(dev, size);
135 if (obj == NULL) 201 if (obj == NULL)
136 return -ENOMEM; 202 return -ENOMEM;
137 203
138 ret = drm_gem_handle_create(file_priv, obj, &handle); 204 ret = drm_gem_handle_create(file, &obj->base, &handle);
139 /* drop reference from allocate - handle holds it now */
140 drm_gem_object_unreference_unlocked(obj);
141 if (ret) { 205 if (ret) {
206 drm_gem_object_release(&obj->base);
207 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
208 kfree(obj);
142 return ret; 209 return ret;
143 } 210 }
144 211
145 args->handle = handle; 212 /* drop reference from allocate - handle holds it now */
213 drm_gem_object_unreference(&obj->base);
214 trace_i915_gem_object_create(obj);
215
216 *handle_p = handle;
146 return 0; 217 return 0;
147} 218}
148 219
149static inline int 220int
150fast_shmem_read(struct page **pages, 221i915_gem_dumb_create(struct drm_file *file,
151 loff_t page_base, int page_offset, 222 struct drm_device *dev,
152 char __user *data, 223 struct drm_mode_create_dumb *args)
153 int length)
154{ 224{
155 char __iomem *vaddr; 225 /* have to work out size/pitch and return them */
156 int unwritten; 226 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
157 227 args->size = args->pitch * args->height;
158 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0); 228 return i915_gem_create(file, dev,
159 if (vaddr == NULL) 229 args->size, &args->handle);
160 return -ENOMEM; 230}
161 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
162 kunmap_atomic(vaddr, KM_USER0);
163 231
164 if (unwritten) 232int i915_gem_dumb_destroy(struct drm_file *file,
165 return -EFAULT; 233 struct drm_device *dev,
234 uint32_t handle)
235{
236 return drm_gem_handle_delete(file, handle);
237}
166 238
167 return 0; 239/**
240 * Creates a new mm object and returns a handle to it.
241 */
242int
243i915_gem_create_ioctl(struct drm_device *dev, void *data,
244 struct drm_file *file)
245{
246 struct drm_i915_gem_create *args = data;
247 return i915_gem_create(file, dev,
248 args->size, &args->handle);
168} 249}
169 250
170static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj) 251static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
171{ 252{
172 drm_i915_private_t *dev_priv = obj->dev->dev_private; 253 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
173 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
174 254
175 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 255 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
176 obj_priv->tiling_mode != I915_TILING_NONE; 256 obj->tiling_mode != I915_TILING_NONE;
177} 257}
178 258
179static inline void 259static inline void
@@ -249,88 +329,58 @@ slow_shmem_bit17_copy(struct page *gpu_page,
249 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). 329 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
250 */ 330 */
251static int 331static int
252i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, 332i915_gem_shmem_pread_fast(struct drm_device *dev,
333 struct drm_i915_gem_object *obj,
253 struct drm_i915_gem_pread *args, 334 struct drm_i915_gem_pread *args,
254 struct drm_file *file_priv) 335 struct drm_file *file)
255{ 336{
256 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 337 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
257 ssize_t remain; 338 ssize_t remain;
258 loff_t offset, page_base; 339 loff_t offset;
259 char __user *user_data; 340 char __user *user_data;
260 int page_offset, page_length; 341 int page_offset, page_length;
261 int ret;
262 342
263 user_data = (char __user *) (uintptr_t) args->data_ptr; 343 user_data = (char __user *) (uintptr_t) args->data_ptr;
264 remain = args->size; 344 remain = args->size;
265 345
266 mutex_lock(&dev->struct_mutex);
267
268 ret = i915_gem_object_get_pages(obj, 0);
269 if (ret != 0)
270 goto fail_unlock;
271
272 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
273 args->size);
274 if (ret != 0)
275 goto fail_put_pages;
276
277 obj_priv = to_intel_bo(obj);
278 offset = args->offset; 346 offset = args->offset;
279 347
280 while (remain > 0) { 348 while (remain > 0) {
349 struct page *page;
350 char *vaddr;
351 int ret;
352
281 /* Operation in this page 353 /* Operation in this page
282 * 354 *
283 * page_base = page offset within aperture
284 * page_offset = offset within page 355 * page_offset = offset within page
285 * page_length = bytes to copy for this page 356 * page_length = bytes to copy for this page
286 */ 357 */
287 page_base = (offset & ~(PAGE_SIZE-1)); 358 page_offset = offset_in_page(offset);
288 page_offset = offset & (PAGE_SIZE-1);
289 page_length = remain; 359 page_length = remain;
290 if ((page_offset + remain) > PAGE_SIZE) 360 if ((page_offset + remain) > PAGE_SIZE)
291 page_length = PAGE_SIZE - page_offset; 361 page_length = PAGE_SIZE - page_offset;
292 362
293 ret = fast_shmem_read(obj_priv->pages, 363 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
294 page_base, page_offset, 364 if (IS_ERR(page))
295 user_data, page_length); 365 return PTR_ERR(page);
366
367 vaddr = kmap_atomic(page);
368 ret = __copy_to_user_inatomic(user_data,
369 vaddr + page_offset,
370 page_length);
371 kunmap_atomic(vaddr);
372
373 mark_page_accessed(page);
374 page_cache_release(page);
296 if (ret) 375 if (ret)
297 goto fail_put_pages; 376 return -EFAULT;
298 377
299 remain -= page_length; 378 remain -= page_length;
300 user_data += page_length; 379 user_data += page_length;
301 offset += page_length; 380 offset += page_length;
302 } 381 }
303 382
304fail_put_pages: 383 return 0;
305 i915_gem_object_put_pages(obj);
306fail_unlock:
307 mutex_unlock(&dev->struct_mutex);
308
309 return ret;
310}
311
312static int
313i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
314{
315 int ret;
316
317 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
318
319 /* If we've insufficient memory to map in the pages, attempt
320 * to make some space by throwing out some old buffers.
321 */
322 if (ret == -ENOMEM) {
323 struct drm_device *dev = obj->dev;
324
325 ret = i915_gem_evict_something(dev, obj->size,
326 i915_gem_get_gtt_alignment(obj));
327 if (ret)
328 return ret;
329
330 ret = i915_gem_object_get_pages(obj, 0);
331 }
332
333 return ret;
334} 384}
335 385
336/** 386/**
@@ -340,18 +390,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
340 * and not take page faults. 390 * and not take page faults.
341 */ 391 */
342static int 392static int
343i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, 393i915_gem_shmem_pread_slow(struct drm_device *dev,
394 struct drm_i915_gem_object *obj,
344 struct drm_i915_gem_pread *args, 395 struct drm_i915_gem_pread *args,
345 struct drm_file *file_priv) 396 struct drm_file *file)
346{ 397{
347 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 398 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
348 struct mm_struct *mm = current->mm; 399 struct mm_struct *mm = current->mm;
349 struct page **user_pages; 400 struct page **user_pages;
350 ssize_t remain; 401 ssize_t remain;
351 loff_t offset, pinned_pages, i; 402 loff_t offset, pinned_pages, i;
352 loff_t first_data_page, last_data_page, num_pages; 403 loff_t first_data_page, last_data_page, num_pages;
353 int shmem_page_index, shmem_page_offset; 404 int shmem_page_offset;
354 int data_page_index, data_page_offset; 405 int data_page_index, data_page_offset;
355 int page_length; 406 int page_length;
356 int ret; 407 int ret;
357 uint64_t data_ptr = args->data_ptr; 408 uint64_t data_ptr = args->data_ptr;
@@ -367,48 +418,44 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
367 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 418 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
368 num_pages = last_data_page - first_data_page + 1; 419 num_pages = last_data_page - first_data_page + 1;
369 420
370 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 421 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
371 if (user_pages == NULL) 422 if (user_pages == NULL)
372 return -ENOMEM; 423 return -ENOMEM;
373 424
425 mutex_unlock(&dev->struct_mutex);
374 down_read(&mm->mmap_sem); 426 down_read(&mm->mmap_sem);
375 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 427 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
376 num_pages, 1, 0, user_pages, NULL); 428 num_pages, 1, 0, user_pages, NULL);
377 up_read(&mm->mmap_sem); 429 up_read(&mm->mmap_sem);
430 mutex_lock(&dev->struct_mutex);
378 if (pinned_pages < num_pages) { 431 if (pinned_pages < num_pages) {
379 ret = -EFAULT; 432 ret = -EFAULT;
380 goto fail_put_user_pages; 433 goto out;
381 } 434 }
382 435
383 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 436 ret = i915_gem_object_set_cpu_read_domain_range(obj,
384 437 args->offset,
385 mutex_lock(&dev->struct_mutex); 438 args->size);
386
387 ret = i915_gem_object_get_pages_or_evict(obj);
388 if (ret) 439 if (ret)
389 goto fail_unlock; 440 goto out;
390 441
391 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, 442 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
392 args->size);
393 if (ret != 0)
394 goto fail_put_pages;
395 443
396 obj_priv = to_intel_bo(obj);
397 offset = args->offset; 444 offset = args->offset;
398 445
399 while (remain > 0) { 446 while (remain > 0) {
447 struct page *page;
448
400 /* Operation in this page 449 /* Operation in this page
401 * 450 *
402 * shmem_page_index = page number within shmem file
403 * shmem_page_offset = offset within page in shmem file 451 * shmem_page_offset = offset within page in shmem file
404 * data_page_index = page number in get_user_pages return 452 * data_page_index = page number in get_user_pages return
405 * data_page_offset = offset with data_page_index page. 453 * data_page_offset = offset with data_page_index page.
406 * page_length = bytes to copy for this page 454 * page_length = bytes to copy for this page
407 */ 455 */
408 shmem_page_index = offset / PAGE_SIZE; 456 shmem_page_offset = offset_in_page(offset);
409 shmem_page_offset = offset & ~PAGE_MASK;
410 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 457 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
411 data_page_offset = data_ptr & ~PAGE_MASK; 458 data_page_offset = offset_in_page(data_ptr);
412 459
413 page_length = remain; 460 page_length = remain;
414 if ((shmem_page_offset + page_length) > PAGE_SIZE) 461 if ((shmem_page_offset + page_length) > PAGE_SIZE)
@@ -416,8 +463,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
416 if ((data_page_offset + page_length) > PAGE_SIZE) 463 if ((data_page_offset + page_length) > PAGE_SIZE)
417 page_length = PAGE_SIZE - data_page_offset; 464 page_length = PAGE_SIZE - data_page_offset;
418 465
466 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
467 if (IS_ERR(page)) {
468 ret = PTR_ERR(page);
469 goto out;
470 }
471
419 if (do_bit17_swizzling) { 472 if (do_bit17_swizzling) {
420 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 473 slow_shmem_bit17_copy(page,
421 shmem_page_offset, 474 shmem_page_offset,
422 user_pages[data_page_index], 475 user_pages[data_page_index],
423 data_page_offset, 476 data_page_offset,
@@ -426,23 +479,23 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
426 } else { 479 } else {
427 slow_shmem_copy(user_pages[data_page_index], 480 slow_shmem_copy(user_pages[data_page_index],
428 data_page_offset, 481 data_page_offset,
429 obj_priv->pages[shmem_page_index], 482 page,
430 shmem_page_offset, 483 shmem_page_offset,
431 page_length); 484 page_length);
432 } 485 }
433 486
487 mark_page_accessed(page);
488 page_cache_release(page);
489
434 remain -= page_length; 490 remain -= page_length;
435 data_ptr += page_length; 491 data_ptr += page_length;
436 offset += page_length; 492 offset += page_length;
437 } 493 }
438 494
439fail_put_pages: 495out:
440 i915_gem_object_put_pages(obj);
441fail_unlock:
442 mutex_unlock(&dev->struct_mutex);
443fail_put_user_pages:
444 for (i = 0; i < pinned_pages; i++) { 496 for (i = 0; i < pinned_pages; i++) {
445 SetPageDirty(user_pages[i]); 497 SetPageDirty(user_pages[i]);
498 mark_page_accessed(user_pages[i]);
446 page_cache_release(user_pages[i]); 499 page_cache_release(user_pages[i]);
447 } 500 }
448 drm_free_large(user_pages); 501 drm_free_large(user_pages);
@@ -457,42 +510,60 @@ fail_put_user_pages:
457 */ 510 */
458int 511int
459i915_gem_pread_ioctl(struct drm_device *dev, void *data, 512i915_gem_pread_ioctl(struct drm_device *dev, void *data,
460 struct drm_file *file_priv) 513 struct drm_file *file)
461{ 514{
462 struct drm_i915_gem_pread *args = data; 515 struct drm_i915_gem_pread *args = data;
463 struct drm_gem_object *obj; 516 struct drm_i915_gem_object *obj;
464 struct drm_i915_gem_object *obj_priv; 517 int ret = 0;
465 int ret;
466
467 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
468 if (obj == NULL)
469 return -ENOENT;
470 obj_priv = to_intel_bo(obj);
471 518
472 /* Bounds check source. */ 519 if (args->size == 0)
473 if (args->offset > obj->size || args->size > obj->size - args->offset) { 520 return 0;
474 ret = -EINVAL;
475 goto err;
476 }
477 521
478 if (!access_ok(VERIFY_WRITE, 522 if (!access_ok(VERIFY_WRITE,
479 (char __user *)(uintptr_t)args->data_ptr, 523 (char __user *)(uintptr_t)args->data_ptr,
480 args->size)) { 524 args->size))
481 ret = -EFAULT; 525 return -EFAULT;
482 goto err; 526
527 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
528 args->size);
529 if (ret)
530 return -EFAULT;
531
532 ret = i915_mutex_lock_interruptible(dev);
533 if (ret)
534 return ret;
535
536 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
537 if (&obj->base == NULL) {
538 ret = -ENOENT;
539 goto unlock;
483 } 540 }
484 541
485 if (i915_gem_object_needs_bit17_swizzle(obj)) { 542 /* Bounds check source. */
486 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 543 if (args->offset > obj->base.size ||
487 } else { 544 args->size > obj->base.size - args->offset) {
488 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 545 ret = -EINVAL;
489 if (ret != 0) 546 goto out;
490 ret = i915_gem_shmem_pread_slow(dev, obj, args,
491 file_priv);
492 } 547 }
493 548
494err: 549 trace_i915_gem_object_pread(obj, args->offset, args->size);
495 drm_gem_object_unreference_unlocked(obj); 550
551 ret = i915_gem_object_set_cpu_read_domain_range(obj,
552 args->offset,
553 args->size);
554 if (ret)
555 goto out;
556
557 ret = -EFAULT;
558 if (!i915_gem_object_needs_bit17_swizzle(obj))
559 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
560 if (ret == -EFAULT)
561 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
562
563out:
564 drm_gem_object_unreference(&obj->base);
565unlock:
566 mutex_unlock(&dev->struct_mutex);
496 return ret; 567 return ret;
497} 568}
498 569
@@ -509,13 +580,11 @@ fast_user_write(struct io_mapping *mapping,
509 char *vaddr_atomic; 580 char *vaddr_atomic;
510 unsigned long unwritten; 581 unsigned long unwritten;
511 582
512 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0); 583 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
513 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, 584 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
514 user_data, length); 585 user_data, length);
515 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0); 586 io_mapping_unmap_atomic(vaddr_atomic);
516 if (unwritten) 587 return unwritten;
517 return -EFAULT;
518 return 0;
519} 588}
520 589
521/* Here's the write path which can sleep for 590/* Here's the write path which can sleep for
@@ -542,59 +611,26 @@ slow_kernel_write(struct io_mapping *mapping,
542 io_mapping_unmap(dst_vaddr); 611 io_mapping_unmap(dst_vaddr);
543} 612}
544 613
545static inline int
546fast_shmem_write(struct page **pages,
547 loff_t page_base, int page_offset,
548 char __user *data,
549 int length)
550{
551 char __iomem *vaddr;
552 unsigned long unwritten;
553
554 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
555 if (vaddr == NULL)
556 return -ENOMEM;
557 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
558 kunmap_atomic(vaddr, KM_USER0);
559
560 if (unwritten)
561 return -EFAULT;
562 return 0;
563}
564
565/** 614/**
566 * This is the fast pwrite path, where we copy the data directly from the 615 * This is the fast pwrite path, where we copy the data directly from the
567 * user into the GTT, uncached. 616 * user into the GTT, uncached.
568 */ 617 */
569static int 618static int
570i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 619i915_gem_gtt_pwrite_fast(struct drm_device *dev,
620 struct drm_i915_gem_object *obj,
571 struct drm_i915_gem_pwrite *args, 621 struct drm_i915_gem_pwrite *args,
572 struct drm_file *file_priv) 622 struct drm_file *file)
573{ 623{
574 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
575 drm_i915_private_t *dev_priv = dev->dev_private; 624 drm_i915_private_t *dev_priv = dev->dev_private;
576 ssize_t remain; 625 ssize_t remain;
577 loff_t offset, page_base; 626 loff_t offset, page_base;
578 char __user *user_data; 627 char __user *user_data;
579 int page_offset, page_length; 628 int page_offset, page_length;
580 int ret;
581 629
582 user_data = (char __user *) (uintptr_t) args->data_ptr; 630 user_data = (char __user *) (uintptr_t) args->data_ptr;
583 remain = args->size; 631 remain = args->size;
584 632
585 633 offset = obj->gtt_offset + args->offset;
586 mutex_lock(&dev->struct_mutex);
587 ret = i915_gem_object_pin(obj, 0);
588 if (ret) {
589 mutex_unlock(&dev->struct_mutex);
590 return ret;
591 }
592 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
593 if (ret)
594 goto fail;
595
596 obj_priv = to_intel_bo(obj);
597 offset = obj_priv->gtt_offset + args->offset;
598 634
599 while (remain > 0) { 635 while (remain > 0) {
600 /* Operation in this page 636 /* Operation in this page
@@ -603,32 +639,26 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
603 * page_offset = offset within page 639 * page_offset = offset within page
604 * page_length = bytes to copy for this page 640 * page_length = bytes to copy for this page
605 */ 641 */
606 page_base = (offset & ~(PAGE_SIZE-1)); 642 page_base = offset & PAGE_MASK;
607 page_offset = offset & (PAGE_SIZE-1); 643 page_offset = offset_in_page(offset);
608 page_length = remain; 644 page_length = remain;
609 if ((page_offset + remain) > PAGE_SIZE) 645 if ((page_offset + remain) > PAGE_SIZE)
610 page_length = PAGE_SIZE - page_offset; 646 page_length = PAGE_SIZE - page_offset;
611 647
612 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
613 page_offset, user_data, page_length);
614
615 /* If we get a fault while copying data, then (presumably) our 648 /* If we get a fault while copying data, then (presumably) our
616 * source page isn't available. Return the error and we'll 649 * source page isn't available. Return the error and we'll
617 * retry in the slow path. 650 * retry in the slow path.
618 */ 651 */
619 if (ret) 652 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
620 goto fail; 653 page_offset, user_data, page_length))
654 return -EFAULT;
621 655
622 remain -= page_length; 656 remain -= page_length;
623 user_data += page_length; 657 user_data += page_length;
624 offset += page_length; 658 offset += page_length;
625 } 659 }
626 660
627fail: 661 return 0;
628 i915_gem_object_unpin(obj);
629 mutex_unlock(&dev->struct_mutex);
630
631 return ret;
632} 662}
633 663
634/** 664/**
@@ -639,11 +669,11 @@ fail:
639 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). 669 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
640 */ 670 */
641static int 671static int
642i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 672i915_gem_gtt_pwrite_slow(struct drm_device *dev,
673 struct drm_i915_gem_object *obj,
643 struct drm_i915_gem_pwrite *args, 674 struct drm_i915_gem_pwrite *args,
644 struct drm_file *file_priv) 675 struct drm_file *file)
645{ 676{
646 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
647 drm_i915_private_t *dev_priv = dev->dev_private; 677 drm_i915_private_t *dev_priv = dev->dev_private;
648 ssize_t remain; 678 ssize_t remain;
649 loff_t gtt_page_base, offset; 679 loff_t gtt_page_base, offset;
@@ -665,30 +695,30 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
665 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 695 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
666 num_pages = last_data_page - first_data_page + 1; 696 num_pages = last_data_page - first_data_page + 1;
667 697
668 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 698 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
669 if (user_pages == NULL) 699 if (user_pages == NULL)
670 return -ENOMEM; 700 return -ENOMEM;
671 701
702 mutex_unlock(&dev->struct_mutex);
672 down_read(&mm->mmap_sem); 703 down_read(&mm->mmap_sem);
673 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 704 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
674 num_pages, 0, 0, user_pages, NULL); 705 num_pages, 0, 0, user_pages, NULL);
675 up_read(&mm->mmap_sem); 706 up_read(&mm->mmap_sem);
707 mutex_lock(&dev->struct_mutex);
676 if (pinned_pages < num_pages) { 708 if (pinned_pages < num_pages) {
677 ret = -EFAULT; 709 ret = -EFAULT;
678 goto out_unpin_pages; 710 goto out_unpin_pages;
679 } 711 }
680 712
681 mutex_lock(&dev->struct_mutex); 713 ret = i915_gem_object_set_to_gtt_domain(obj, true);
682 ret = i915_gem_object_pin(obj, 0);
683 if (ret) 714 if (ret)
684 goto out_unlock; 715 goto out_unpin_pages;
685 716
686 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 717 ret = i915_gem_object_put_fence(obj);
687 if (ret) 718 if (ret)
688 goto out_unpin_object; 719 goto out_unpin_pages;
689 720
690 obj_priv = to_intel_bo(obj); 721 offset = obj->gtt_offset + args->offset;
691 offset = obj_priv->gtt_offset + args->offset;
692 722
693 while (remain > 0) { 723 while (remain > 0) {
694 /* Operation in this page 724 /* Operation in this page
@@ -700,9 +730,9 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
700 * page_length = bytes to copy for this page 730 * page_length = bytes to copy for this page
701 */ 731 */
702 gtt_page_base = offset & PAGE_MASK; 732 gtt_page_base = offset & PAGE_MASK;
703 gtt_page_offset = offset & ~PAGE_MASK; 733 gtt_page_offset = offset_in_page(offset);
704 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 734 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
705 data_page_offset = data_ptr & ~PAGE_MASK; 735 data_page_offset = offset_in_page(data_ptr);
706 736
707 page_length = remain; 737 page_length = remain;
708 if ((gtt_page_offset + page_length) > PAGE_SIZE) 738 if ((gtt_page_offset + page_length) > PAGE_SIZE)
@@ -721,10 +751,6 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
721 data_ptr += page_length; 751 data_ptr += page_length;
722 } 752 }
723 753
724out_unpin_object:
725 i915_gem_object_unpin(obj);
726out_unlock:
727 mutex_unlock(&dev->struct_mutex);
728out_unpin_pages: 754out_unpin_pages:
729 for (i = 0; i < pinned_pages; i++) 755 for (i = 0; i < pinned_pages; i++)
730 page_cache_release(user_pages[i]); 756 page_cache_release(user_pages[i]);
@@ -738,64 +764,65 @@ out_unpin_pages:
738 * copy_from_user into the kmapped pages backing the object. 764 * copy_from_user into the kmapped pages backing the object.
739 */ 765 */
740static int 766static int
741i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 767i915_gem_shmem_pwrite_fast(struct drm_device *dev,
768 struct drm_i915_gem_object *obj,
742 struct drm_i915_gem_pwrite *args, 769 struct drm_i915_gem_pwrite *args,
743 struct drm_file *file_priv) 770 struct drm_file *file)
744{ 771{
745 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 772 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
746 ssize_t remain; 773 ssize_t remain;
747 loff_t offset, page_base; 774 loff_t offset;
748 char __user *user_data; 775 char __user *user_data;
749 int page_offset, page_length; 776 int page_offset, page_length;
750 int ret;
751 777
752 user_data = (char __user *) (uintptr_t) args->data_ptr; 778 user_data = (char __user *) (uintptr_t) args->data_ptr;
753 remain = args->size; 779 remain = args->size;
754 780
755 mutex_lock(&dev->struct_mutex);
756
757 ret = i915_gem_object_get_pages(obj, 0);
758 if (ret != 0)
759 goto fail_unlock;
760
761 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
762 if (ret != 0)
763 goto fail_put_pages;
764
765 obj_priv = to_intel_bo(obj);
766 offset = args->offset; 781 offset = args->offset;
767 obj_priv->dirty = 1; 782 obj->dirty = 1;
768 783
769 while (remain > 0) { 784 while (remain > 0) {
785 struct page *page;
786 char *vaddr;
787 int ret;
788
770 /* Operation in this page 789 /* Operation in this page
771 * 790 *
772 * page_base = page offset within aperture
773 * page_offset = offset within page 791 * page_offset = offset within page
774 * page_length = bytes to copy for this page 792 * page_length = bytes to copy for this page
775 */ 793 */
776 page_base = (offset & ~(PAGE_SIZE-1)); 794 page_offset = offset_in_page(offset);
777 page_offset = offset & (PAGE_SIZE-1);
778 page_length = remain; 795 page_length = remain;
779 if ((page_offset + remain) > PAGE_SIZE) 796 if ((page_offset + remain) > PAGE_SIZE)
780 page_length = PAGE_SIZE - page_offset; 797 page_length = PAGE_SIZE - page_offset;
781 798
782 ret = fast_shmem_write(obj_priv->pages, 799 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
783 page_base, page_offset, 800 if (IS_ERR(page))
784 user_data, page_length); 801 return PTR_ERR(page);
802
803 vaddr = kmap_atomic(page, KM_USER0);
804 ret = __copy_from_user_inatomic(vaddr + page_offset,
805 user_data,
806 page_length);
807 kunmap_atomic(vaddr, KM_USER0);
808
809 set_page_dirty(page);
810 mark_page_accessed(page);
811 page_cache_release(page);
812
813 /* If we get a fault while copying data, then (presumably) our
814 * source page isn't available. Return the error and we'll
815 * retry in the slow path.
816 */
785 if (ret) 817 if (ret)
786 goto fail_put_pages; 818 return -EFAULT;
787 819
788 remain -= page_length; 820 remain -= page_length;
789 user_data += page_length; 821 user_data += page_length;
790 offset += page_length; 822 offset += page_length;
791 } 823 }
792 824
793fail_put_pages: 825 return 0;
794 i915_gem_object_put_pages(obj);
795fail_unlock:
796 mutex_unlock(&dev->struct_mutex);
797
798 return ret;
799} 826}
800 827
801/** 828/**
@@ -806,17 +833,18 @@ fail_unlock:
806 * struct_mutex is held. 833 * struct_mutex is held.
807 */ 834 */
808static int 835static int
809i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 836i915_gem_shmem_pwrite_slow(struct drm_device *dev,
837 struct drm_i915_gem_object *obj,
810 struct drm_i915_gem_pwrite *args, 838 struct drm_i915_gem_pwrite *args,
811 struct drm_file *file_priv) 839 struct drm_file *file)
812{ 840{
813 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 841 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
814 struct mm_struct *mm = current->mm; 842 struct mm_struct *mm = current->mm;
815 struct page **user_pages; 843 struct page **user_pages;
816 ssize_t remain; 844 ssize_t remain;
817 loff_t offset, pinned_pages, i; 845 loff_t offset, pinned_pages, i;
818 loff_t first_data_page, last_data_page, num_pages; 846 loff_t first_data_page, last_data_page, num_pages;
819 int shmem_page_index, shmem_page_offset; 847 int shmem_page_offset;
820 int data_page_index, data_page_offset; 848 int data_page_index, data_page_offset;
821 int page_length; 849 int page_length;
822 int ret; 850 int ret;
@@ -833,48 +861,43 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
833 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 861 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
834 num_pages = last_data_page - first_data_page + 1; 862 num_pages = last_data_page - first_data_page + 1;
835 863
836 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 864 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
837 if (user_pages == NULL) 865 if (user_pages == NULL)
838 return -ENOMEM; 866 return -ENOMEM;
839 867
868 mutex_unlock(&dev->struct_mutex);
840 down_read(&mm->mmap_sem); 869 down_read(&mm->mmap_sem);
841 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 870 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
842 num_pages, 0, 0, user_pages, NULL); 871 num_pages, 0, 0, user_pages, NULL);
843 up_read(&mm->mmap_sem); 872 up_read(&mm->mmap_sem);
873 mutex_lock(&dev->struct_mutex);
844 if (pinned_pages < num_pages) { 874 if (pinned_pages < num_pages) {
845 ret = -EFAULT; 875 ret = -EFAULT;
846 goto fail_put_user_pages; 876 goto out;
847 } 877 }
848 878
849 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 879 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
850
851 mutex_lock(&dev->struct_mutex);
852
853 ret = i915_gem_object_get_pages_or_evict(obj);
854 if (ret) 880 if (ret)
855 goto fail_unlock; 881 goto out;
856 882
857 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 883 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
858 if (ret != 0)
859 goto fail_put_pages;
860 884
861 obj_priv = to_intel_bo(obj);
862 offset = args->offset; 885 offset = args->offset;
863 obj_priv->dirty = 1; 886 obj->dirty = 1;
864 887
865 while (remain > 0) { 888 while (remain > 0) {
889 struct page *page;
890
866 /* Operation in this page 891 /* Operation in this page
867 * 892 *
868 * shmem_page_index = page number within shmem file
869 * shmem_page_offset = offset within page in shmem file 893 * shmem_page_offset = offset within page in shmem file
870 * data_page_index = page number in get_user_pages return 894 * data_page_index = page number in get_user_pages return
871 * data_page_offset = offset with data_page_index page. 895 * data_page_offset = offset with data_page_index page.
872 * page_length = bytes to copy for this page 896 * page_length = bytes to copy for this page
873 */ 897 */
874 shmem_page_index = offset / PAGE_SIZE; 898 shmem_page_offset = offset_in_page(offset);
875 shmem_page_offset = offset & ~PAGE_MASK;
876 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 899 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
877 data_page_offset = data_ptr & ~PAGE_MASK; 900 data_page_offset = offset_in_page(data_ptr);
878 901
879 page_length = remain; 902 page_length = remain;
880 if ((shmem_page_offset + page_length) > PAGE_SIZE) 903 if ((shmem_page_offset + page_length) > PAGE_SIZE)
@@ -882,31 +905,37 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
882 if ((data_page_offset + page_length) > PAGE_SIZE) 905 if ((data_page_offset + page_length) > PAGE_SIZE)
883 page_length = PAGE_SIZE - data_page_offset; 906 page_length = PAGE_SIZE - data_page_offset;
884 907
908 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
909 if (IS_ERR(page)) {
910 ret = PTR_ERR(page);
911 goto out;
912 }
913
885 if (do_bit17_swizzling) { 914 if (do_bit17_swizzling) {
886 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 915 slow_shmem_bit17_copy(page,
887 shmem_page_offset, 916 shmem_page_offset,
888 user_pages[data_page_index], 917 user_pages[data_page_index],
889 data_page_offset, 918 data_page_offset,
890 page_length, 919 page_length,
891 0); 920 0);
892 } else { 921 } else {
893 slow_shmem_copy(obj_priv->pages[shmem_page_index], 922 slow_shmem_copy(page,
894 shmem_page_offset, 923 shmem_page_offset,
895 user_pages[data_page_index], 924 user_pages[data_page_index],
896 data_page_offset, 925 data_page_offset,
897 page_length); 926 page_length);
898 } 927 }
899 928
929 set_page_dirty(page);
930 mark_page_accessed(page);
931 page_cache_release(page);
932
900 remain -= page_length; 933 remain -= page_length;
901 data_ptr += page_length; 934 data_ptr += page_length;
902 offset += page_length; 935 offset += page_length;
903 } 936 }
904 937
905fail_put_pages: 938out:
906 i915_gem_object_put_pages(obj);
907fail_unlock:
908 mutex_unlock(&dev->struct_mutex);
909fail_put_user_pages:
910 for (i = 0; i < pinned_pages; i++) 939 for (i = 0; i < pinned_pages; i++)
911 page_cache_release(user_pages[i]); 940 page_cache_release(user_pages[i]);
912 drm_free_large(user_pages); 941 drm_free_large(user_pages);
@@ -921,30 +950,43 @@ fail_put_user_pages:
921 */ 950 */
922int 951int
923i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 952i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
924 struct drm_file *file_priv) 953 struct drm_file *file)
925{ 954{
926 struct drm_i915_gem_pwrite *args = data; 955 struct drm_i915_gem_pwrite *args = data;
927 struct drm_gem_object *obj; 956 struct drm_i915_gem_object *obj;
928 struct drm_i915_gem_object *obj_priv; 957 int ret;
929 int ret = 0;
930 958
931 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 959 if (args->size == 0)
932 if (obj == NULL) 960 return 0;
933 return -ENOENT; 961
934 obj_priv = to_intel_bo(obj); 962 if (!access_ok(VERIFY_READ,
963 (char __user *)(uintptr_t)args->data_ptr,
964 args->size))
965 return -EFAULT;
966
967 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
968 args->size);
969 if (ret)
970 return -EFAULT;
971
972 ret = i915_mutex_lock_interruptible(dev);
973 if (ret)
974 return ret;
975
976 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
977 if (&obj->base == NULL) {
978 ret = -ENOENT;
979 goto unlock;
980 }
935 981
936 /* Bounds check destination. */ 982 /* Bounds check destination. */
937 if (args->offset > obj->size || args->size > obj->size - args->offset) { 983 if (args->offset > obj->base.size ||
984 args->size > obj->base.size - args->offset) {
938 ret = -EINVAL; 985 ret = -EINVAL;
939 goto err; 986 goto out;
940 } 987 }
941 988
942 if (!access_ok(VERIFY_READ, 989 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
943 (char __user *)(uintptr_t)args->data_ptr,
944 args->size)) {
945 ret = -EFAULT;
946 goto err;
947 }
948 990
949 /* We can only do the GTT pwrite on untiled buffers, as otherwise 991 /* We can only do the GTT pwrite on untiled buffers, as otherwise
950 * it would end up going through the fenced access, and we'll get 992 * it would end up going through the fenced access, and we'll get
@@ -952,33 +994,44 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
952 * pread/pwrite currently are reading and writing from the CPU 994 * pread/pwrite currently are reading and writing from the CPU
953 * perspective, requiring manual detiling by the client. 995 * perspective, requiring manual detiling by the client.
954 */ 996 */
955 if (obj_priv->phys_obj) 997 if (obj->phys_obj)
956 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv); 998 ret = i915_gem_phys_pwrite(dev, obj, args, file);
957 else if (obj_priv->tiling_mode == I915_TILING_NONE && 999 else if (obj->gtt_space &&
958 dev->gtt_total != 0 && 1000 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
959 obj->write_domain != I915_GEM_DOMAIN_CPU) { 1001 ret = i915_gem_object_pin(obj, 0, true);
960 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv); 1002 if (ret)
961 if (ret == -EFAULT) { 1003 goto out;
962 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, 1004
963 file_priv); 1005 ret = i915_gem_object_set_to_gtt_domain(obj, true);
964 } 1006 if (ret)
965 } else if (i915_gem_object_needs_bit17_swizzle(obj)) { 1007 goto out_unpin;
966 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv); 1008
1009 ret = i915_gem_object_put_fence(obj);
1010 if (ret)
1011 goto out_unpin;
1012
1013 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1014 if (ret == -EFAULT)
1015 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1016
1017out_unpin:
1018 i915_gem_object_unpin(obj);
967 } else { 1019 } else {
968 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv); 1020 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
969 if (ret == -EFAULT) { 1021 if (ret)
970 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, 1022 goto out;
971 file_priv);
972 }
973 }
974 1023
975#if WATCH_PWRITE 1024 ret = -EFAULT;
976 if (ret) 1025 if (!i915_gem_object_needs_bit17_swizzle(obj))
977 DRM_INFO("pwrite failed %d\n", ret); 1026 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
978#endif 1027 if (ret == -EFAULT)
1028 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1029 }
979 1030
980err: 1031out:
981 drm_gem_object_unreference_unlocked(obj); 1032 drm_gem_object_unreference(&obj->base);
1033unlock:
1034 mutex_unlock(&dev->struct_mutex);
982 return ret; 1035 return ret;
983} 1036}
984 1037
@@ -988,12 +1041,10 @@ err:
988 */ 1041 */
989int 1042int
990i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1043i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
991 struct drm_file *file_priv) 1044 struct drm_file *file)
992{ 1045{
993 struct drm_i915_private *dev_priv = dev->dev_private;
994 struct drm_i915_gem_set_domain *args = data; 1046 struct drm_i915_gem_set_domain *args = data;
995 struct drm_gem_object *obj; 1047 struct drm_i915_gem_object *obj;
996 struct drm_i915_gem_object *obj_priv;
997 uint32_t read_domains = args->read_domains; 1048 uint32_t read_domains = args->read_domains;
998 uint32_t write_domain = args->write_domain; 1049 uint32_t write_domain = args->write_domain;
999 int ret; 1050 int ret;
@@ -1014,32 +1065,19 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1014 if (write_domain != 0 && read_domains != write_domain) 1065 if (write_domain != 0 && read_domains != write_domain)
1015 return -EINVAL; 1066 return -EINVAL;
1016 1067
1017 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1068 ret = i915_mutex_lock_interruptible(dev);
1018 if (obj == NULL) 1069 if (ret)
1019 return -ENOENT; 1070 return ret;
1020 obj_priv = to_intel_bo(obj);
1021
1022 mutex_lock(&dev->struct_mutex);
1023 1071
1024 intel_mark_busy(dev, obj); 1072 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1073 if (&obj->base == NULL) {
1074 ret = -ENOENT;
1075 goto unlock;
1076 }
1025 1077
1026#if WATCH_BUF
1027 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1028 obj, obj->size, read_domains, write_domain);
1029#endif
1030 if (read_domains & I915_GEM_DOMAIN_GTT) { 1078 if (read_domains & I915_GEM_DOMAIN_GTT) {
1031 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1079 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1032 1080
1033 /* Update the LRU on the fence for the CPU access that's
1034 * about to occur.
1035 */
1036 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1037 struct drm_i915_fence_reg *reg =
1038 &dev_priv->fence_regs[obj_priv->fence_reg];
1039 list_move_tail(&reg->lru_list,
1040 &dev_priv->mm.fence_list);
1041 }
1042
1043 /* Silently promote "you're not bound, there was nothing to do" 1081 /* Silently promote "you're not bound, there was nothing to do"
1044 * to success, since the client was just asking us to 1082 * to success, since the client was just asking us to
1045 * make sure everything was done. 1083 * make sure everything was done.
@@ -1050,12 +1088,8 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1050 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1088 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1051 } 1089 }
1052 1090
1053 1091 drm_gem_object_unreference(&obj->base);
1054 /* Maintain LRU order of "inactive" objects */ 1092unlock:
1055 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1056 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1057
1058 drm_gem_object_unreference(obj);
1059 mutex_unlock(&dev->struct_mutex); 1093 mutex_unlock(&dev->struct_mutex);
1060 return ret; 1094 return ret;
1061} 1095}
@@ -1065,34 +1099,31 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1065 */ 1099 */
1066int 1100int
1067i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1101i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1068 struct drm_file *file_priv) 1102 struct drm_file *file)
1069{ 1103{
1070 struct drm_i915_gem_sw_finish *args = data; 1104 struct drm_i915_gem_sw_finish *args = data;
1071 struct drm_gem_object *obj; 1105 struct drm_i915_gem_object *obj;
1072 struct drm_i915_gem_object *obj_priv;
1073 int ret = 0; 1106 int ret = 0;
1074 1107
1075 if (!(dev->driver->driver_features & DRIVER_GEM)) 1108 if (!(dev->driver->driver_features & DRIVER_GEM))
1076 return -ENODEV; 1109 return -ENODEV;
1077 1110
1078 mutex_lock(&dev->struct_mutex); 1111 ret = i915_mutex_lock_interruptible(dev);
1079 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1112 if (ret)
1080 if (obj == NULL) { 1113 return ret;
1081 mutex_unlock(&dev->struct_mutex);
1082 return -ENOENT;
1083 }
1084 1114
1085#if WATCH_BUF 1115 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1086 DRM_INFO("%s: sw_finish %d (%p %zd)\n", 1116 if (&obj->base == NULL) {
1087 __func__, args->handle, obj, obj->size); 1117 ret = -ENOENT;
1088#endif 1118 goto unlock;
1089 obj_priv = to_intel_bo(obj); 1119 }
1090 1120
1091 /* Pinned buffers may be scanout, so flush the cache */ 1121 /* Pinned buffers may be scanout, so flush the cache */
1092 if (obj_priv->pin_count) 1122 if (obj->pin_count)
1093 i915_gem_object_flush_cpu_write_domain(obj); 1123 i915_gem_object_flush_cpu_write_domain(obj);
1094 1124
1095 drm_gem_object_unreference(obj); 1125 drm_gem_object_unreference(&obj->base);
1126unlock:
1096 mutex_unlock(&dev->struct_mutex); 1127 mutex_unlock(&dev->struct_mutex);
1097 return ret; 1128 return ret;
1098} 1129}
@@ -1106,21 +1137,24 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1106 */ 1137 */
1107int 1138int
1108i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1139i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1109 struct drm_file *file_priv) 1140 struct drm_file *file)
1110{ 1141{
1142 struct drm_i915_private *dev_priv = dev->dev_private;
1111 struct drm_i915_gem_mmap *args = data; 1143 struct drm_i915_gem_mmap *args = data;
1112 struct drm_gem_object *obj; 1144 struct drm_gem_object *obj;
1113 loff_t offset;
1114 unsigned long addr; 1145 unsigned long addr;
1115 1146
1116 if (!(dev->driver->driver_features & DRIVER_GEM)) 1147 if (!(dev->driver->driver_features & DRIVER_GEM))
1117 return -ENODEV; 1148 return -ENODEV;
1118 1149
1119 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1150 obj = drm_gem_object_lookup(dev, file, args->handle);
1120 if (obj == NULL) 1151 if (obj == NULL)
1121 return -ENOENT; 1152 return -ENOENT;
1122 1153
1123 offset = args->offset; 1154 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1155 drm_gem_object_unreference_unlocked(obj);
1156 return -E2BIG;
1157 }
1124 1158
1125 down_write(&current->mm->mmap_sem); 1159 down_write(&current->mm->mmap_sem);
1126 addr = do_mmap(obj->filp, 0, args->size, 1160 addr = do_mmap(obj->filp, 0, args->size,
@@ -1154,10 +1188,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1154 */ 1188 */
1155int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1189int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1156{ 1190{
1157 struct drm_gem_object *obj = vma->vm_private_data; 1191 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1158 struct drm_device *dev = obj->dev; 1192 struct drm_device *dev = obj->base.dev;
1159 drm_i915_private_t *dev_priv = dev->dev_private; 1193 drm_i915_private_t *dev_priv = dev->dev_private;
1160 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1161 pgoff_t page_offset; 1194 pgoff_t page_offset;
1162 unsigned long pfn; 1195 unsigned long pfn;
1163 int ret = 0; 1196 int ret = 0;
@@ -1167,42 +1200,64 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1167 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> 1200 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1168 PAGE_SHIFT; 1201 PAGE_SHIFT;
1169 1202
1203 ret = i915_mutex_lock_interruptible(dev);
1204 if (ret)
1205 goto out;
1206
1207 trace_i915_gem_object_fault(obj, page_offset, true, write);
1208
1170 /* Now bind it into the GTT if needed */ 1209 /* Now bind it into the GTT if needed */
1171 mutex_lock(&dev->struct_mutex); 1210 if (!obj->map_and_fenceable) {
1172 if (!obj_priv->gtt_space) { 1211 ret = i915_gem_object_unbind(obj);
1173 ret = i915_gem_object_bind_to_gtt(obj, 0);
1174 if (ret) 1212 if (ret)
1175 goto unlock; 1213 goto unlock;
1176 1214 }
1177 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1215 if (!obj->gtt_space) {
1216 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1178 if (ret) 1217 if (ret)
1179 goto unlock; 1218 goto unlock;
1180 }
1181 1219
1182 /* Need a new fence register? */ 1220 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1183 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1184 ret = i915_gem_object_get_fence_reg(obj);
1185 if (ret) 1221 if (ret)
1186 goto unlock; 1222 goto unlock;
1187 } 1223 }
1188 1224
1189 if (i915_gem_object_is_inactive(obj_priv)) 1225 if (obj->tiling_mode == I915_TILING_NONE)
1190 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1226 ret = i915_gem_object_put_fence(obj);
1227 else
1228 ret = i915_gem_object_get_fence(obj, NULL);
1229 if (ret)
1230 goto unlock;
1231
1232 if (i915_gem_object_is_inactive(obj))
1233 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1234
1235 obj->fault_mappable = true;
1191 1236
1192 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1237 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1193 page_offset; 1238 page_offset;
1194 1239
1195 /* Finally, remap it using the new GTT offset */ 1240 /* Finally, remap it using the new GTT offset */
1196 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); 1241 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1197unlock: 1242unlock:
1198 mutex_unlock(&dev->struct_mutex); 1243 mutex_unlock(&dev->struct_mutex);
1199 1244out:
1200 switch (ret) { 1245 switch (ret) {
1246 case -EIO:
1247 case -EAGAIN:
1248 /* Give the error handler a chance to run and move the
1249 * objects off the GPU active list. Next time we service the
1250 * fault, we should be able to transition the page into the
1251 * GTT without touching the GPU (and so avoid further
1252 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1253 * with coherency, just lost writes.
1254 */
1255 set_need_resched();
1201 case 0: 1256 case 0:
1202 case -ERESTARTSYS: 1257 case -ERESTARTSYS:
1258 case -EINTR:
1203 return VM_FAULT_NOPAGE; 1259 return VM_FAULT_NOPAGE;
1204 case -ENOMEM: 1260 case -ENOMEM:
1205 case -EAGAIN:
1206 return VM_FAULT_OOM; 1261 return VM_FAULT_OOM;
1207 default: 1262 default:
1208 return VM_FAULT_SIGBUS; 1263 return VM_FAULT_SIGBUS;
@@ -1221,59 +1276,58 @@ unlock:
1221 * This routine allocates and attaches a fake offset for @obj. 1276 * This routine allocates and attaches a fake offset for @obj.
1222 */ 1277 */
1223static int 1278static int
1224i915_gem_create_mmap_offset(struct drm_gem_object *obj) 1279i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1225{ 1280{
1226 struct drm_device *dev = obj->dev; 1281 struct drm_device *dev = obj->base.dev;
1227 struct drm_gem_mm *mm = dev->mm_private; 1282 struct drm_gem_mm *mm = dev->mm_private;
1228 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1229 struct drm_map_list *list; 1283 struct drm_map_list *list;
1230 struct drm_local_map *map; 1284 struct drm_local_map *map;
1231 int ret = 0; 1285 int ret = 0;
1232 1286
1233 /* Set the object up for mmap'ing */ 1287 /* Set the object up for mmap'ing */
1234 list = &obj->map_list; 1288 list = &obj->base.map_list;
1235 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); 1289 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1236 if (!list->map) 1290 if (!list->map)
1237 return -ENOMEM; 1291 return -ENOMEM;
1238 1292
1239 map = list->map; 1293 map = list->map;
1240 map->type = _DRM_GEM; 1294 map->type = _DRM_GEM;
1241 map->size = obj->size; 1295 map->size = obj->base.size;
1242 map->handle = obj; 1296 map->handle = obj;
1243 1297
1244 /* Get a DRM GEM mmap offset allocated... */ 1298 /* Get a DRM GEM mmap offset allocated... */
1245 list->file_offset_node = drm_mm_search_free(&mm->offset_manager, 1299 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1246 obj->size / PAGE_SIZE, 0, 0); 1300 obj->base.size / PAGE_SIZE,
1301 0, 0);
1247 if (!list->file_offset_node) { 1302 if (!list->file_offset_node) {
1248 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); 1303 DRM_ERROR("failed to allocate offset for bo %d\n",
1249 ret = -ENOMEM; 1304 obj->base.name);
1305 ret = -ENOSPC;
1250 goto out_free_list; 1306 goto out_free_list;
1251 } 1307 }
1252 1308
1253 list->file_offset_node = drm_mm_get_block(list->file_offset_node, 1309 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1254 obj->size / PAGE_SIZE, 0); 1310 obj->base.size / PAGE_SIZE,
1311 0);
1255 if (!list->file_offset_node) { 1312 if (!list->file_offset_node) {
1256 ret = -ENOMEM; 1313 ret = -ENOMEM;
1257 goto out_free_list; 1314 goto out_free_list;
1258 } 1315 }
1259 1316
1260 list->hash.key = list->file_offset_node->start; 1317 list->hash.key = list->file_offset_node->start;
1261 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) { 1318 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1319 if (ret) {
1262 DRM_ERROR("failed to add to map hash\n"); 1320 DRM_ERROR("failed to add to map hash\n");
1263 ret = -ENOMEM;
1264 goto out_free_mm; 1321 goto out_free_mm;
1265 } 1322 }
1266 1323
1267 /* By now we should be all set, any drm_mmap request on the offset
1268 * below will get to our mmap & fault handler */
1269 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1270
1271 return 0; 1324 return 0;
1272 1325
1273out_free_mm: 1326out_free_mm:
1274 drm_mm_put_block(list->file_offset_node); 1327 drm_mm_put_block(list->file_offset_node);
1275out_free_list: 1328out_free_list:
1276 kfree(list->map); 1329 kfree(list->map);
1330 list->map = NULL;
1277 1331
1278 return ret; 1332 return ret;
1279} 1333}
@@ -1293,38 +1347,51 @@ out_free_list:
1293 * fixup by i915_gem_fault(). 1347 * fixup by i915_gem_fault().
1294 */ 1348 */
1295void 1349void
1296i915_gem_release_mmap(struct drm_gem_object *obj) 1350i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1297{ 1351{
1298 struct drm_device *dev = obj->dev; 1352 if (!obj->fault_mappable)
1299 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1353 return;
1354
1355 if (obj->base.dev->dev_mapping)
1356 unmap_mapping_range(obj->base.dev->dev_mapping,
1357 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1358 obj->base.size, 1);
1300 1359
1301 if (dev->dev_mapping) 1360 obj->fault_mappable = false;
1302 unmap_mapping_range(dev->dev_mapping,
1303 obj_priv->mmap_offset, obj->size, 1);
1304} 1361}
1305 1362
1306static void 1363static void
1307i915_gem_free_mmap_offset(struct drm_gem_object *obj) 1364i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1308{ 1365{
1309 struct drm_device *dev = obj->dev; 1366 struct drm_device *dev = obj->base.dev;
1310 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1311 struct drm_gem_mm *mm = dev->mm_private; 1367 struct drm_gem_mm *mm = dev->mm_private;
1312 struct drm_map_list *list; 1368 struct drm_map_list *list = &obj->base.map_list;
1313 1369
1314 list = &obj->map_list;
1315 drm_ht_remove_item(&mm->offset_hash, &list->hash); 1370 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1371 drm_mm_put_block(list->file_offset_node);
1372 kfree(list->map);
1373 list->map = NULL;
1374}
1316 1375
1317 if (list->file_offset_node) { 1376static uint32_t
1318 drm_mm_put_block(list->file_offset_node); 1377i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1319 list->file_offset_node = NULL; 1378{
1320 } 1379 uint32_t gtt_size;
1321 1380
1322 if (list->map) { 1381 if (INTEL_INFO(dev)->gen >= 4 ||
1323 kfree(list->map); 1382 tiling_mode == I915_TILING_NONE)
1324 list->map = NULL; 1383 return size;
1325 } 1384
1385 /* Previous chips need a power-of-two fence region when tiling */
1386 if (INTEL_INFO(dev)->gen == 3)
1387 gtt_size = 1024*1024;
1388 else
1389 gtt_size = 512*1024;
1390
1391 while (gtt_size < size)
1392 gtt_size <<= 1;
1326 1393
1327 obj_priv->mmap_offset = 0; 1394 return gtt_size;
1328} 1395}
1329 1396
1330/** 1397/**
@@ -1332,42 +1399,111 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1332 * @obj: object to check 1399 * @obj: object to check
1333 * 1400 *
1334 * Return the required GTT alignment for an object, taking into account 1401 * Return the required GTT alignment for an object, taking into account
1335 * potential fence register mapping if needed. 1402 * potential fence register mapping.
1336 */ 1403 */
1337static uint32_t 1404static uint32_t
1338i915_gem_get_gtt_alignment(struct drm_gem_object *obj) 1405i915_gem_get_gtt_alignment(struct drm_device *dev,
1406 uint32_t size,
1407 int tiling_mode)
1339{ 1408{
1340 struct drm_device *dev = obj->dev;
1341 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1342 int start, i;
1343
1344 /* 1409 /*
1345 * Minimum alignment is 4k (GTT page size), but might be greater 1410 * Minimum alignment is 4k (GTT page size), but might be greater
1346 * if a fence register is needed for the object. 1411 * if a fence register is needed for the object.
1347 */ 1412 */
1348 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE) 1413 if (INTEL_INFO(dev)->gen >= 4 ||
1414 tiling_mode == I915_TILING_NONE)
1349 return 4096; 1415 return 4096;
1350 1416
1351 /* 1417 /*
1352 * Previous chips need to be aligned to the size of the smallest 1418 * Previous chips need to be aligned to the size of the smallest
1353 * fence register that can contain the object. 1419 * fence register that can contain the object.
1354 */ 1420 */
1355 if (IS_I9XX(dev)) 1421 return i915_gem_get_gtt_size(dev, size, tiling_mode);
1356 start = 1024*1024; 1422}
1357 else
1358 start = 512*1024;
1359 1423
1360 for (i = start; i < obj->size; i <<= 1) 1424/**
1361 ; 1425 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1426 * unfenced object
1427 * @dev: the device
1428 * @size: size of the object
1429 * @tiling_mode: tiling mode of the object
1430 *
1431 * Return the required GTT alignment for an object, only taking into account
1432 * unfenced tiled surface requirements.
1433 */
1434uint32_t
1435i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1436 uint32_t size,
1437 int tiling_mode)
1438{
1439 /*
1440 * Minimum alignment is 4k (GTT page size) for sane hw.
1441 */
1442 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1443 tiling_mode == I915_TILING_NONE)
1444 return 4096;
1362 1445
1363 return i; 1446 /* Previous hardware however needs to be aligned to a power-of-two
1447 * tile height. The simplest method for determining this is to reuse
1448 * the power-of-tile object size.
1449 */
1450 return i915_gem_get_gtt_size(dev, size, tiling_mode);
1451}
1452
1453int
1454i915_gem_mmap_gtt(struct drm_file *file,
1455 struct drm_device *dev,
1456 uint32_t handle,
1457 uint64_t *offset)
1458{
1459 struct drm_i915_private *dev_priv = dev->dev_private;
1460 struct drm_i915_gem_object *obj;
1461 int ret;
1462
1463 if (!(dev->driver->driver_features & DRIVER_GEM))
1464 return -ENODEV;
1465
1466 ret = i915_mutex_lock_interruptible(dev);
1467 if (ret)
1468 return ret;
1469
1470 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1471 if (&obj->base == NULL) {
1472 ret = -ENOENT;
1473 goto unlock;
1474 }
1475
1476 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1477 ret = -E2BIG;
1478 goto unlock;
1479 }
1480
1481 if (obj->madv != I915_MADV_WILLNEED) {
1482 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1483 ret = -EINVAL;
1484 goto out;
1485 }
1486
1487 if (!obj->base.map_list.map) {
1488 ret = i915_gem_create_mmap_offset(obj);
1489 if (ret)
1490 goto out;
1491 }
1492
1493 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1494
1495out:
1496 drm_gem_object_unreference(&obj->base);
1497unlock:
1498 mutex_unlock(&dev->struct_mutex);
1499 return ret;
1364} 1500}
1365 1501
1366/** 1502/**
1367 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1503 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1368 * @dev: DRM device 1504 * @dev: DRM device
1369 * @data: GTT mapping ioctl data 1505 * @data: GTT mapping ioctl data
1370 * @file_priv: GEM object info 1506 * @file: GEM object info
1371 * 1507 *
1372 * Simply returns the fake offset to userspace so it can mmap it. 1508 * Simply returns the fake offset to userspace so it can mmap it.
1373 * The mmap call will end up in drm_gem_mmap(), which will set things 1509 * The mmap call will end up in drm_gem_mmap(), which will set things
@@ -1380,236 +1516,233 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1380 */ 1516 */
1381int 1517int
1382i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1518i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1383 struct drm_file *file_priv) 1519 struct drm_file *file)
1384{ 1520{
1385 struct drm_i915_gem_mmap_gtt *args = data; 1521 struct drm_i915_gem_mmap_gtt *args = data;
1386 struct drm_gem_object *obj;
1387 struct drm_i915_gem_object *obj_priv;
1388 int ret;
1389 1522
1390 if (!(dev->driver->driver_features & DRIVER_GEM)) 1523 if (!(dev->driver->driver_features & DRIVER_GEM))
1391 return -ENODEV; 1524 return -ENODEV;
1392 1525
1393 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1526 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1394 if (obj == NULL) 1527}
1395 return -ENOENT;
1396
1397 mutex_lock(&dev->struct_mutex);
1398 1528
1399 obj_priv = to_intel_bo(obj);
1400 1529
1401 if (obj_priv->madv != I915_MADV_WILLNEED) { 1530static int
1402 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1531i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1403 drm_gem_object_unreference(obj); 1532 gfp_t gfpmask)
1404 mutex_unlock(&dev->struct_mutex); 1533{
1405 return -EINVAL; 1534 int page_count, i;
1406 } 1535 struct address_space *mapping;
1536 struct inode *inode;
1537 struct page *page;
1407 1538
1539 /* Get the list of pages out of our struct file. They'll be pinned
1540 * at this point until we release them.
1541 */
1542 page_count = obj->base.size / PAGE_SIZE;
1543 BUG_ON(obj->pages != NULL);
1544 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1545 if (obj->pages == NULL)
1546 return -ENOMEM;
1408 1547
1409 if (!obj_priv->mmap_offset) { 1548 inode = obj->base.filp->f_path.dentry->d_inode;
1410 ret = i915_gem_create_mmap_offset(obj); 1549 mapping = inode->i_mapping;
1411 if (ret) { 1550 gfpmask |= mapping_gfp_mask(mapping);
1412 drm_gem_object_unreference(obj);
1413 mutex_unlock(&dev->struct_mutex);
1414 return ret;
1415 }
1416 }
1417 1551
1418 args->offset = obj_priv->mmap_offset; 1552 for (i = 0; i < page_count; i++) {
1553 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1554 if (IS_ERR(page))
1555 goto err_pages;
1419 1556
1420 /* 1557 obj->pages[i] = page;
1421 * Pull it into the GTT so that we have a page list (makes the
1422 * initial fault faster and any subsequent flushing possible).
1423 */
1424 if (!obj_priv->agp_mem) {
1425 ret = i915_gem_object_bind_to_gtt(obj, 0);
1426 if (ret) {
1427 drm_gem_object_unreference(obj);
1428 mutex_unlock(&dev->struct_mutex);
1429 return ret;
1430 }
1431 } 1558 }
1432 1559
1433 drm_gem_object_unreference(obj); 1560 if (obj->tiling_mode != I915_TILING_NONE)
1434 mutex_unlock(&dev->struct_mutex); 1561 i915_gem_object_do_bit_17_swizzle(obj);
1435 1562
1436 return 0; 1563 return 0;
1564
1565err_pages:
1566 while (i--)
1567 page_cache_release(obj->pages[i]);
1568
1569 drm_free_large(obj->pages);
1570 obj->pages = NULL;
1571 return PTR_ERR(page);
1437} 1572}
1438 1573
1439void 1574static void
1440i915_gem_object_put_pages(struct drm_gem_object *obj) 1575i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1441{ 1576{
1442 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1577 int page_count = obj->base.size / PAGE_SIZE;
1443 int page_count = obj->size / PAGE_SIZE;
1444 int i; 1578 int i;
1445 1579
1446 BUG_ON(obj_priv->pages_refcount == 0); 1580 BUG_ON(obj->madv == __I915_MADV_PURGED);
1447 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1448 1581
1449 if (--obj_priv->pages_refcount != 0) 1582 if (obj->tiling_mode != I915_TILING_NONE)
1450 return;
1451
1452 if (obj_priv->tiling_mode != I915_TILING_NONE)
1453 i915_gem_object_save_bit_17_swizzle(obj); 1583 i915_gem_object_save_bit_17_swizzle(obj);
1454 1584
1455 if (obj_priv->madv == I915_MADV_DONTNEED) 1585 if (obj->madv == I915_MADV_DONTNEED)
1456 obj_priv->dirty = 0; 1586 obj->dirty = 0;
1457 1587
1458 for (i = 0; i < page_count; i++) { 1588 for (i = 0; i < page_count; i++) {
1459 if (obj_priv->dirty) 1589 if (obj->dirty)
1460 set_page_dirty(obj_priv->pages[i]); 1590 set_page_dirty(obj->pages[i]);
1461 1591
1462 if (obj_priv->madv == I915_MADV_WILLNEED) 1592 if (obj->madv == I915_MADV_WILLNEED)
1463 mark_page_accessed(obj_priv->pages[i]); 1593 mark_page_accessed(obj->pages[i]);
1464 1594
1465 page_cache_release(obj_priv->pages[i]); 1595 page_cache_release(obj->pages[i]);
1466 } 1596 }
1467 obj_priv->dirty = 0; 1597 obj->dirty = 0;
1468 1598
1469 drm_free_large(obj_priv->pages); 1599 drm_free_large(obj->pages);
1470 obj_priv->pages = NULL; 1600 obj->pages = NULL;
1471} 1601}
1472 1602
1473static void 1603void
1474i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno, 1604i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1475 struct intel_ring_buffer *ring) 1605 struct intel_ring_buffer *ring,
1606 u32 seqno)
1476{ 1607{
1477 struct drm_device *dev = obj->dev; 1608 struct drm_device *dev = obj->base.dev;
1478 drm_i915_private_t *dev_priv = dev->dev_private; 1609 struct drm_i915_private *dev_priv = dev->dev_private;
1479 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1610
1480 BUG_ON(ring == NULL); 1611 BUG_ON(ring == NULL);
1481 obj_priv->ring = ring; 1612 obj->ring = ring;
1482 1613
1483 /* Add a reference if we're newly entering the active list. */ 1614 /* Add a reference if we're newly entering the active list. */
1484 if (!obj_priv->active) { 1615 if (!obj->active) {
1485 drm_gem_object_reference(obj); 1616 drm_gem_object_reference(&obj->base);
1486 obj_priv->active = 1; 1617 obj->active = 1;
1487 } 1618 }
1619
1488 /* Move from whatever list we were on to the tail of execution. */ 1620 /* Move from whatever list we were on to the tail of execution. */
1489 spin_lock(&dev_priv->mm.active_list_lock); 1621 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1490 list_move_tail(&obj_priv->list, &ring->active_list); 1622 list_move_tail(&obj->ring_list, &ring->active_list);
1491 spin_unlock(&dev_priv->mm.active_list_lock); 1623
1492 obj_priv->last_rendering_seqno = seqno; 1624 obj->last_rendering_seqno = seqno;
1625 if (obj->fenced_gpu_access) {
1626 struct drm_i915_fence_reg *reg;
1627
1628 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1629
1630 obj->last_fenced_seqno = seqno;
1631 obj->last_fenced_ring = ring;
1632
1633 reg = &dev_priv->fence_regs[obj->fence_reg];
1634 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1635 }
1493} 1636}
1494 1637
1495static void 1638static void
1496i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 1639i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1497{ 1640{
1498 struct drm_device *dev = obj->dev; 1641 list_del_init(&obj->ring_list);
1642 obj->last_rendering_seqno = 0;
1643}
1644
1645static void
1646i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1647{
1648 struct drm_device *dev = obj->base.dev;
1499 drm_i915_private_t *dev_priv = dev->dev_private; 1649 drm_i915_private_t *dev_priv = dev->dev_private;
1500 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1501 1650
1502 BUG_ON(!obj_priv->active); 1651 BUG_ON(!obj->active);
1503 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list); 1652 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1504 obj_priv->last_rendering_seqno = 0; 1653
1654 i915_gem_object_move_off_active(obj);
1655}
1656
1657static void
1658i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1659{
1660 struct drm_device *dev = obj->base.dev;
1661 struct drm_i915_private *dev_priv = dev->dev_private;
1662
1663 if (obj->pin_count != 0)
1664 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1665 else
1666 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1667
1668 BUG_ON(!list_empty(&obj->gpu_write_list));
1669 BUG_ON(!obj->active);
1670 obj->ring = NULL;
1671
1672 i915_gem_object_move_off_active(obj);
1673 obj->fenced_gpu_access = false;
1674
1675 obj->active = 0;
1676 obj->pending_gpu_write = false;
1677 drm_gem_object_unreference(&obj->base);
1678
1679 WARN_ON(i915_verify_lists(dev));
1505} 1680}
1506 1681
1507/* Immediately discard the backing storage */ 1682/* Immediately discard the backing storage */
1508static void 1683static void
1509i915_gem_object_truncate(struct drm_gem_object *obj) 1684i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1510{ 1685{
1511 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1512 struct inode *inode; 1686 struct inode *inode;
1513 1687
1514 /* Our goal here is to return as much of the memory as 1688 /* Our goal here is to return as much of the memory as
1515 * is possible back to the system as we are called from OOM. 1689 * is possible back to the system as we are called from OOM.
1516 * To do this we must instruct the shmfs to drop all of its 1690 * To do this we must instruct the shmfs to drop all of its
1517 * backing pages, *now*. Here we mirror the actions taken 1691 * backing pages, *now*.
1518 * when by shmem_delete_inode() to release the backing store.
1519 */ 1692 */
1520 inode = obj->filp->f_path.dentry->d_inode; 1693 inode = obj->base.filp->f_path.dentry->d_inode;
1521 truncate_inode_pages(inode->i_mapping, 0); 1694 shmem_truncate_range(inode, 0, (loff_t)-1);
1522 if (inode->i_op->truncate_range)
1523 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1524 1695
1525 obj_priv->madv = __I915_MADV_PURGED; 1696 obj->madv = __I915_MADV_PURGED;
1526} 1697}
1527 1698
1528static inline int 1699static inline int
1529i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) 1700i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1530{
1531 return obj_priv->madv == I915_MADV_DONTNEED;
1532}
1533
1534static void
1535i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1536{ 1701{
1537 struct drm_device *dev = obj->dev; 1702 return obj->madv == I915_MADV_DONTNEED;
1538 drm_i915_private_t *dev_priv = dev->dev_private;
1539 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1540
1541 i915_verify_inactive(dev, __FILE__, __LINE__);
1542 if (obj_priv->pin_count != 0)
1543 list_del_init(&obj_priv->list);
1544 else
1545 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1546
1547 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1548
1549 obj_priv->last_rendering_seqno = 0;
1550 obj_priv->ring = NULL;
1551 if (obj_priv->active) {
1552 obj_priv->active = 0;
1553 drm_gem_object_unreference(obj);
1554 }
1555 i915_verify_inactive(dev, __FILE__, __LINE__);
1556} 1703}
1557 1704
1558static void 1705static void
1559i915_gem_process_flushing_list(struct drm_device *dev, 1706i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1560 uint32_t flush_domains, uint32_t seqno, 1707 uint32_t flush_domains)
1561 struct intel_ring_buffer *ring)
1562{ 1708{
1563 drm_i915_private_t *dev_priv = dev->dev_private; 1709 struct drm_i915_gem_object *obj, *next;
1564 struct drm_i915_gem_object *obj_priv, *next;
1565 1710
1566 list_for_each_entry_safe(obj_priv, next, 1711 list_for_each_entry_safe(obj, next,
1567 &dev_priv->mm.gpu_write_list, 1712 &ring->gpu_write_list,
1568 gpu_write_list) { 1713 gpu_write_list) {
1569 struct drm_gem_object *obj = &obj_priv->base; 1714 if (obj->base.write_domain & flush_domains) {
1570 1715 uint32_t old_write_domain = obj->base.write_domain;
1571 if ((obj->write_domain & flush_domains) == 1716
1572 obj->write_domain && 1717 obj->base.write_domain = 0;
1573 obj_priv->ring->ring_flag == ring->ring_flag) { 1718 list_del_init(&obj->gpu_write_list);
1574 uint32_t old_write_domain = obj->write_domain; 1719 i915_gem_object_move_to_active(obj, ring,
1575 1720 i915_gem_next_request_seqno(ring));
1576 obj->write_domain = 0;
1577 list_del_init(&obj_priv->gpu_write_list);
1578 i915_gem_object_move_to_active(obj, seqno, ring);
1579
1580 /* update the fence lru list */
1581 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1582 struct drm_i915_fence_reg *reg =
1583 &dev_priv->fence_regs[obj_priv->fence_reg];
1584 list_move_tail(&reg->lru_list,
1585 &dev_priv->mm.fence_list);
1586 }
1587 1721
1588 trace_i915_gem_object_change_domain(obj, 1722 trace_i915_gem_object_change_domain(obj,
1589 obj->read_domains, 1723 obj->base.read_domains,
1590 old_write_domain); 1724 old_write_domain);
1591 } 1725 }
1592 } 1726 }
1593} 1727}
1594 1728
1595uint32_t 1729int
1596i915_add_request(struct drm_device *dev, struct drm_file *file_priv, 1730i915_add_request(struct intel_ring_buffer *ring,
1597 uint32_t flush_domains, struct intel_ring_buffer *ring) 1731 struct drm_file *file,
1732 struct drm_i915_gem_request *request)
1598{ 1733{
1599 drm_i915_private_t *dev_priv = dev->dev_private; 1734 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1600 struct drm_i915_file_private *i915_file_priv = NULL;
1601 struct drm_i915_gem_request *request;
1602 uint32_t seqno; 1735 uint32_t seqno;
1603 int was_empty; 1736 int was_empty;
1737 int ret;
1604 1738
1605 if (file_priv != NULL) 1739 BUG_ON(request == NULL);
1606 i915_file_priv = file_priv->driver_priv;
1607 1740
1608 request = kzalloc(sizeof(*request), GFP_KERNEL); 1741 ret = ring->add_request(ring, &seqno);
1609 if (request == NULL) 1742 if (ret)
1610 return 0; 1743 return ret;
1611 1744
1612 seqno = ring->add_request(dev, ring, file_priv, flush_domains); 1745 trace_i915_gem_request_add(ring, seqno);
1613 1746
1614 request->seqno = seqno; 1747 request->seqno = seqno;
1615 request->ring = ring; 1748 request->ring = ring;
@@ -1617,260 +1750,353 @@ i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1617 was_empty = list_empty(&ring->request_list); 1750 was_empty = list_empty(&ring->request_list);
1618 list_add_tail(&request->list, &ring->request_list); 1751 list_add_tail(&request->list, &ring->request_list);
1619 1752
1620 if (i915_file_priv) { 1753 if (file) {
1754 struct drm_i915_file_private *file_priv = file->driver_priv;
1755
1756 spin_lock(&file_priv->mm.lock);
1757 request->file_priv = file_priv;
1621 list_add_tail(&request->client_list, 1758 list_add_tail(&request->client_list,
1622 &i915_file_priv->mm.request_list); 1759 &file_priv->mm.request_list);
1623 } else { 1760 spin_unlock(&file_priv->mm.lock);
1624 INIT_LIST_HEAD(&request->client_list);
1625 } 1761 }
1626 1762
1627 /* Associate any objects on the flushing list matching the write 1763 ring->outstanding_lazy_request = false;
1628 * domain we're flushing with our flush.
1629 */
1630 if (flush_domains != 0)
1631 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1632 1764
1633 if (!dev_priv->mm.suspended) { 1765 if (!dev_priv->mm.suspended) {
1634 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD); 1766 mod_timer(&dev_priv->hangcheck_timer,
1767 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1635 if (was_empty) 1768 if (was_empty)
1636 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 1769 queue_delayed_work(dev_priv->wq,
1770 &dev_priv->mm.retire_work, HZ);
1637 } 1771 }
1638 return seqno; 1772 return 0;
1639} 1773}
1640 1774
1641/** 1775static inline void
1642 * Command execution barrier 1776i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1643 *
1644 * Ensures that all commands in the ring are finished
1645 * before signalling the CPU
1646 */
1647static uint32_t
1648i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1649{ 1777{
1650 uint32_t flush_domains = 0; 1778 struct drm_i915_file_private *file_priv = request->file_priv;
1651 1779
1652 /* The sampler always gets flushed on i965 (sigh) */ 1780 if (!file_priv)
1653 if (IS_I965G(dev)) 1781 return;
1654 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1655 1782
1656 ring->flush(dev, ring, 1783 spin_lock(&file_priv->mm.lock);
1657 I915_GEM_DOMAIN_COMMAND, flush_domains); 1784 if (request->file_priv) {
1658 return flush_domains; 1785 list_del(&request->client_list);
1786 request->file_priv = NULL;
1787 }
1788 spin_unlock(&file_priv->mm.lock);
1659} 1789}
1660 1790
1661/** 1791static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1662 * Moves buffers associated only with the given active seqno from the active 1792 struct intel_ring_buffer *ring)
1663 * to inactive list, potentially freeing them.
1664 */
1665static void
1666i915_gem_retire_request(struct drm_device *dev,
1667 struct drm_i915_gem_request *request)
1668{ 1793{
1669 drm_i915_private_t *dev_priv = dev->dev_private; 1794 while (!list_empty(&ring->request_list)) {
1795 struct drm_i915_gem_request *request;
1670 1796
1671 trace_i915_gem_request_retire(dev, request->seqno); 1797 request = list_first_entry(&ring->request_list,
1798 struct drm_i915_gem_request,
1799 list);
1672 1800
1673 /* Move any buffers on the active list that are no longer referenced 1801 list_del(&request->list);
1674 * by the ringbuffer to the flushing/inactive lists as appropriate. 1802 i915_gem_request_remove_from_client(request);
1675 */ 1803 kfree(request);
1676 spin_lock(&dev_priv->mm.active_list_lock); 1804 }
1677 while (!list_empty(&request->ring->active_list)) {
1678 struct drm_gem_object *obj;
1679 struct drm_i915_gem_object *obj_priv;
1680
1681 obj_priv = list_first_entry(&request->ring->active_list,
1682 struct drm_i915_gem_object,
1683 list);
1684 obj = &obj_priv->base;
1685
1686 /* If the seqno being retired doesn't match the oldest in the
1687 * list, then the oldest in the list must still be newer than
1688 * this seqno.
1689 */
1690 if (obj_priv->last_rendering_seqno != request->seqno)
1691 goto out;
1692 1805
1693#if WATCH_LRU 1806 while (!list_empty(&ring->active_list)) {
1694 DRM_INFO("%s: retire %d moves to inactive list %p\n", 1807 struct drm_i915_gem_object *obj;
1695 __func__, request->seqno, obj);
1696#endif
1697 1808
1698 if (obj->write_domain != 0) 1809 obj = list_first_entry(&ring->active_list,
1699 i915_gem_object_move_to_flushing(obj); 1810 struct drm_i915_gem_object,
1700 else { 1811 ring_list);
1701 /* Take a reference on the object so it won't be 1812
1702 * freed while the spinlock is held. The list 1813 obj->base.write_domain = 0;
1703 * protection for this spinlock is safe when breaking 1814 list_del_init(&obj->gpu_write_list);
1704 * the lock like this since the next thing we do 1815 i915_gem_object_move_to_inactive(obj);
1705 * is just get the head of the list again.
1706 */
1707 drm_gem_object_reference(obj);
1708 i915_gem_object_move_to_inactive(obj);
1709 spin_unlock(&dev_priv->mm.active_list_lock);
1710 drm_gem_object_unreference(obj);
1711 spin_lock(&dev_priv->mm.active_list_lock);
1712 }
1713 } 1816 }
1714out:
1715 spin_unlock(&dev_priv->mm.active_list_lock);
1716} 1817}
1717 1818
1718/** 1819static void i915_gem_reset_fences(struct drm_device *dev)
1719 * Returns true if seq1 is later than seq2.
1720 */
1721bool
1722i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1723{ 1820{
1724 return (int32_t)(seq1 - seq2) >= 0; 1821 struct drm_i915_private *dev_priv = dev->dev_private;
1822 int i;
1823
1824 for (i = 0; i < 16; i++) {
1825 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1826 struct drm_i915_gem_object *obj = reg->obj;
1827
1828 if (!obj)
1829 continue;
1830
1831 if (obj->tiling_mode)
1832 i915_gem_release_mmap(obj);
1833
1834 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1835 reg->obj->fenced_gpu_access = false;
1836 reg->obj->last_fenced_seqno = 0;
1837 reg->obj->last_fenced_ring = NULL;
1838 i915_gem_clear_fence_reg(dev, reg);
1839 }
1725} 1840}
1726 1841
1727uint32_t 1842void i915_gem_reset(struct drm_device *dev)
1728i915_get_gem_seqno(struct drm_device *dev,
1729 struct intel_ring_buffer *ring)
1730{ 1843{
1731 return ring->get_gem_seqno(dev, ring); 1844 struct drm_i915_private *dev_priv = dev->dev_private;
1845 struct drm_i915_gem_object *obj;
1846 int i;
1847
1848 for (i = 0; i < I915_NUM_RINGS; i++)
1849 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1850
1851 /* Remove anything from the flushing lists. The GPU cache is likely
1852 * to be lost on reset along with the data, so simply move the
1853 * lost bo to the inactive list.
1854 */
1855 while (!list_empty(&dev_priv->mm.flushing_list)) {
1856 obj= list_first_entry(&dev_priv->mm.flushing_list,
1857 struct drm_i915_gem_object,
1858 mm_list);
1859
1860 obj->base.write_domain = 0;
1861 list_del_init(&obj->gpu_write_list);
1862 i915_gem_object_move_to_inactive(obj);
1863 }
1864
1865 /* Move everything out of the GPU domains to ensure we do any
1866 * necessary invalidation upon reuse.
1867 */
1868 list_for_each_entry(obj,
1869 &dev_priv->mm.inactive_list,
1870 mm_list)
1871 {
1872 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1873 }
1874
1875 /* The fence registers are invalidated so clear them out */
1876 i915_gem_reset_fences(dev);
1732} 1877}
1733 1878
1734/** 1879/**
1735 * This function clears the request list as sequence numbers are passed. 1880 * This function clears the request list as sequence numbers are passed.
1736 */ 1881 */
1737static void 1882static void
1738i915_gem_retire_requests_ring(struct drm_device *dev, 1883i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1739 struct intel_ring_buffer *ring)
1740{ 1884{
1741 drm_i915_private_t *dev_priv = dev->dev_private;
1742 uint32_t seqno; 1885 uint32_t seqno;
1886 int i;
1743 1887
1744 if (!ring->status_page.page_addr 1888 if (list_empty(&ring->request_list))
1745 || list_empty(&ring->request_list))
1746 return; 1889 return;
1747 1890
1748 seqno = i915_get_gem_seqno(dev, ring); 1891 WARN_ON(i915_verify_lists(ring->dev));
1892
1893 seqno = ring->get_seqno(ring);
1894
1895 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1896 if (seqno >= ring->sync_seqno[i])
1897 ring->sync_seqno[i] = 0;
1749 1898
1750 while (!list_empty(&ring->request_list)) { 1899 while (!list_empty(&ring->request_list)) {
1751 struct drm_i915_gem_request *request; 1900 struct drm_i915_gem_request *request;
1752 uint32_t retiring_seqno;
1753 1901
1754 request = list_first_entry(&ring->request_list, 1902 request = list_first_entry(&ring->request_list,
1755 struct drm_i915_gem_request, 1903 struct drm_i915_gem_request,
1756 list); 1904 list);
1757 retiring_seqno = request->seqno;
1758 1905
1759 if (i915_seqno_passed(seqno, retiring_seqno) || 1906 if (!i915_seqno_passed(seqno, request->seqno))
1760 atomic_read(&dev_priv->mm.wedged)) {
1761 i915_gem_retire_request(dev, request);
1762
1763 list_del(&request->list);
1764 list_del(&request->client_list);
1765 kfree(request);
1766 } else
1767 break; 1907 break;
1908
1909 trace_i915_gem_request_retire(ring, request->seqno);
1910
1911 list_del(&request->list);
1912 i915_gem_request_remove_from_client(request);
1913 kfree(request);
1768 } 1914 }
1769 1915
1770 if (unlikely (dev_priv->trace_irq_seqno && 1916 /* Move any buffers on the active list that are no longer referenced
1771 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { 1917 * by the ringbuffer to the flushing/inactive lists as appropriate.
1918 */
1919 while (!list_empty(&ring->active_list)) {
1920 struct drm_i915_gem_object *obj;
1921
1922 obj= list_first_entry(&ring->active_list,
1923 struct drm_i915_gem_object,
1924 ring_list);
1925
1926 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1927 break;
1928
1929 if (obj->base.write_domain != 0)
1930 i915_gem_object_move_to_flushing(obj);
1931 else
1932 i915_gem_object_move_to_inactive(obj);
1933 }
1772 1934
1773 ring->user_irq_put(dev, ring); 1935 if (unlikely(ring->trace_irq_seqno &&
1774 dev_priv->trace_irq_seqno = 0; 1936 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1937 ring->irq_put(ring);
1938 ring->trace_irq_seqno = 0;
1775 } 1939 }
1940
1941 WARN_ON(i915_verify_lists(ring->dev));
1776} 1942}
1777 1943
1778void 1944void
1779i915_gem_retire_requests(struct drm_device *dev) 1945i915_gem_retire_requests(struct drm_device *dev)
1780{ 1946{
1781 drm_i915_private_t *dev_priv = dev->dev_private; 1947 drm_i915_private_t *dev_priv = dev->dev_private;
1948 int i;
1782 1949
1783 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 1950 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1784 struct drm_i915_gem_object *obj_priv, *tmp; 1951 struct drm_i915_gem_object *obj, *next;
1785 1952
1786 /* We must be careful that during unbind() we do not 1953 /* We must be careful that during unbind() we do not
1787 * accidentally infinitely recurse into retire requests. 1954 * accidentally infinitely recurse into retire requests.
1788 * Currently: 1955 * Currently:
1789 * retire -> free -> unbind -> wait -> retire_ring 1956 * retire -> free -> unbind -> wait -> retire_ring
1790 */ 1957 */
1791 list_for_each_entry_safe(obj_priv, tmp, 1958 list_for_each_entry_safe(obj, next,
1792 &dev_priv->mm.deferred_free_list, 1959 &dev_priv->mm.deferred_free_list,
1793 list) 1960 mm_list)
1794 i915_gem_free_object_tail(&obj_priv->base); 1961 i915_gem_free_object_tail(obj);
1795 } 1962 }
1796 1963
1797 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); 1964 for (i = 0; i < I915_NUM_RINGS; i++)
1798 if (HAS_BSD(dev)) 1965 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1799 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1800} 1966}
1801 1967
1802void 1968static void
1803i915_gem_retire_work_handler(struct work_struct *work) 1969i915_gem_retire_work_handler(struct work_struct *work)
1804{ 1970{
1805 drm_i915_private_t *dev_priv; 1971 drm_i915_private_t *dev_priv;
1806 struct drm_device *dev; 1972 struct drm_device *dev;
1973 bool idle;
1974 int i;
1807 1975
1808 dev_priv = container_of(work, drm_i915_private_t, 1976 dev_priv = container_of(work, drm_i915_private_t,
1809 mm.retire_work.work); 1977 mm.retire_work.work);
1810 dev = dev_priv->dev; 1978 dev = dev_priv->dev;
1811 1979
1812 mutex_lock(&dev->struct_mutex); 1980 /* Come back later if the device is busy... */
1981 if (!mutex_trylock(&dev->struct_mutex)) {
1982 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1983 return;
1984 }
1985
1813 i915_gem_retire_requests(dev); 1986 i915_gem_retire_requests(dev);
1814 1987
1815 if (!dev_priv->mm.suspended && 1988 /* Send a periodic flush down the ring so we don't hold onto GEM
1816 (!list_empty(&dev_priv->render_ring.request_list) || 1989 * objects indefinitely.
1817 (HAS_BSD(dev) && 1990 */
1818 !list_empty(&dev_priv->bsd_ring.request_list)))) 1991 idle = true;
1992 for (i = 0; i < I915_NUM_RINGS; i++) {
1993 struct intel_ring_buffer *ring = &dev_priv->ring[i];
1994
1995 if (!list_empty(&ring->gpu_write_list)) {
1996 struct drm_i915_gem_request *request;
1997 int ret;
1998
1999 ret = i915_gem_flush_ring(ring,
2000 0, I915_GEM_GPU_DOMAINS);
2001 request = kzalloc(sizeof(*request), GFP_KERNEL);
2002 if (ret || request == NULL ||
2003 i915_add_request(ring, NULL, request))
2004 kfree(request);
2005 }
2006
2007 idle &= list_empty(&ring->request_list);
2008 }
2009
2010 if (!dev_priv->mm.suspended && !idle)
1819 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 2011 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2012
1820 mutex_unlock(&dev->struct_mutex); 2013 mutex_unlock(&dev->struct_mutex);
1821} 2014}
1822 2015
2016/**
2017 * Waits for a sequence number to be signaled, and cleans up the
2018 * request and object lists appropriately for that event.
2019 */
1823int 2020int
1824i915_do_wait_request(struct drm_device *dev, uint32_t seqno, 2021i915_wait_request(struct intel_ring_buffer *ring,
1825 int interruptible, struct intel_ring_buffer *ring) 2022 uint32_t seqno)
1826{ 2023{
1827 drm_i915_private_t *dev_priv = dev->dev_private; 2024 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1828 u32 ier; 2025 u32 ier;
1829 int ret = 0; 2026 int ret = 0;
1830 2027
1831 BUG_ON(seqno == 0); 2028 BUG_ON(seqno == 0);
1832 2029
1833 if (atomic_read(&dev_priv->mm.wedged)) 2030 if (atomic_read(&dev_priv->mm.wedged)) {
1834 return -EIO; 2031 struct completion *x = &dev_priv->error_completion;
2032 bool recovery_complete;
2033 unsigned long flags;
2034
2035 /* Give the error handler a chance to run. */
2036 spin_lock_irqsave(&x->wait.lock, flags);
2037 recovery_complete = x->done > 0;
2038 spin_unlock_irqrestore(&x->wait.lock, flags);
2039
2040 return recovery_complete ? -EIO : -EAGAIN;
2041 }
1835 2042
1836 if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) { 2043 if (seqno == ring->outstanding_lazy_request) {
1837 if (HAS_PCH_SPLIT(dev)) 2044 struct drm_i915_gem_request *request;
2045
2046 request = kzalloc(sizeof(*request), GFP_KERNEL);
2047 if (request == NULL)
2048 return -ENOMEM;
2049
2050 ret = i915_add_request(ring, NULL, request);
2051 if (ret) {
2052 kfree(request);
2053 return ret;
2054 }
2055
2056 seqno = request->seqno;
2057 }
2058
2059 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2060 if (HAS_PCH_SPLIT(ring->dev))
1838 ier = I915_READ(DEIER) | I915_READ(GTIER); 2061 ier = I915_READ(DEIER) | I915_READ(GTIER);
1839 else 2062 else
1840 ier = I915_READ(IER); 2063 ier = I915_READ(IER);
1841 if (!ier) { 2064 if (!ier) {
1842 DRM_ERROR("something (likely vbetool) disabled " 2065 DRM_ERROR("something (likely vbetool) disabled "
1843 "interrupts, re-enabling\n"); 2066 "interrupts, re-enabling\n");
1844 i915_driver_irq_preinstall(dev); 2067 ring->dev->driver->irq_preinstall(ring->dev);
1845 i915_driver_irq_postinstall(dev); 2068 ring->dev->driver->irq_postinstall(ring->dev);
1846 } 2069 }
1847 2070
1848 trace_i915_gem_request_wait_begin(dev, seqno); 2071 trace_i915_gem_request_wait_begin(ring, seqno);
1849 2072
1850 ring->waiting_gem_seqno = seqno; 2073 ring->waiting_seqno = seqno;
1851 ring->user_irq_get(dev, ring); 2074 if (ring->irq_get(ring)) {
1852 if (interruptible) 2075 if (dev_priv->mm.interruptible)
1853 ret = wait_event_interruptible(ring->irq_queue, 2076 ret = wait_event_interruptible(ring->irq_queue,
1854 i915_seqno_passed( 2077 i915_seqno_passed(ring->get_seqno(ring), seqno)
1855 ring->get_gem_seqno(dev, ring), seqno) 2078 || atomic_read(&dev_priv->mm.wedged));
1856 || atomic_read(&dev_priv->mm.wedged)); 2079 else
1857 else 2080 wait_event(ring->irq_queue,
1858 wait_event(ring->irq_queue, 2081 i915_seqno_passed(ring->get_seqno(ring), seqno)
1859 i915_seqno_passed( 2082 || atomic_read(&dev_priv->mm.wedged));
1860 ring->get_gem_seqno(dev, ring), seqno)
1861 || atomic_read(&dev_priv->mm.wedged));
1862 2083
1863 ring->user_irq_put(dev, ring); 2084 ring->irq_put(ring);
1864 ring->waiting_gem_seqno = 0; 2085 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2086 seqno) ||
2087 atomic_read(&dev_priv->mm.wedged), 3000))
2088 ret = -EBUSY;
2089 ring->waiting_seqno = 0;
1865 2090
1866 trace_i915_gem_request_wait_end(dev, seqno); 2091 trace_i915_gem_request_wait_end(ring, seqno);
1867 } 2092 }
1868 if (atomic_read(&dev_priv->mm.wedged)) 2093 if (atomic_read(&dev_priv->mm.wedged))
1869 ret = -EIO; 2094 ret = -EAGAIN;
1870 2095
1871 if (ret && ret != -ERESTARTSYS) 2096 if (ret && ret != -ERESTARTSYS)
1872 DRM_ERROR("%s returns %d (awaiting %d at %d)\n", 2097 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
1873 __func__, ret, seqno, ring->get_gem_seqno(dev, ring)); 2098 __func__, ret, seqno, ring->get_seqno(ring),
2099 dev_priv->next_seqno);
1874 2100
1875 /* Directly dispatch request retiring. While we have the work queue 2101 /* Directly dispatch request retiring. While we have the work queue
1876 * to handle this, the waiter on a request often wants an associated 2102 * to handle this, the waiter on a request often wants an associated
@@ -1878,67 +2104,31 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1878 * a separate wait queue to handle that. 2104 * a separate wait queue to handle that.
1879 */ 2105 */
1880 if (ret == 0) 2106 if (ret == 0)
1881 i915_gem_retire_requests_ring(dev, ring); 2107 i915_gem_retire_requests_ring(ring);
1882 2108
1883 return ret; 2109 return ret;
1884} 2110}
1885 2111
1886/** 2112/**
1887 * Waits for a sequence number to be signaled, and cleans up the
1888 * request and object lists appropriately for that event.
1889 */
1890static int
1891i915_wait_request(struct drm_device *dev, uint32_t seqno,
1892 struct intel_ring_buffer *ring)
1893{
1894 return i915_do_wait_request(dev, seqno, 1, ring);
1895}
1896
1897static void
1898i915_gem_flush(struct drm_device *dev,
1899 uint32_t invalidate_domains,
1900 uint32_t flush_domains)
1901{
1902 drm_i915_private_t *dev_priv = dev->dev_private;
1903 if (flush_domains & I915_GEM_DOMAIN_CPU)
1904 drm_agp_chipset_flush(dev);
1905 dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1906 invalidate_domains,
1907 flush_domains);
1908
1909 if (HAS_BSD(dev))
1910 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1911 invalidate_domains,
1912 flush_domains);
1913}
1914
1915/**
1916 * Ensures that all rendering to the object has completed and the object is 2113 * Ensures that all rendering to the object has completed and the object is
1917 * safe to unbind from the GTT or access from the CPU. 2114 * safe to unbind from the GTT or access from the CPU.
1918 */ 2115 */
1919static int 2116int
1920i915_gem_object_wait_rendering(struct drm_gem_object *obj) 2117i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
1921{ 2118{
1922 struct drm_device *dev = obj->dev;
1923 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1924 int ret; 2119 int ret;
1925 2120
1926 /* This function only exists to support waiting for existing rendering, 2121 /* This function only exists to support waiting for existing rendering,
1927 * not for emitting required flushes. 2122 * not for emitting required flushes.
1928 */ 2123 */
1929 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); 2124 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
1930 2125
1931 /* If there is rendering queued on the buffer being evicted, wait for 2126 /* If there is rendering queued on the buffer being evicted, wait for
1932 * it. 2127 * it.
1933 */ 2128 */
1934 if (obj_priv->active) { 2129 if (obj->active) {
1935#if WATCH_BUF 2130 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
1936 DRM_INFO("%s: object %p wait for seqno %08x\n", 2131 if (ret)
1937 __func__, obj, obj_priv->last_rendering_seqno);
1938#endif
1939 ret = i915_wait_request(dev,
1940 obj_priv->last_rendering_seqno, obj_priv->ring);
1941 if (ret != 0)
1942 return ret; 2132 return ret;
1943 } 2133 }
1944 2134
@@ -1949,21 +2139,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1949 * Unbinds an object from the GTT aperture. 2139 * Unbinds an object from the GTT aperture.
1950 */ 2140 */
1951int 2141int
1952i915_gem_object_unbind(struct drm_gem_object *obj) 2142i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1953{ 2143{
1954 struct drm_device *dev = obj->dev;
1955 drm_i915_private_t *dev_priv = dev->dev_private;
1956 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1957 int ret = 0; 2144 int ret = 0;
1958 2145
1959#if WATCH_BUF 2146 if (obj->gtt_space == NULL)
1960 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1961 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1962#endif
1963 if (obj_priv->gtt_space == NULL)
1964 return 0; 2147 return 0;
1965 2148
1966 if (obj_priv->pin_count != 0) { 2149 if (obj->pin_count != 0) {
1967 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2150 DRM_ERROR("Attempting to unbind pinned buffer\n");
1968 return -EINVAL; 2151 return -EINVAL;
1969 } 2152 }
@@ -1984,319 +2167,383 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
1984 * should be safe and we need to cleanup or else we might 2167 * should be safe and we need to cleanup or else we might
1985 * cause memory corruption through use-after-free. 2168 * cause memory corruption through use-after-free.
1986 */ 2169 */
2170 if (ret) {
2171 i915_gem_clflush_object(obj);
2172 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2173 }
1987 2174
1988 /* release the fence reg _after_ flushing */ 2175 /* release the fence reg _after_ flushing */
1989 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 2176 ret = i915_gem_object_put_fence(obj);
1990 i915_gem_clear_fence_reg(obj); 2177 if (ret == -ERESTARTSYS)
2178 return ret;
1991 2179
1992 if (obj_priv->agp_mem != NULL) { 2180 trace_i915_gem_object_unbind(obj);
1993 drm_unbind_agp(obj_priv->agp_mem);
1994 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1995 obj_priv->agp_mem = NULL;
1996 }
1997 2181
1998 i915_gem_object_put_pages(obj); 2182 i915_gem_gtt_unbind_object(obj);
1999 BUG_ON(obj_priv->pages_refcount); 2183 i915_gem_object_put_pages_gtt(obj);
2000 2184
2001 if (obj_priv->gtt_space) { 2185 list_del_init(&obj->gtt_list);
2002 atomic_dec(&dev->gtt_count); 2186 list_del_init(&obj->mm_list);
2003 atomic_sub(obj->size, &dev->gtt_memory); 2187 /* Avoid an unnecessary call to unbind on rebind. */
2188 obj->map_and_fenceable = true;
2004 2189
2005 drm_mm_put_block(obj_priv->gtt_space); 2190 drm_mm_put_block(obj->gtt_space);
2006 obj_priv->gtt_space = NULL; 2191 obj->gtt_space = NULL;
2007 } 2192 obj->gtt_offset = 0;
2008 2193
2009 /* Remove ourselves from the LRU list if present. */ 2194 if (i915_gem_object_is_purgeable(obj))
2010 spin_lock(&dev_priv->mm.active_list_lock);
2011 if (!list_empty(&obj_priv->list))
2012 list_del_init(&obj_priv->list);
2013 spin_unlock(&dev_priv->mm.active_list_lock);
2014
2015 if (i915_gem_object_is_purgeable(obj_priv))
2016 i915_gem_object_truncate(obj); 2195 i915_gem_object_truncate(obj);
2017 2196
2018 trace_i915_gem_object_unbind(obj);
2019
2020 return ret; 2197 return ret;
2021} 2198}
2022 2199
2023int 2200int
2024i915_gpu_idle(struct drm_device *dev) 2201i915_gem_flush_ring(struct intel_ring_buffer *ring,
2202 uint32_t invalidate_domains,
2203 uint32_t flush_domains)
2025{ 2204{
2026 drm_i915_private_t *dev_priv = dev->dev_private;
2027 bool lists_empty;
2028 uint32_t seqno1, seqno2;
2029 int ret; 2205 int ret;
2030 2206
2031 spin_lock(&dev_priv->mm.active_list_lock); 2207 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2032 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2033 list_empty(&dev_priv->render_ring.active_list) &&
2034 (!HAS_BSD(dev) ||
2035 list_empty(&dev_priv->bsd_ring.active_list)));
2036 spin_unlock(&dev_priv->mm.active_list_lock);
2037
2038 if (lists_empty)
2039 return 0; 2208 return 0;
2040 2209
2041 /* Flush everything onto the inactive list. */ 2210 trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2042 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2043 seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2044 &dev_priv->render_ring);
2045 if (seqno1 == 0)
2046 return -ENOMEM;
2047 ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2048 2211
2049 if (HAS_BSD(dev)) { 2212 ret = ring->flush(ring, invalidate_domains, flush_domains);
2050 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS, 2213 if (ret)
2051 &dev_priv->bsd_ring); 2214 return ret;
2052 if (seqno2 == 0) 2215
2053 return -ENOMEM; 2216 if (flush_domains & I915_GEM_GPU_DOMAINS)
2217 i915_gem_process_flushing_list(ring, flush_domains);
2218
2219 return 0;
2220}
2221
2222static int i915_ring_idle(struct intel_ring_buffer *ring)
2223{
2224 int ret;
2225
2226 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2227 return 0;
2054 2228
2055 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring); 2229 if (!list_empty(&ring->gpu_write_list)) {
2230 ret = i915_gem_flush_ring(ring,
2231 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2056 if (ret) 2232 if (ret)
2057 return ret; 2233 return ret;
2058 } 2234 }
2059 2235
2060 2236 return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2061 return ret;
2062} 2237}
2063 2238
2064int 2239int
2065i915_gem_object_get_pages(struct drm_gem_object *obj, 2240i915_gpu_idle(struct drm_device *dev)
2066 gfp_t gfpmask)
2067{ 2241{
2068 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2242 drm_i915_private_t *dev_priv = dev->dev_private;
2069 int page_count, i; 2243 bool lists_empty;
2070 struct address_space *mapping; 2244 int ret, i;
2071 struct inode *inode;
2072 struct page *page;
2073
2074 BUG_ON(obj_priv->pages_refcount
2075 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2076 2245
2077 if (obj_priv->pages_refcount++ != 0) 2246 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2247 list_empty(&dev_priv->mm.active_list));
2248 if (lists_empty)
2078 return 0; 2249 return 0;
2079 2250
2080 /* Get the list of pages out of our struct file. They'll be pinned 2251 /* Flush everything onto the inactive list. */
2081 * at this point until we release them. 2252 for (i = 0; i < I915_NUM_RINGS; i++) {
2082 */ 2253 ret = i915_ring_idle(&dev_priv->ring[i]);
2083 page_count = obj->size / PAGE_SIZE; 2254 if (ret)
2084 BUG_ON(obj_priv->pages != NULL); 2255 return ret;
2085 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2086 if (obj_priv->pages == NULL) {
2087 obj_priv->pages_refcount--;
2088 return -ENOMEM;
2089 }
2090
2091 inode = obj->filp->f_path.dentry->d_inode;
2092 mapping = inode->i_mapping;
2093 for (i = 0; i < page_count; i++) {
2094 page = read_cache_page_gfp(mapping, i,
2095 GFP_HIGHUSER |
2096 __GFP_COLD |
2097 __GFP_RECLAIMABLE |
2098 gfpmask);
2099 if (IS_ERR(page))
2100 goto err_pages;
2101
2102 obj_priv->pages[i] = page;
2103 } 2256 }
2104 2257
2105 if (obj_priv->tiling_mode != I915_TILING_NONE)
2106 i915_gem_object_do_bit_17_swizzle(obj);
2107
2108 return 0; 2258 return 0;
2109
2110err_pages:
2111 while (i--)
2112 page_cache_release(obj_priv->pages[i]);
2113
2114 drm_free_large(obj_priv->pages);
2115 obj_priv->pages = NULL;
2116 obj_priv->pages_refcount--;
2117 return PTR_ERR(page);
2118} 2259}
2119 2260
2120static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) 2261static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2262 struct intel_ring_buffer *pipelined)
2121{ 2263{
2122 struct drm_gem_object *obj = reg->obj; 2264 struct drm_device *dev = obj->base.dev;
2123 struct drm_device *dev = obj->dev;
2124 drm_i915_private_t *dev_priv = dev->dev_private; 2265 drm_i915_private_t *dev_priv = dev->dev_private;
2125 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2266 u32 size = obj->gtt_space->size;
2126 int regnum = obj_priv->fence_reg; 2267 int regnum = obj->fence_reg;
2127 uint64_t val; 2268 uint64_t val;
2128 2269
2129 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2270 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2130 0xfffff000) << 32; 2271 0xfffff000) << 32;
2131 val |= obj_priv->gtt_offset & 0xfffff000; 2272 val |= obj->gtt_offset & 0xfffff000;
2132 val |= (uint64_t)((obj_priv->stride / 128) - 1) << 2273 val |= (uint64_t)((obj->stride / 128) - 1) <<
2133 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2274 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2134 2275
2135 if (obj_priv->tiling_mode == I915_TILING_Y) 2276 if (obj->tiling_mode == I915_TILING_Y)
2136 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2277 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2137 val |= I965_FENCE_REG_VALID; 2278 val |= I965_FENCE_REG_VALID;
2138 2279
2139 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val); 2280 if (pipelined) {
2281 int ret = intel_ring_begin(pipelined, 6);
2282 if (ret)
2283 return ret;
2284
2285 intel_ring_emit(pipelined, MI_NOOP);
2286 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2287 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2288 intel_ring_emit(pipelined, (u32)val);
2289 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2290 intel_ring_emit(pipelined, (u32)(val >> 32));
2291 intel_ring_advance(pipelined);
2292 } else
2293 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2294
2295 return 0;
2140} 2296}
2141 2297
2142static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) 2298static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2299 struct intel_ring_buffer *pipelined)
2143{ 2300{
2144 struct drm_gem_object *obj = reg->obj; 2301 struct drm_device *dev = obj->base.dev;
2145 struct drm_device *dev = obj->dev;
2146 drm_i915_private_t *dev_priv = dev->dev_private; 2302 drm_i915_private_t *dev_priv = dev->dev_private;
2147 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2303 u32 size = obj->gtt_space->size;
2148 int regnum = obj_priv->fence_reg; 2304 int regnum = obj->fence_reg;
2149 uint64_t val; 2305 uint64_t val;
2150 2306
2151 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2307 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2152 0xfffff000) << 32; 2308 0xfffff000) << 32;
2153 val |= obj_priv->gtt_offset & 0xfffff000; 2309 val |= obj->gtt_offset & 0xfffff000;
2154 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2310 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2155 if (obj_priv->tiling_mode == I915_TILING_Y) 2311 if (obj->tiling_mode == I915_TILING_Y)
2156 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2312 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2157 val |= I965_FENCE_REG_VALID; 2313 val |= I965_FENCE_REG_VALID;
2158 2314
2159 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); 2315 if (pipelined) {
2316 int ret = intel_ring_begin(pipelined, 6);
2317 if (ret)
2318 return ret;
2319
2320 intel_ring_emit(pipelined, MI_NOOP);
2321 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2322 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2323 intel_ring_emit(pipelined, (u32)val);
2324 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2325 intel_ring_emit(pipelined, (u32)(val >> 32));
2326 intel_ring_advance(pipelined);
2327 } else
2328 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2329
2330 return 0;
2160} 2331}
2161 2332
2162static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) 2333static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2334 struct intel_ring_buffer *pipelined)
2163{ 2335{
2164 struct drm_gem_object *obj = reg->obj; 2336 struct drm_device *dev = obj->base.dev;
2165 struct drm_device *dev = obj->dev;
2166 drm_i915_private_t *dev_priv = dev->dev_private; 2337 drm_i915_private_t *dev_priv = dev->dev_private;
2167 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2338 u32 size = obj->gtt_space->size;
2168 int regnum = obj_priv->fence_reg; 2339 u32 fence_reg, val, pitch_val;
2169 int tile_width; 2340 int tile_width;
2170 uint32_t fence_reg, val;
2171 uint32_t pitch_val;
2172 2341
2173 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || 2342 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2174 (obj_priv->gtt_offset & (obj->size - 1))) { 2343 (size & -size) != size ||
2175 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", 2344 (obj->gtt_offset & (size - 1)),
2176 __func__, obj_priv->gtt_offset, obj->size); 2345 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2177 return; 2346 obj->gtt_offset, obj->map_and_fenceable, size))
2178 } 2347 return -EINVAL;
2179 2348
2180 if (obj_priv->tiling_mode == I915_TILING_Y && 2349 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2181 HAS_128_BYTE_Y_TILING(dev))
2182 tile_width = 128; 2350 tile_width = 128;
2183 else 2351 else
2184 tile_width = 512; 2352 tile_width = 512;
2185 2353
2186 /* Note: pitch better be a power of two tile widths */ 2354 /* Note: pitch better be a power of two tile widths */
2187 pitch_val = obj_priv->stride / tile_width; 2355 pitch_val = obj->stride / tile_width;
2188 pitch_val = ffs(pitch_val) - 1; 2356 pitch_val = ffs(pitch_val) - 1;
2189 2357
2190 if (obj_priv->tiling_mode == I915_TILING_Y && 2358 val = obj->gtt_offset;
2191 HAS_128_BYTE_Y_TILING(dev)) 2359 if (obj->tiling_mode == I915_TILING_Y)
2192 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2193 else
2194 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2195
2196 val = obj_priv->gtt_offset;
2197 if (obj_priv->tiling_mode == I915_TILING_Y)
2198 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2360 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2199 val |= I915_FENCE_SIZE_BITS(obj->size); 2361 val |= I915_FENCE_SIZE_BITS(size);
2200 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2362 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2201 val |= I830_FENCE_REG_VALID; 2363 val |= I830_FENCE_REG_VALID;
2202 2364
2203 if (regnum < 8) 2365 fence_reg = obj->fence_reg;
2204 fence_reg = FENCE_REG_830_0 + (regnum * 4); 2366 if (fence_reg < 8)
2367 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2205 else 2368 else
2206 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); 2369 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2207 I915_WRITE(fence_reg, val); 2370
2371 if (pipelined) {
2372 int ret = intel_ring_begin(pipelined, 4);
2373 if (ret)
2374 return ret;
2375
2376 intel_ring_emit(pipelined, MI_NOOP);
2377 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2378 intel_ring_emit(pipelined, fence_reg);
2379 intel_ring_emit(pipelined, val);
2380 intel_ring_advance(pipelined);
2381 } else
2382 I915_WRITE(fence_reg, val);
2383
2384 return 0;
2208} 2385}
2209 2386
2210static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) 2387static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2388 struct intel_ring_buffer *pipelined)
2211{ 2389{
2212 struct drm_gem_object *obj = reg->obj; 2390 struct drm_device *dev = obj->base.dev;
2213 struct drm_device *dev = obj->dev;
2214 drm_i915_private_t *dev_priv = dev->dev_private; 2391 drm_i915_private_t *dev_priv = dev->dev_private;
2215 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2392 u32 size = obj->gtt_space->size;
2216 int regnum = obj_priv->fence_reg; 2393 int regnum = obj->fence_reg;
2217 uint32_t val; 2394 uint32_t val;
2218 uint32_t pitch_val; 2395 uint32_t pitch_val;
2219 uint32_t fence_size_bits;
2220 2396
2221 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || 2397 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2222 (obj_priv->gtt_offset & (obj->size - 1))) { 2398 (size & -size) != size ||
2223 WARN(1, "%s: object 0x%08x not 512K or size aligned\n", 2399 (obj->gtt_offset & (size - 1)),
2224 __func__, obj_priv->gtt_offset); 2400 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2225 return; 2401 obj->gtt_offset, size))
2226 } 2402 return -EINVAL;
2227 2403
2228 pitch_val = obj_priv->stride / 128; 2404 pitch_val = obj->stride / 128;
2229 pitch_val = ffs(pitch_val) - 1; 2405 pitch_val = ffs(pitch_val) - 1;
2230 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2231 2406
2232 val = obj_priv->gtt_offset; 2407 val = obj->gtt_offset;
2233 if (obj_priv->tiling_mode == I915_TILING_Y) 2408 if (obj->tiling_mode == I915_TILING_Y)
2234 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2409 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2235 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); 2410 val |= I830_FENCE_SIZE_BITS(size);
2236 WARN_ON(fence_size_bits & ~0x00000f00);
2237 val |= fence_size_bits;
2238 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2411 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2239 val |= I830_FENCE_REG_VALID; 2412 val |= I830_FENCE_REG_VALID;
2240 2413
2241 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); 2414 if (pipelined) {
2415 int ret = intel_ring_begin(pipelined, 4);
2416 if (ret)
2417 return ret;
2418
2419 intel_ring_emit(pipelined, MI_NOOP);
2420 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2421 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2422 intel_ring_emit(pipelined, val);
2423 intel_ring_advance(pipelined);
2424 } else
2425 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2426
2427 return 0;
2428}
2429
2430static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2431{
2432 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2433}
2434
2435static int
2436i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2437 struct intel_ring_buffer *pipelined)
2438{
2439 int ret;
2440
2441 if (obj->fenced_gpu_access) {
2442 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2443 ret = i915_gem_flush_ring(obj->last_fenced_ring,
2444 0, obj->base.write_domain);
2445 if (ret)
2446 return ret;
2447 }
2448
2449 obj->fenced_gpu_access = false;
2450 }
2451
2452 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2453 if (!ring_passed_seqno(obj->last_fenced_ring,
2454 obj->last_fenced_seqno)) {
2455 ret = i915_wait_request(obj->last_fenced_ring,
2456 obj->last_fenced_seqno);
2457 if (ret)
2458 return ret;
2459 }
2460
2461 obj->last_fenced_seqno = 0;
2462 obj->last_fenced_ring = NULL;
2463 }
2464
2465 /* Ensure that all CPU reads are completed before installing a fence
2466 * and all writes before removing the fence.
2467 */
2468 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2469 mb();
2470
2471 return 0;
2472}
2473
2474int
2475i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2476{
2477 int ret;
2478
2479 if (obj->tiling_mode)
2480 i915_gem_release_mmap(obj);
2481
2482 ret = i915_gem_object_flush_fence(obj, NULL);
2483 if (ret)
2484 return ret;
2485
2486 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2487 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2488 i915_gem_clear_fence_reg(obj->base.dev,
2489 &dev_priv->fence_regs[obj->fence_reg]);
2490
2491 obj->fence_reg = I915_FENCE_REG_NONE;
2492 }
2493
2494 return 0;
2242} 2495}
2243 2496
2244static int i915_find_fence_reg(struct drm_device *dev) 2497static struct drm_i915_fence_reg *
2498i915_find_fence_reg(struct drm_device *dev,
2499 struct intel_ring_buffer *pipelined)
2245{ 2500{
2246 struct drm_i915_fence_reg *reg = NULL;
2247 struct drm_i915_gem_object *obj_priv = NULL;
2248 struct drm_i915_private *dev_priv = dev->dev_private; 2501 struct drm_i915_private *dev_priv = dev->dev_private;
2249 struct drm_gem_object *obj = NULL; 2502 struct drm_i915_fence_reg *reg, *first, *avail;
2250 int i, avail, ret; 2503 int i;
2251 2504
2252 /* First try to find a free reg */ 2505 /* First try to find a free reg */
2253 avail = 0; 2506 avail = NULL;
2254 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2507 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2255 reg = &dev_priv->fence_regs[i]; 2508 reg = &dev_priv->fence_regs[i];
2256 if (!reg->obj) 2509 if (!reg->obj)
2257 return i; 2510 return reg;
2258 2511
2259 obj_priv = to_intel_bo(reg->obj); 2512 if (!reg->obj->pin_count)
2260 if (!obj_priv->pin_count) 2513 avail = reg;
2261 avail++;
2262 } 2514 }
2263 2515
2264 if (avail == 0) 2516 if (avail == NULL)
2265 return -ENOSPC; 2517 return NULL;
2266 2518
2267 /* None available, try to steal one or wait for a user to finish */ 2519 /* None available, try to steal one or wait for a user to finish */
2268 i = I915_FENCE_REG_NONE; 2520 avail = first = NULL;
2269 list_for_each_entry(reg, &dev_priv->mm.fence_list, 2521 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2270 lru_list) { 2522 if (reg->obj->pin_count)
2271 obj = reg->obj;
2272 obj_priv = to_intel_bo(obj);
2273
2274 if (obj_priv->pin_count)
2275 continue; 2523 continue;
2276 2524
2277 /* found one! */ 2525 if (first == NULL)
2278 i = obj_priv->fence_reg; 2526 first = reg;
2279 break;
2280 }
2281 2527
2282 BUG_ON(i == I915_FENCE_REG_NONE); 2528 if (!pipelined ||
2529 !reg->obj->last_fenced_ring ||
2530 reg->obj->last_fenced_ring == pipelined) {
2531 avail = reg;
2532 break;
2533 }
2534 }
2283 2535
2284 /* We only have a reference on obj from the active list. put_fence_reg 2536 if (avail == NULL)
2285 * might drop that one, causing a use-after-free in it. So hold a 2537 avail = first;
2286 * private reference to obj like the other callers of put_fence_reg
2287 * (set_tiling ioctl) do. */
2288 drm_gem_object_reference(obj);
2289 ret = i915_gem_object_put_fence_reg(obj);
2290 drm_gem_object_unreference(obj);
2291 if (ret != 0)
2292 return ret;
2293 2538
2294 return i; 2539 return avail;
2295} 2540}
2296 2541
2297/** 2542/**
2298 * i915_gem_object_get_fence_reg - set up a fence reg for an object 2543 * i915_gem_object_get_fence - set up a fence reg for an object
2299 * @obj: object to map through a fence reg 2544 * @obj: object to map through a fence reg
2545 * @pipelined: ring on which to queue the change, or NULL for CPU access
2546 * @interruptible: must we wait uninterruptibly for the register to retire?
2300 * 2547 *
2301 * When mapping objects through the GTT, userspace wants to be able to write 2548 * When mapping objects through the GTT, userspace wants to be able to write
2302 * to them without having to worry about swizzling if the object is tiled. 2549 * to them without having to worry about swizzling if the object is tiled.
@@ -2308,71 +2555,125 @@ static int i915_find_fence_reg(struct drm_device *dev)
2308 * and tiling format. 2555 * and tiling format.
2309 */ 2556 */
2310int 2557int
2311i915_gem_object_get_fence_reg(struct drm_gem_object *obj) 2558i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2559 struct intel_ring_buffer *pipelined)
2312{ 2560{
2313 struct drm_device *dev = obj->dev; 2561 struct drm_device *dev = obj->base.dev;
2314 struct drm_i915_private *dev_priv = dev->dev_private; 2562 struct drm_i915_private *dev_priv = dev->dev_private;
2315 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2563 struct drm_i915_fence_reg *reg;
2316 struct drm_i915_fence_reg *reg = NULL;
2317 int ret; 2564 int ret;
2318 2565
2319 /* Just update our place in the LRU if our fence is getting used. */ 2566 /* XXX disable pipelining. There are bugs. Shocking. */
2320 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { 2567 pipelined = NULL;
2321 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2568
2569 /* Just update our place in the LRU if our fence is getting reused. */
2570 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2571 reg = &dev_priv->fence_regs[obj->fence_reg];
2322 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2572 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2573
2574 if (obj->tiling_changed) {
2575 ret = i915_gem_object_flush_fence(obj, pipelined);
2576 if (ret)
2577 return ret;
2578
2579 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2580 pipelined = NULL;
2581
2582 if (pipelined) {
2583 reg->setup_seqno =
2584 i915_gem_next_request_seqno(pipelined);
2585 obj->last_fenced_seqno = reg->setup_seqno;
2586 obj->last_fenced_ring = pipelined;
2587 }
2588
2589 goto update;
2590 }
2591
2592 if (!pipelined) {
2593 if (reg->setup_seqno) {
2594 if (!ring_passed_seqno(obj->last_fenced_ring,
2595 reg->setup_seqno)) {
2596 ret = i915_wait_request(obj->last_fenced_ring,
2597 reg->setup_seqno);
2598 if (ret)
2599 return ret;
2600 }
2601
2602 reg->setup_seqno = 0;
2603 }
2604 } else if (obj->last_fenced_ring &&
2605 obj->last_fenced_ring != pipelined) {
2606 ret = i915_gem_object_flush_fence(obj, pipelined);
2607 if (ret)
2608 return ret;
2609 }
2610
2323 return 0; 2611 return 0;
2324 } 2612 }
2325 2613
2326 switch (obj_priv->tiling_mode) { 2614 reg = i915_find_fence_reg(dev, pipelined);
2327 case I915_TILING_NONE: 2615 if (reg == NULL)
2328 WARN(1, "allocating a fence for non-tiled object?\n"); 2616 return -ENOSPC;
2329 break;
2330 case I915_TILING_X:
2331 if (!obj_priv->stride)
2332 return -EINVAL;
2333 WARN((obj_priv->stride & (512 - 1)),
2334 "object 0x%08x is X tiled but has non-512B pitch\n",
2335 obj_priv->gtt_offset);
2336 break;
2337 case I915_TILING_Y:
2338 if (!obj_priv->stride)
2339 return -EINVAL;
2340 WARN((obj_priv->stride & (128 - 1)),
2341 "object 0x%08x is Y tiled but has non-128B pitch\n",
2342 obj_priv->gtt_offset);
2343 break;
2344 }
2345 2617
2346 ret = i915_find_fence_reg(dev); 2618 ret = i915_gem_object_flush_fence(obj, pipelined);
2347 if (ret < 0) 2619 if (ret)
2348 return ret; 2620 return ret;
2349 2621
2350 obj_priv->fence_reg = ret; 2622 if (reg->obj) {
2351 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2623 struct drm_i915_gem_object *old = reg->obj;
2352 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2624
2625 drm_gem_object_reference(&old->base);
2626
2627 if (old->tiling_mode)
2628 i915_gem_release_mmap(old);
2629
2630 ret = i915_gem_object_flush_fence(old, pipelined);
2631 if (ret) {
2632 drm_gem_object_unreference(&old->base);
2633 return ret;
2634 }
2635
2636 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2637 pipelined = NULL;
2638
2639 old->fence_reg = I915_FENCE_REG_NONE;
2640 old->last_fenced_ring = pipelined;
2641 old->last_fenced_seqno =
2642 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2643
2644 drm_gem_object_unreference(&old->base);
2645 } else if (obj->last_fenced_seqno == 0)
2646 pipelined = NULL;
2353 2647
2354 reg->obj = obj; 2648 reg->obj = obj;
2649 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2650 obj->fence_reg = reg - dev_priv->fence_regs;
2651 obj->last_fenced_ring = pipelined;
2355 2652
2653 reg->setup_seqno =
2654 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2655 obj->last_fenced_seqno = reg->setup_seqno;
2656
2657update:
2658 obj->tiling_changed = false;
2356 switch (INTEL_INFO(dev)->gen) { 2659 switch (INTEL_INFO(dev)->gen) {
2660 case 7:
2357 case 6: 2661 case 6:
2358 sandybridge_write_fence_reg(reg); 2662 ret = sandybridge_write_fence_reg(obj, pipelined);
2359 break; 2663 break;
2360 case 5: 2664 case 5:
2361 case 4: 2665 case 4:
2362 i965_write_fence_reg(reg); 2666 ret = i965_write_fence_reg(obj, pipelined);
2363 break; 2667 break;
2364 case 3: 2668 case 3:
2365 i915_write_fence_reg(reg); 2669 ret = i915_write_fence_reg(obj, pipelined);
2366 break; 2670 break;
2367 case 2: 2671 case 2:
2368 i830_write_fence_reg(reg); 2672 ret = i830_write_fence_reg(obj, pipelined);
2369 break; 2673 break;
2370 } 2674 }
2371 2675
2372 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg, 2676 return ret;
2373 obj_priv->tiling_mode);
2374
2375 return 0;
2376} 2677}
2377 2678
2378/** 2679/**
@@ -2380,157 +2681,133 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2380 * @obj: object to clear 2681 * @obj: object to clear
2381 * 2682 *
2382 * Zeroes out the fence register itself and clears out the associated 2683 * Zeroes out the fence register itself and clears out the associated
2383 * data structures in dev_priv and obj_priv. 2684 * data structures in dev_priv and obj.
2384 */ 2685 */
2385static void 2686static void
2386i915_gem_clear_fence_reg(struct drm_gem_object *obj) 2687i915_gem_clear_fence_reg(struct drm_device *dev,
2688 struct drm_i915_fence_reg *reg)
2387{ 2689{
2388 struct drm_device *dev = obj->dev;
2389 drm_i915_private_t *dev_priv = dev->dev_private; 2690 drm_i915_private_t *dev_priv = dev->dev_private;
2390 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2691 uint32_t fence_reg = reg - dev_priv->fence_regs;
2391 struct drm_i915_fence_reg *reg =
2392 &dev_priv->fence_regs[obj_priv->fence_reg];
2393 uint32_t fence_reg;
2394 2692
2395 switch (INTEL_INFO(dev)->gen) { 2693 switch (INTEL_INFO(dev)->gen) {
2694 case 7:
2396 case 6: 2695 case 6:
2397 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + 2696 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2398 (obj_priv->fence_reg * 8), 0);
2399 break; 2697 break;
2400 case 5: 2698 case 5:
2401 case 4: 2699 case 4:
2402 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); 2700 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2403 break; 2701 break;
2404 case 3: 2702 case 3:
2405 if (obj_priv->fence_reg >= 8) 2703 if (fence_reg >= 8)
2406 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; 2704 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2407 else 2705 else
2408 case 2: 2706 case 2:
2409 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; 2707 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2410 2708
2411 I915_WRITE(fence_reg, 0); 2709 I915_WRITE(fence_reg, 0);
2412 break; 2710 break;
2413 } 2711 }
2414 2712
2415 reg->obj = NULL;
2416 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2417 list_del_init(&reg->lru_list); 2713 list_del_init(&reg->lru_list);
2418} 2714 reg->obj = NULL;
2419 2715 reg->setup_seqno = 0;
2420/**
2421 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2422 * to the buffer to finish, and then resets the fence register.
2423 * @obj: tiled object holding a fence register.
2424 *
2425 * Zeroes out the fence register itself and clears out the associated
2426 * data structures in dev_priv and obj_priv.
2427 */
2428int
2429i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2430{
2431 struct drm_device *dev = obj->dev;
2432 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2433
2434 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2435 return 0;
2436
2437 /* If we've changed tiling, GTT-mappings of the object
2438 * need to re-fault to ensure that the correct fence register
2439 * setup is in place.
2440 */
2441 i915_gem_release_mmap(obj);
2442
2443 /* On the i915, GPU access to tiled buffers is via a fence,
2444 * therefore we must wait for any outstanding access to complete
2445 * before clearing the fence.
2446 */
2447 if (!IS_I965G(dev)) {
2448 int ret;
2449
2450 ret = i915_gem_object_flush_gpu_write_domain(obj);
2451 if (ret != 0)
2452 return ret;
2453
2454 ret = i915_gem_object_wait_rendering(obj);
2455 if (ret != 0)
2456 return ret;
2457 }
2458
2459 i915_gem_object_flush_gtt_write_domain(obj);
2460 i915_gem_clear_fence_reg (obj);
2461
2462 return 0;
2463} 2716}
2464 2717
2465/** 2718/**
2466 * Finds free space in the GTT aperture and binds the object there. 2719 * Finds free space in the GTT aperture and binds the object there.
2467 */ 2720 */
2468static int 2721static int
2469i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) 2722i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2723 unsigned alignment,
2724 bool map_and_fenceable)
2470{ 2725{
2471 struct drm_device *dev = obj->dev; 2726 struct drm_device *dev = obj->base.dev;
2472 drm_i915_private_t *dev_priv = dev->dev_private; 2727 drm_i915_private_t *dev_priv = dev->dev_private;
2473 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2474 struct drm_mm_node *free_space; 2728 struct drm_mm_node *free_space;
2475 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; 2729 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2730 u32 size, fence_size, fence_alignment, unfenced_alignment;
2731 bool mappable, fenceable;
2476 int ret; 2732 int ret;
2477 2733
2478 if (obj_priv->madv != I915_MADV_WILLNEED) { 2734 if (obj->madv != I915_MADV_WILLNEED) {
2479 DRM_ERROR("Attempting to bind a purgeable object\n"); 2735 DRM_ERROR("Attempting to bind a purgeable object\n");
2480 return -EINVAL; 2736 return -EINVAL;
2481 } 2737 }
2482 2738
2739 fence_size = i915_gem_get_gtt_size(dev,
2740 obj->base.size,
2741 obj->tiling_mode);
2742 fence_alignment = i915_gem_get_gtt_alignment(dev,
2743 obj->base.size,
2744 obj->tiling_mode);
2745 unfenced_alignment =
2746 i915_gem_get_unfenced_gtt_alignment(dev,
2747 obj->base.size,
2748 obj->tiling_mode);
2749
2483 if (alignment == 0) 2750 if (alignment == 0)
2484 alignment = i915_gem_get_gtt_alignment(obj); 2751 alignment = map_and_fenceable ? fence_alignment :
2485 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { 2752 unfenced_alignment;
2753 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2486 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2754 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2487 return -EINVAL; 2755 return -EINVAL;
2488 } 2756 }
2489 2757
2758 size = map_and_fenceable ? fence_size : obj->base.size;
2759
2490 /* If the object is bigger than the entire aperture, reject it early 2760 /* If the object is bigger than the entire aperture, reject it early
2491 * before evicting everything in a vain attempt to find space. 2761 * before evicting everything in a vain attempt to find space.
2492 */ 2762 */
2493 if (obj->size > dev->gtt_total) { 2763 if (obj->base.size >
2764 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2494 DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 2765 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2495 return -E2BIG; 2766 return -E2BIG;
2496 } 2767 }
2497 2768
2498 search_free: 2769 search_free:
2499 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2770 if (map_and_fenceable)
2500 obj->size, alignment, 0); 2771 free_space =
2772 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2773 size, alignment, 0,
2774 dev_priv->mm.gtt_mappable_end,
2775 0);
2776 else
2777 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2778 size, alignment, 0);
2779
2501 if (free_space != NULL) { 2780 if (free_space != NULL) {
2502 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2781 if (map_and_fenceable)
2503 alignment); 2782 obj->gtt_space =
2504 if (obj_priv->gtt_space != NULL) 2783 drm_mm_get_block_range_generic(free_space,
2505 obj_priv->gtt_offset = obj_priv->gtt_space->start; 2784 size, alignment, 0,
2785 dev_priv->mm.gtt_mappable_end,
2786 0);
2787 else
2788 obj->gtt_space =
2789 drm_mm_get_block(free_space, size, alignment);
2506 } 2790 }
2507 if (obj_priv->gtt_space == NULL) { 2791 if (obj->gtt_space == NULL) {
2508 /* If the gtt is empty and we're still having trouble 2792 /* If the gtt is empty and we're still having trouble
2509 * fitting our object in, we're out of memory. 2793 * fitting our object in, we're out of memory.
2510 */ 2794 */
2511#if WATCH_LRU 2795 ret = i915_gem_evict_something(dev, size, alignment,
2512 DRM_INFO("%s: GTT full, evicting something\n", __func__); 2796 map_and_fenceable);
2513#endif
2514 ret = i915_gem_evict_something(dev, obj->size, alignment);
2515 if (ret) 2797 if (ret)
2516 return ret; 2798 return ret;
2517 2799
2518 goto search_free; 2800 goto search_free;
2519 } 2801 }
2520 2802
2521#if WATCH_BUF 2803 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2522 DRM_INFO("Binding object of size %zd at 0x%08x\n",
2523 obj->size, obj_priv->gtt_offset);
2524#endif
2525 ret = i915_gem_object_get_pages(obj, gfpmask);
2526 if (ret) { 2804 if (ret) {
2527 drm_mm_put_block(obj_priv->gtt_space); 2805 drm_mm_put_block(obj->gtt_space);
2528 obj_priv->gtt_space = NULL; 2806 obj->gtt_space = NULL;
2529 2807
2530 if (ret == -ENOMEM) { 2808 if (ret == -ENOMEM) {
2531 /* first try to clear up some space from the GTT */ 2809 /* first try to reclaim some memory by clearing the GTT */
2532 ret = i915_gem_evict_something(dev, obj->size, 2810 ret = i915_gem_evict_everything(dev, false);
2533 alignment);
2534 if (ret) { 2811 if (ret) {
2535 /* now try to shrink everyone else */ 2812 /* now try to shrink everyone else */
2536 if (gfpmask) { 2813 if (gfpmask) {
@@ -2538,7 +2815,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2538 goto search_free; 2815 goto search_free;
2539 } 2816 }
2540 2817
2541 return ret; 2818 return -ENOMEM;
2542 } 2819 }
2543 2820
2544 goto search_free; 2821 goto search_free;
@@ -2547,144 +2824,126 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2547 return ret; 2824 return ret;
2548 } 2825 }
2549 2826
2550 /* Create an AGP memory structure pointing at our pages, and bind it 2827 ret = i915_gem_gtt_bind_object(obj);
2551 * into the GTT. 2828 if (ret) {
2552 */ 2829 i915_gem_object_put_pages_gtt(obj);
2553 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2830 drm_mm_put_block(obj->gtt_space);
2554 obj_priv->pages, 2831 obj->gtt_space = NULL;
2555 obj->size >> PAGE_SHIFT, 2832
2556 obj_priv->gtt_offset, 2833 if (i915_gem_evict_everything(dev, false))
2557 obj_priv->agp_type);
2558 if (obj_priv->agp_mem == NULL) {
2559 i915_gem_object_put_pages(obj);
2560 drm_mm_put_block(obj_priv->gtt_space);
2561 obj_priv->gtt_space = NULL;
2562
2563 ret = i915_gem_evict_something(dev, obj->size, alignment);
2564 if (ret)
2565 return ret; 2834 return ret;
2566 2835
2567 goto search_free; 2836 goto search_free;
2568 } 2837 }
2569 atomic_inc(&dev->gtt_count);
2570 atomic_add(obj->size, &dev->gtt_memory);
2571 2838
2572 /* keep track of bounds object by adding it to the inactive list */ 2839 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2573 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 2840 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2574 2841
2575 /* Assert that the object is not currently in any GPU domain. As it 2842 /* Assert that the object is not currently in any GPU domain. As it
2576 * wasn't in the GTT, there shouldn't be any way it could have been in 2843 * wasn't in the GTT, there shouldn't be any way it could have been in
2577 * a GPU cache 2844 * a GPU cache
2578 */ 2845 */
2579 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2846 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2580 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2847 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2848
2849 obj->gtt_offset = obj->gtt_space->start;
2850
2851 fenceable =
2852 obj->gtt_space->size == fence_size &&
2853 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2854
2855 mappable =
2856 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2581 2857
2582 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset); 2858 obj->map_and_fenceable = mappable && fenceable;
2583 2859
2860 trace_i915_gem_object_bind(obj, map_and_fenceable);
2584 return 0; 2861 return 0;
2585} 2862}
2586 2863
2587void 2864void
2588i915_gem_clflush_object(struct drm_gem_object *obj) 2865i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2589{ 2866{
2590 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2591
2592 /* If we don't have a page list set up, then we're not pinned 2867 /* If we don't have a page list set up, then we're not pinned
2593 * to GPU, and we can ignore the cache flush because it'll happen 2868 * to GPU, and we can ignore the cache flush because it'll happen
2594 * again at bind time. 2869 * again at bind time.
2595 */ 2870 */
2596 if (obj_priv->pages == NULL) 2871 if (obj->pages == NULL)
2872 return;
2873
2874 /* If the GPU is snooping the contents of the CPU cache,
2875 * we do not need to manually clear the CPU cache lines. However,
2876 * the caches are only snooped when the render cache is
2877 * flushed/invalidated. As we always have to emit invalidations
2878 * and flushes when moving into and out of the RENDER domain, correct
2879 * snooping behaviour occurs naturally as the result of our domain
2880 * tracking.
2881 */
2882 if (obj->cache_level != I915_CACHE_NONE)
2597 return; 2883 return;
2598 2884
2599 trace_i915_gem_object_clflush(obj); 2885 trace_i915_gem_object_clflush(obj);
2600 2886
2601 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); 2887 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2602} 2888}
2603 2889
2604/** Flushes any GPU write domain for the object if it's dirty. */ 2890/** Flushes any GPU write domain for the object if it's dirty. */
2605static int 2891static int
2606i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) 2892i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2607{ 2893{
2608 struct drm_device *dev = obj->dev; 2894 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2609 uint32_t old_write_domain;
2610 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2611
2612 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2613 return 0; 2895 return 0;
2614 2896
2615 /* Queue the GPU write cache flushing we need. */ 2897 /* Queue the GPU write cache flushing we need. */
2616 old_write_domain = obj->write_domain; 2898 return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2617 i915_gem_flush(dev, 0, obj->write_domain);
2618 if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2619 return -ENOMEM;
2620
2621 trace_i915_gem_object_change_domain(obj,
2622 obj->read_domains,
2623 old_write_domain);
2624 return 0;
2625} 2899}
2626 2900
2627/** Flushes the GTT write domain for the object if it's dirty. */ 2901/** Flushes the GTT write domain for the object if it's dirty. */
2628static void 2902static void
2629i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 2903i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2630{ 2904{
2631 uint32_t old_write_domain; 2905 uint32_t old_write_domain;
2632 2906
2633 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 2907 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2634 return; 2908 return;
2635 2909
2636 /* No actual flushing is required for the GTT write domain. Writes 2910 /* No actual flushing is required for the GTT write domain. Writes
2637 * to it immediately go to main memory as far as we know, so there's 2911 * to it immediately go to main memory as far as we know, so there's
2638 * no chipset flush. It also doesn't land in render cache. 2912 * no chipset flush. It also doesn't land in render cache.
2913 *
2914 * However, we do have to enforce the order so that all writes through
2915 * the GTT land before any writes to the device, such as updates to
2916 * the GATT itself.
2639 */ 2917 */
2640 old_write_domain = obj->write_domain; 2918 wmb();
2641 obj->write_domain = 0; 2919
2920 old_write_domain = obj->base.write_domain;
2921 obj->base.write_domain = 0;
2642 2922
2643 trace_i915_gem_object_change_domain(obj, 2923 trace_i915_gem_object_change_domain(obj,
2644 obj->read_domains, 2924 obj->base.read_domains,
2645 old_write_domain); 2925 old_write_domain);
2646} 2926}
2647 2927
2648/** Flushes the CPU write domain for the object if it's dirty. */ 2928/** Flushes the CPU write domain for the object if it's dirty. */
2649static void 2929static void
2650i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 2930i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2651{ 2931{
2652 struct drm_device *dev = obj->dev;
2653 uint32_t old_write_domain; 2932 uint32_t old_write_domain;
2654 2933
2655 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 2934 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2656 return; 2935 return;
2657 2936
2658 i915_gem_clflush_object(obj); 2937 i915_gem_clflush_object(obj);
2659 drm_agp_chipset_flush(dev); 2938 intel_gtt_chipset_flush();
2660 old_write_domain = obj->write_domain; 2939 old_write_domain = obj->base.write_domain;
2661 obj->write_domain = 0; 2940 obj->base.write_domain = 0;
2662 2941
2663 trace_i915_gem_object_change_domain(obj, 2942 trace_i915_gem_object_change_domain(obj,
2664 obj->read_domains, 2943 obj->base.read_domains,
2665 old_write_domain); 2944 old_write_domain);
2666} 2945}
2667 2946
2668int
2669i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2670{
2671 int ret = 0;
2672
2673 switch (obj->write_domain) {
2674 case I915_GEM_DOMAIN_GTT:
2675 i915_gem_object_flush_gtt_write_domain(obj);
2676 break;
2677 case I915_GEM_DOMAIN_CPU:
2678 i915_gem_object_flush_cpu_write_domain(obj);
2679 break;
2680 default:
2681 ret = i915_gem_object_flush_gpu_write_domain(obj);
2682 break;
2683 }
2684
2685 return ret;
2686}
2687
2688/** 2947/**
2689 * Moves a single object to the GTT read, and possibly write domain. 2948 * Moves a single object to the GTT read, and possibly write domain.
2690 * 2949 *
@@ -2692,44 +2951,42 @@ i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2692 * flushes to occur. 2951 * flushes to occur.
2693 */ 2952 */
2694int 2953int
2695i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 2954i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2696{ 2955{
2697 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2698 uint32_t old_write_domain, old_read_domains; 2956 uint32_t old_write_domain, old_read_domains;
2699 int ret; 2957 int ret;
2700 2958
2701 /* Not valid to be called on unbound objects. */ 2959 /* Not valid to be called on unbound objects. */
2702 if (obj_priv->gtt_space == NULL) 2960 if (obj->gtt_space == NULL)
2703 return -EINVAL; 2961 return -EINVAL;
2704 2962
2705 ret = i915_gem_object_flush_gpu_write_domain(obj); 2963 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2706 if (ret != 0) 2964 return 0;
2707 return ret;
2708 2965
2709 /* Wait on any GPU rendering and flushing to occur. */ 2966 ret = i915_gem_object_flush_gpu_write_domain(obj);
2710 ret = i915_gem_object_wait_rendering(obj); 2967 if (ret)
2711 if (ret != 0)
2712 return ret; 2968 return ret;
2713 2969
2714 old_write_domain = obj->write_domain; 2970 if (obj->pending_gpu_write || write) {
2715 old_read_domains = obj->read_domains; 2971 ret = i915_gem_object_wait_rendering(obj);
2716 2972 if (ret)
2717 /* If we're writing through the GTT domain, then CPU and GPU caches 2973 return ret;
2718 * will need to be invalidated at next use. 2974 }
2719 */
2720 if (write)
2721 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2722 2975
2723 i915_gem_object_flush_cpu_write_domain(obj); 2976 i915_gem_object_flush_cpu_write_domain(obj);
2724 2977
2978 old_write_domain = obj->base.write_domain;
2979 old_read_domains = obj->base.read_domains;
2980
2725 /* It should now be out of any other write domains, and we can update 2981 /* It should now be out of any other write domains, and we can update
2726 * the domain values for our changes. 2982 * the domain values for our changes.
2727 */ 2983 */
2728 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 2984 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2729 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2985 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2730 if (write) { 2986 if (write) {
2731 obj->write_domain = I915_GEM_DOMAIN_GTT; 2987 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2732 obj_priv->dirty = 1; 2988 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2989 obj->dirty = 1;
2733 } 2990 }
2734 2991
2735 trace_i915_gem_object_change_domain(obj, 2992 trace_i915_gem_object_change_domain(obj,
@@ -2744,55 +3001,57 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2744 * wait, as in modesetting process we're not supposed to be interrupted. 3001 * wait, as in modesetting process we're not supposed to be interrupted.
2745 */ 3002 */
2746int 3003int
2747i915_gem_object_set_to_display_plane(struct drm_gem_object *obj) 3004i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
3005 struct intel_ring_buffer *pipelined)
2748{ 3006{
2749 struct drm_device *dev = obj->dev; 3007 uint32_t old_read_domains;
2750 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2751 uint32_t old_write_domain, old_read_domains;
2752 int ret; 3008 int ret;
2753 3009
2754 /* Not valid to be called on unbound objects. */ 3010 /* Not valid to be called on unbound objects. */
2755 if (obj_priv->gtt_space == NULL) 3011 if (obj->gtt_space == NULL)
2756 return -EINVAL; 3012 return -EINVAL;
2757 3013
2758 ret = i915_gem_object_flush_gpu_write_domain(obj); 3014 ret = i915_gem_object_flush_gpu_write_domain(obj);
2759 if (ret) 3015 if (ret)
2760 return ret; 3016 return ret;
2761 3017
2762 /* Wait on any GPU rendering and flushing to occur. */ 3018
2763 if (obj_priv->active) { 3019 /* Currently, we are always called from an non-interruptible context. */
2764#if WATCH_BUF 3020 if (pipelined != obj->ring) {
2765 DRM_INFO("%s: object %p wait for seqno %08x\n", 3021 ret = i915_gem_object_wait_rendering(obj);
2766 __func__, obj, obj_priv->last_rendering_seqno); 3022 if (ret)
2767#endif
2768 ret = i915_do_wait_request(dev,
2769 obj_priv->last_rendering_seqno,
2770 0,
2771 obj_priv->ring);
2772 if (ret != 0)
2773 return ret; 3023 return ret;
2774 } 3024 }
2775 3025
2776 i915_gem_object_flush_cpu_write_domain(obj); 3026 i915_gem_object_flush_cpu_write_domain(obj);
2777 3027
2778 old_write_domain = obj->write_domain; 3028 old_read_domains = obj->base.read_domains;
2779 old_read_domains = obj->read_domains; 3029 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2780
2781 /* It should now be out of any other write domains, and we can update
2782 * the domain values for our changes.
2783 */
2784 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2785 obj->read_domains = I915_GEM_DOMAIN_GTT;
2786 obj->write_domain = I915_GEM_DOMAIN_GTT;
2787 obj_priv->dirty = 1;
2788 3030
2789 trace_i915_gem_object_change_domain(obj, 3031 trace_i915_gem_object_change_domain(obj,
2790 old_read_domains, 3032 old_read_domains,
2791 old_write_domain); 3033 obj->base.write_domain);
2792 3034
2793 return 0; 3035 return 0;
2794} 3036}
2795 3037
3038int
3039i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj)
3040{
3041 int ret;
3042
3043 if (!obj->active)
3044 return 0;
3045
3046 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3047 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3048 if (ret)
3049 return ret;
3050 }
3051
3052 return i915_gem_object_wait_rendering(obj);
3053}
3054
2796/** 3055/**
2797 * Moves a single object to the CPU read, and possibly write domain. 3056 * Moves a single object to the CPU read, and possibly write domain.
2798 * 3057 *
@@ -2800,18 +3059,20 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2800 * flushes to occur. 3059 * flushes to occur.
2801 */ 3060 */
2802static int 3061static int
2803i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 3062i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2804{ 3063{
2805 uint32_t old_write_domain, old_read_domains; 3064 uint32_t old_write_domain, old_read_domains;
2806 int ret; 3065 int ret;
2807 3066
3067 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3068 return 0;
3069
2808 ret = i915_gem_object_flush_gpu_write_domain(obj); 3070 ret = i915_gem_object_flush_gpu_write_domain(obj);
2809 if (ret) 3071 if (ret)
2810 return ret; 3072 return ret;
2811 3073
2812 /* Wait on any GPU rendering and flushing to occur. */
2813 ret = i915_gem_object_wait_rendering(obj); 3074 ret = i915_gem_object_wait_rendering(obj);
2814 if (ret != 0) 3075 if (ret)
2815 return ret; 3076 return ret;
2816 3077
2817 i915_gem_object_flush_gtt_write_domain(obj); 3078 i915_gem_object_flush_gtt_write_domain(obj);
@@ -2821,27 +3082,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2821 */ 3082 */
2822 i915_gem_object_set_to_full_cpu_read_domain(obj); 3083 i915_gem_object_set_to_full_cpu_read_domain(obj);
2823 3084
2824 old_write_domain = obj->write_domain; 3085 old_write_domain = obj->base.write_domain;
2825 old_read_domains = obj->read_domains; 3086 old_read_domains = obj->base.read_domains;
2826 3087
2827 /* Flush the CPU cache if it's still invalid. */ 3088 /* Flush the CPU cache if it's still invalid. */
2828 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 3089 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2829 i915_gem_clflush_object(obj); 3090 i915_gem_clflush_object(obj);
2830 3091
2831 obj->read_domains |= I915_GEM_DOMAIN_CPU; 3092 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2832 } 3093 }
2833 3094
2834 /* It should now be out of any other write domains, and we can update 3095 /* It should now be out of any other write domains, and we can update
2835 * the domain values for our changes. 3096 * the domain values for our changes.
2836 */ 3097 */
2837 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3098 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2838 3099
2839 /* If we're writing through the CPU, then the GPU read domains will 3100 /* If we're writing through the CPU, then the GPU read domains will
2840 * need to be invalidated at next use. 3101 * need to be invalidated at next use.
2841 */ 3102 */
2842 if (write) { 3103 if (write) {
2843 obj->read_domains &= I915_GEM_DOMAIN_CPU; 3104 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2844 obj->write_domain = I915_GEM_DOMAIN_CPU; 3105 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2845 } 3106 }
2846 3107
2847 trace_i915_gem_object_change_domain(obj, 3108 trace_i915_gem_object_change_domain(obj,
@@ -2851,205 +3112,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2851 return 0; 3112 return 0;
2852} 3113}
2853 3114
2854/*
2855 * Set the next domain for the specified object. This
2856 * may not actually perform the necessary flushing/invaliding though,
2857 * as that may want to be batched with other set_domain operations
2858 *
2859 * This is (we hope) the only really tricky part of gem. The goal
2860 * is fairly simple -- track which caches hold bits of the object
2861 * and make sure they remain coherent. A few concrete examples may
2862 * help to explain how it works. For shorthand, we use the notation
2863 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2864 * a pair of read and write domain masks.
2865 *
2866 * Case 1: the batch buffer
2867 *
2868 * 1. Allocated
2869 * 2. Written by CPU
2870 * 3. Mapped to GTT
2871 * 4. Read by GPU
2872 * 5. Unmapped from GTT
2873 * 6. Freed
2874 *
2875 * Let's take these a step at a time
2876 *
2877 * 1. Allocated
2878 * Pages allocated from the kernel may still have
2879 * cache contents, so we set them to (CPU, CPU) always.
2880 * 2. Written by CPU (using pwrite)
2881 * The pwrite function calls set_domain (CPU, CPU) and
2882 * this function does nothing (as nothing changes)
2883 * 3. Mapped by GTT
2884 * This function asserts that the object is not
2885 * currently in any GPU-based read or write domains
2886 * 4. Read by GPU
2887 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2888 * As write_domain is zero, this function adds in the
2889 * current read domains (CPU+COMMAND, 0).
2890 * flush_domains is set to CPU.
2891 * invalidate_domains is set to COMMAND
2892 * clflush is run to get data out of the CPU caches
2893 * then i915_dev_set_domain calls i915_gem_flush to
2894 * emit an MI_FLUSH and drm_agp_chipset_flush
2895 * 5. Unmapped from GTT
2896 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2897 * flush_domains and invalidate_domains end up both zero
2898 * so no flushing/invalidating happens
2899 * 6. Freed
2900 * yay, done
2901 *
2902 * Case 2: The shared render buffer
2903 *
2904 * 1. Allocated
2905 * 2. Mapped to GTT
2906 * 3. Read/written by GPU
2907 * 4. set_domain to (CPU,CPU)
2908 * 5. Read/written by CPU
2909 * 6. Read/written by GPU
2910 *
2911 * 1. Allocated
2912 * Same as last example, (CPU, CPU)
2913 * 2. Mapped to GTT
2914 * Nothing changes (assertions find that it is not in the GPU)
2915 * 3. Read/written by GPU
2916 * execbuffer calls set_domain (RENDER, RENDER)
2917 * flush_domains gets CPU
2918 * invalidate_domains gets GPU
2919 * clflush (obj)
2920 * MI_FLUSH and drm_agp_chipset_flush
2921 * 4. set_domain (CPU, CPU)
2922 * flush_domains gets GPU
2923 * invalidate_domains gets CPU
2924 * wait_rendering (obj) to make sure all drawing is complete.
2925 * This will include an MI_FLUSH to get the data from GPU
2926 * to memory
2927 * clflush (obj) to invalidate the CPU cache
2928 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2929 * 5. Read/written by CPU
2930 * cache lines are loaded and dirtied
2931 * 6. Read written by GPU
2932 * Same as last GPU access
2933 *
2934 * Case 3: The constant buffer
2935 *
2936 * 1. Allocated
2937 * 2. Written by CPU
2938 * 3. Read by GPU
2939 * 4. Updated (written) by CPU again
2940 * 5. Read by GPU
2941 *
2942 * 1. Allocated
2943 * (CPU, CPU)
2944 * 2. Written by CPU
2945 * (CPU, CPU)
2946 * 3. Read by GPU
2947 * (CPU+RENDER, 0)
2948 * flush_domains = CPU
2949 * invalidate_domains = RENDER
2950 * clflush (obj)
2951 * MI_FLUSH
2952 * drm_agp_chipset_flush
2953 * 4. Updated (written) by CPU again
2954 * (CPU, CPU)
2955 * flush_domains = 0 (no previous write domain)
2956 * invalidate_domains = 0 (no new read domains)
2957 * 5. Read by GPU
2958 * (CPU+RENDER, 0)
2959 * flush_domains = CPU
2960 * invalidate_domains = RENDER
2961 * clflush (obj)
2962 * MI_FLUSH
2963 * drm_agp_chipset_flush
2964 */
2965static void
2966i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2967{
2968 struct drm_device *dev = obj->dev;
2969 drm_i915_private_t *dev_priv = dev->dev_private;
2970 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2971 uint32_t invalidate_domains = 0;
2972 uint32_t flush_domains = 0;
2973 uint32_t old_read_domains;
2974
2975 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2976 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2977
2978 intel_mark_busy(dev, obj);
2979
2980#if WATCH_BUF
2981 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2982 __func__, obj,
2983 obj->read_domains, obj->pending_read_domains,
2984 obj->write_domain, obj->pending_write_domain);
2985#endif
2986 /*
2987 * If the object isn't moving to a new write domain,
2988 * let the object stay in multiple read domains
2989 */
2990 if (obj->pending_write_domain == 0)
2991 obj->pending_read_domains |= obj->read_domains;
2992 else
2993 obj_priv->dirty = 1;
2994
2995 /*
2996 * Flush the current write domain if
2997 * the new read domains don't match. Invalidate
2998 * any read domains which differ from the old
2999 * write domain
3000 */
3001 if (obj->write_domain &&
3002 obj->write_domain != obj->pending_read_domains) {
3003 flush_domains |= obj->write_domain;
3004 invalidate_domains |=
3005 obj->pending_read_domains & ~obj->write_domain;
3006 }
3007 /*
3008 * Invalidate any read caches which may have
3009 * stale data. That is, any new read domains.
3010 */
3011 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3012 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3013#if WATCH_BUF
3014 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3015 __func__, flush_domains, invalidate_domains);
3016#endif
3017 i915_gem_clflush_object(obj);
3018 }
3019
3020 old_read_domains = obj->read_domains;
3021
3022 /* The actual obj->write_domain will be updated with
3023 * pending_write_domain after we emit the accumulated flush for all
3024 * of our domain changes in execbuffers (which clears objects'
3025 * write_domains). So if we have a current write domain that we
3026 * aren't changing, set pending_write_domain to that.
3027 */
3028 if (flush_domains == 0 && obj->pending_write_domain == 0)
3029 obj->pending_write_domain = obj->write_domain;
3030 obj->read_domains = obj->pending_read_domains;
3031
3032 if (flush_domains & I915_GEM_GPU_DOMAINS) {
3033 if (obj_priv->ring == &dev_priv->render_ring)
3034 dev_priv->flush_rings |= FLUSH_RENDER_RING;
3035 else if (obj_priv->ring == &dev_priv->bsd_ring)
3036 dev_priv->flush_rings |= FLUSH_BSD_RING;
3037 }
3038
3039 dev->invalidate_domains |= invalidate_domains;
3040 dev->flush_domains |= flush_domains;
3041#if WATCH_BUF
3042 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3043 __func__,
3044 obj->read_domains, obj->write_domain,
3045 dev->invalidate_domains, dev->flush_domains);
3046#endif
3047
3048 trace_i915_gem_object_change_domain(obj,
3049 old_read_domains,
3050 obj->write_domain);
3051}
3052
3053/** 3115/**
3054 * Moves the object from a partially CPU read to a full one. 3116 * Moves the object from a partially CPU read to a full one.
3055 * 3117 *
@@ -3057,30 +3119,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3057 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 3119 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3058 */ 3120 */
3059static void 3121static void
3060i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 3122i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3061{ 3123{
3062 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3124 if (!obj->page_cpu_valid)
3063
3064 if (!obj_priv->page_cpu_valid)
3065 return; 3125 return;
3066 3126
3067 /* If we're partially in the CPU read domain, finish moving it in. 3127 /* If we're partially in the CPU read domain, finish moving it in.
3068 */ 3128 */
3069 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 3129 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3070 int i; 3130 int i;
3071 3131
3072 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 3132 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3073 if (obj_priv->page_cpu_valid[i]) 3133 if (obj->page_cpu_valid[i])
3074 continue; 3134 continue;
3075 drm_clflush_pages(obj_priv->pages + i, 1); 3135 drm_clflush_pages(obj->pages + i, 1);
3076 } 3136 }
3077 } 3137 }
3078 3138
3079 /* Free the page_cpu_valid mappings which are now stale, whether 3139 /* Free the page_cpu_valid mappings which are now stale, whether
3080 * or not we've got I915_GEM_DOMAIN_CPU. 3140 * or not we've got I915_GEM_DOMAIN_CPU.
3081 */ 3141 */
3082 kfree(obj_priv->page_cpu_valid); 3142 kfree(obj->page_cpu_valid);
3083 obj_priv->page_cpu_valid = NULL; 3143 obj->page_cpu_valid = NULL;
3084} 3144}
3085 3145
3086/** 3146/**
@@ -3096,282 +3156,66 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3096 * flushes to occur. 3156 * flushes to occur.
3097 */ 3157 */
3098static int 3158static int
3099i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 3159i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3100 uint64_t offset, uint64_t size) 3160 uint64_t offset, uint64_t size)
3101{ 3161{
3102 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3103 uint32_t old_read_domains; 3162 uint32_t old_read_domains;
3104 int i, ret; 3163 int i, ret;
3105 3164
3106 if (offset == 0 && size == obj->size) 3165 if (offset == 0 && size == obj->base.size)
3107 return i915_gem_object_set_to_cpu_domain(obj, 0); 3166 return i915_gem_object_set_to_cpu_domain(obj, 0);
3108 3167
3109 ret = i915_gem_object_flush_gpu_write_domain(obj); 3168 ret = i915_gem_object_flush_gpu_write_domain(obj);
3110 if (ret) 3169 if (ret)
3111 return ret; 3170 return ret;
3112 3171
3113 /* Wait on any GPU rendering and flushing to occur. */
3114 ret = i915_gem_object_wait_rendering(obj); 3172 ret = i915_gem_object_wait_rendering(obj);
3115 if (ret != 0) 3173 if (ret)
3116 return ret; 3174 return ret;
3175
3117 i915_gem_object_flush_gtt_write_domain(obj); 3176 i915_gem_object_flush_gtt_write_domain(obj);
3118 3177
3119 /* If we're already fully in the CPU read domain, we're done. */ 3178 /* If we're already fully in the CPU read domain, we're done. */
3120 if (obj_priv->page_cpu_valid == NULL && 3179 if (obj->page_cpu_valid == NULL &&
3121 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 3180 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3122 return 0; 3181 return 0;
3123 3182
3124 /* Otherwise, create/clear the per-page CPU read domain flag if we're 3183 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3125 * newly adding I915_GEM_DOMAIN_CPU 3184 * newly adding I915_GEM_DOMAIN_CPU
3126 */ 3185 */
3127 if (obj_priv->page_cpu_valid == NULL) { 3186 if (obj->page_cpu_valid == NULL) {
3128 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE, 3187 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3129 GFP_KERNEL); 3188 GFP_KERNEL);
3130 if (obj_priv->page_cpu_valid == NULL) 3189 if (obj->page_cpu_valid == NULL)
3131 return -ENOMEM; 3190 return -ENOMEM;
3132 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 3191 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3133 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 3192 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3134 3193
3135 /* Flush the cache on any pages that are still invalid from the CPU's 3194 /* Flush the cache on any pages that are still invalid from the CPU's
3136 * perspective. 3195 * perspective.
3137 */ 3196 */
3138 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 3197 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3139 i++) { 3198 i++) {
3140 if (obj_priv->page_cpu_valid[i]) 3199 if (obj->page_cpu_valid[i])
3141 continue; 3200 continue;
3142 3201
3143 drm_clflush_pages(obj_priv->pages + i, 1); 3202 drm_clflush_pages(obj->pages + i, 1);
3144 3203
3145 obj_priv->page_cpu_valid[i] = 1; 3204 obj->page_cpu_valid[i] = 1;
3146 } 3205 }
3147 3206
3148 /* It should now be out of any other write domains, and we can update 3207 /* It should now be out of any other write domains, and we can update
3149 * the domain values for our changes. 3208 * the domain values for our changes.
3150 */ 3209 */
3151 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3210 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3152 3211
3153 old_read_domains = obj->read_domains; 3212 old_read_domains = obj->base.read_domains;
3154 obj->read_domains |= I915_GEM_DOMAIN_CPU; 3213 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3155 3214
3156 trace_i915_gem_object_change_domain(obj, 3215 trace_i915_gem_object_change_domain(obj,
3157 old_read_domains, 3216 old_read_domains,
3158 obj->write_domain); 3217 obj->base.write_domain);
3159
3160 return 0;
3161}
3162
3163/**
3164 * Pin an object to the GTT and evaluate the relocations landing in it.
3165 */
3166static int
3167i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3168 struct drm_file *file_priv,
3169 struct drm_i915_gem_exec_object2 *entry,
3170 struct drm_i915_gem_relocation_entry *relocs)
3171{
3172 struct drm_device *dev = obj->dev;
3173 drm_i915_private_t *dev_priv = dev->dev_private;
3174 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3175 int i, ret;
3176 void __iomem *reloc_page;
3177 bool need_fence;
3178
3179 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3180 obj_priv->tiling_mode != I915_TILING_NONE;
3181
3182 /* Check fence reg constraints and rebind if necessary */
3183 if (need_fence &&
3184 !i915_gem_object_fence_offset_ok(obj,
3185 obj_priv->tiling_mode)) {
3186 ret = i915_gem_object_unbind(obj);
3187 if (ret)
3188 return ret;
3189 }
3190
3191 /* Choose the GTT offset for our buffer and put it there. */
3192 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3193 if (ret)
3194 return ret;
3195
3196 /*
3197 * Pre-965 chips need a fence register set up in order to
3198 * properly handle blits to/from tiled surfaces.
3199 */
3200 if (need_fence) {
3201 ret = i915_gem_object_get_fence_reg(obj);
3202 if (ret != 0) {
3203 i915_gem_object_unpin(obj);
3204 return ret;
3205 }
3206 }
3207
3208 entry->offset = obj_priv->gtt_offset;
3209
3210 /* Apply the relocations, using the GTT aperture to avoid cache
3211 * flushing requirements.
3212 */
3213 for (i = 0; i < entry->relocation_count; i++) {
3214 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3215 struct drm_gem_object *target_obj;
3216 struct drm_i915_gem_object *target_obj_priv;
3217 uint32_t reloc_val, reloc_offset;
3218 uint32_t __iomem *reloc_entry;
3219
3220 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3221 reloc->target_handle);
3222 if (target_obj == NULL) {
3223 i915_gem_object_unpin(obj);
3224 return -ENOENT;
3225 }
3226 target_obj_priv = to_intel_bo(target_obj);
3227
3228#if WATCH_RELOC
3229 DRM_INFO("%s: obj %p offset %08x target %d "
3230 "read %08x write %08x gtt %08x "
3231 "presumed %08x delta %08x\n",
3232 __func__,
3233 obj,
3234 (int) reloc->offset,
3235 (int) reloc->target_handle,
3236 (int) reloc->read_domains,
3237 (int) reloc->write_domain,
3238 (int) target_obj_priv->gtt_offset,
3239 (int) reloc->presumed_offset,
3240 reloc->delta);
3241#endif
3242
3243 /* The target buffer should have appeared before us in the
3244 * exec_object list, so it should have a GTT space bound by now.
3245 */
3246 if (target_obj_priv->gtt_space == NULL) {
3247 DRM_ERROR("No GTT space found for object %d\n",
3248 reloc->target_handle);
3249 drm_gem_object_unreference(target_obj);
3250 i915_gem_object_unpin(obj);
3251 return -EINVAL;
3252 }
3253 3218
3254 /* Validate that the target is in a valid r/w GPU domain */
3255 if (reloc->write_domain & (reloc->write_domain - 1)) {
3256 DRM_ERROR("reloc with multiple write domains: "
3257 "obj %p target %d offset %d "
3258 "read %08x write %08x",
3259 obj, reloc->target_handle,
3260 (int) reloc->offset,
3261 reloc->read_domains,
3262 reloc->write_domain);
3263 drm_gem_object_unreference(target_obj);
3264 i915_gem_object_unpin(obj);
3265 return -EINVAL;
3266 }
3267 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3268 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3269 DRM_ERROR("reloc with read/write CPU domains: "
3270 "obj %p target %d offset %d "
3271 "read %08x write %08x",
3272 obj, reloc->target_handle,
3273 (int) reloc->offset,
3274 reloc->read_domains,
3275 reloc->write_domain);
3276 drm_gem_object_unreference(target_obj);
3277 i915_gem_object_unpin(obj);
3278 return -EINVAL;
3279 }
3280 if (reloc->write_domain && target_obj->pending_write_domain &&
3281 reloc->write_domain != target_obj->pending_write_domain) {
3282 DRM_ERROR("Write domain conflict: "
3283 "obj %p target %d offset %d "
3284 "new %08x old %08x\n",
3285 obj, reloc->target_handle,
3286 (int) reloc->offset,
3287 reloc->write_domain,
3288 target_obj->pending_write_domain);
3289 drm_gem_object_unreference(target_obj);
3290 i915_gem_object_unpin(obj);
3291 return -EINVAL;
3292 }
3293
3294 target_obj->pending_read_domains |= reloc->read_domains;
3295 target_obj->pending_write_domain |= reloc->write_domain;
3296
3297 /* If the relocation already has the right value in it, no
3298 * more work needs to be done.
3299 */
3300 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3301 drm_gem_object_unreference(target_obj);
3302 continue;
3303 }
3304
3305 /* Check that the relocation address is valid... */
3306 if (reloc->offset > obj->size - 4) {
3307 DRM_ERROR("Relocation beyond object bounds: "
3308 "obj %p target %d offset %d size %d.\n",
3309 obj, reloc->target_handle,
3310 (int) reloc->offset, (int) obj->size);
3311 drm_gem_object_unreference(target_obj);
3312 i915_gem_object_unpin(obj);
3313 return -EINVAL;
3314 }
3315 if (reloc->offset & 3) {
3316 DRM_ERROR("Relocation not 4-byte aligned: "
3317 "obj %p target %d offset %d.\n",
3318 obj, reloc->target_handle,
3319 (int) reloc->offset);
3320 drm_gem_object_unreference(target_obj);
3321 i915_gem_object_unpin(obj);
3322 return -EINVAL;
3323 }
3324
3325 /* and points to somewhere within the target object. */
3326 if (reloc->delta >= target_obj->size) {
3327 DRM_ERROR("Relocation beyond target object bounds: "
3328 "obj %p target %d delta %d size %d.\n",
3329 obj, reloc->target_handle,
3330 (int) reloc->delta, (int) target_obj->size);
3331 drm_gem_object_unreference(target_obj);
3332 i915_gem_object_unpin(obj);
3333 return -EINVAL;
3334 }
3335
3336 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3337 if (ret != 0) {
3338 drm_gem_object_unreference(target_obj);
3339 i915_gem_object_unpin(obj);
3340 return -EINVAL;
3341 }
3342
3343 /* Map the page containing the relocation we're going to
3344 * perform.
3345 */
3346 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3347 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3348 (reloc_offset &
3349 ~(PAGE_SIZE - 1)),
3350 KM_USER0);
3351 reloc_entry = (uint32_t __iomem *)(reloc_page +
3352 (reloc_offset & (PAGE_SIZE - 1)));
3353 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3354
3355#if WATCH_BUF
3356 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3357 obj, (unsigned int) reloc->offset,
3358 readl(reloc_entry), reloc_val);
3359#endif
3360 writel(reloc_val, reloc_entry);
3361 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3362
3363 /* The updated presumed offset for this entry will be
3364 * copied back out to the user.
3365 */
3366 reloc->presumed_offset = target_obj_priv->gtt_offset;
3367
3368 drm_gem_object_unreference(target_obj);
3369 }
3370
3371#if WATCH_BUF
3372 if (0)
3373 i915_gem_dump_object(obj, 128, __func__, ~0);
3374#endif
3375 return 0; 3219 return 0;
3376} 3220}
3377 3221
@@ -3386,857 +3230,254 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3386 * relatively low latency when blocking on a particular request to finish. 3230 * relatively low latency when blocking on a particular request to finish.
3387 */ 3231 */
3388static int 3232static int
3389i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv) 3233i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3390{ 3234{
3391 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; 3235 struct drm_i915_private *dev_priv = dev->dev_private;
3392 int ret = 0; 3236 struct drm_i915_file_private *file_priv = file->driver_priv;
3393 unsigned long recent_enough = jiffies - msecs_to_jiffies(20); 3237 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3394 3238 struct drm_i915_gem_request *request;
3395 mutex_lock(&dev->struct_mutex);
3396 while (!list_empty(&i915_file_priv->mm.request_list)) {
3397 struct drm_i915_gem_request *request;
3398
3399 request = list_first_entry(&i915_file_priv->mm.request_list,
3400 struct drm_i915_gem_request,
3401 client_list);
3402
3403 if (time_after_eq(request->emitted_jiffies, recent_enough))
3404 break;
3405
3406 ret = i915_wait_request(dev, request->seqno, request->ring);
3407 if (ret != 0)
3408 break;
3409 }
3410 mutex_unlock(&dev->struct_mutex);
3411
3412 return ret;
3413}
3414
3415static int
3416i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3417 uint32_t buffer_count,
3418 struct drm_i915_gem_relocation_entry **relocs)
3419{
3420 uint32_t reloc_count = 0, reloc_index = 0, i;
3421 int ret;
3422
3423 *relocs = NULL;
3424 for (i = 0; i < buffer_count; i++) {
3425 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3426 return -EINVAL;
3427 reloc_count += exec_list[i].relocation_count;
3428 }
3429
3430 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3431 if (*relocs == NULL) {
3432 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3433 return -ENOMEM;
3434 }
3435
3436 for (i = 0; i < buffer_count; i++) {
3437 struct drm_i915_gem_relocation_entry __user *user_relocs;
3438
3439 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3440
3441 ret = copy_from_user(&(*relocs)[reloc_index],
3442 user_relocs,
3443 exec_list[i].relocation_count *
3444 sizeof(**relocs));
3445 if (ret != 0) {
3446 drm_free_large(*relocs);
3447 *relocs = NULL;
3448 return -EFAULT;
3449 }
3450
3451 reloc_index += exec_list[i].relocation_count;
3452 }
3453
3454 return 0;
3455}
3456
3457static int
3458i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3459 uint32_t buffer_count,
3460 struct drm_i915_gem_relocation_entry *relocs)
3461{
3462 uint32_t reloc_count = 0, i;
3463 int ret = 0;
3464
3465 if (relocs == NULL)
3466 return 0;
3467
3468 for (i = 0; i < buffer_count; i++) {
3469 struct drm_i915_gem_relocation_entry __user *user_relocs;
3470 int unwritten;
3471
3472 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3473
3474 unwritten = copy_to_user(user_relocs,
3475 &relocs[reloc_count],
3476 exec_list[i].relocation_count *
3477 sizeof(*relocs));
3478
3479 if (unwritten) {
3480 ret = -EFAULT;
3481 goto err;
3482 }
3483
3484 reloc_count += exec_list[i].relocation_count;
3485 }
3486
3487err:
3488 drm_free_large(relocs);
3489
3490 return ret;
3491}
3492
3493static int
3494i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3495 uint64_t exec_offset)
3496{
3497 uint32_t exec_start, exec_len;
3498
3499 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3500 exec_len = (uint32_t) exec->batch_len;
3501
3502 if ((exec_start | exec_len) & 0x7)
3503 return -EINVAL;
3504
3505 if (!exec_start)
3506 return -EINVAL;
3507
3508 return 0;
3509}
3510
3511static int
3512i915_gem_wait_for_pending_flip(struct drm_device *dev,
3513 struct drm_gem_object **object_list,
3514 int count)
3515{
3516 drm_i915_private_t *dev_priv = dev->dev_private;
3517 struct drm_i915_gem_object *obj_priv;
3518 DEFINE_WAIT(wait);
3519 int i, ret = 0;
3520
3521 for (;;) {
3522 prepare_to_wait(&dev_priv->pending_flip_queue,
3523 &wait, TASK_INTERRUPTIBLE);
3524 for (i = 0; i < count; i++) {
3525 obj_priv = to_intel_bo(object_list[i]);
3526 if (atomic_read(&obj_priv->pending_flip) > 0)
3527 break;
3528 }
3529 if (i == count)
3530 break;
3531
3532 if (!signal_pending(current)) {
3533 mutex_unlock(&dev->struct_mutex);
3534 schedule();
3535 mutex_lock(&dev->struct_mutex);
3536 continue;
3537 }
3538 ret = -ERESTARTSYS;
3539 break;
3540 }
3541 finish_wait(&dev_priv->pending_flip_queue, &wait);
3542
3543 return ret;
3544}
3545
3546
3547int
3548i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3549 struct drm_file *file_priv,
3550 struct drm_i915_gem_execbuffer2 *args,
3551 struct drm_i915_gem_exec_object2 *exec_list)
3552{
3553 drm_i915_private_t *dev_priv = dev->dev_private;
3554 struct drm_gem_object **object_list = NULL;
3555 struct drm_gem_object *batch_obj;
3556 struct drm_i915_gem_object *obj_priv;
3557 struct drm_clip_rect *cliprects = NULL;
3558 struct drm_i915_gem_relocation_entry *relocs = NULL;
3559 int ret = 0, ret2, i, pinned = 0;
3560 uint64_t exec_offset;
3561 uint32_t seqno, flush_domains, reloc_index;
3562 int pin_tries, flips;
3563
3564 struct intel_ring_buffer *ring = NULL; 3239 struct intel_ring_buffer *ring = NULL;
3240 u32 seqno = 0;
3241 int ret;
3565 3242
3566#if WATCH_EXEC 3243 if (atomic_read(&dev_priv->mm.wedged))
3567 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", 3244 return -EIO;
3568 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3569#endif
3570 if (args->flags & I915_EXEC_BSD) {
3571 if (!HAS_BSD(dev)) {
3572 DRM_ERROR("execbuf with wrong flag\n");
3573 return -EINVAL;
3574 }
3575 ring = &dev_priv->bsd_ring;
3576 } else {
3577 ring = &dev_priv->render_ring;
3578 }
3579
3580 if (args->buffer_count < 1) {
3581 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3582 return -EINVAL;
3583 }
3584 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3585 if (object_list == NULL) {
3586 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3587 args->buffer_count);
3588 ret = -ENOMEM;
3589 goto pre_mutex_err;
3590 }
3591
3592 if (args->num_cliprects != 0) {
3593 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3594 GFP_KERNEL);
3595 if (cliprects == NULL) {
3596 ret = -ENOMEM;
3597 goto pre_mutex_err;
3598 }
3599
3600 ret = copy_from_user(cliprects,
3601 (struct drm_clip_rect __user *)
3602 (uintptr_t) args->cliprects_ptr,
3603 sizeof(*cliprects) * args->num_cliprects);
3604 if (ret != 0) {
3605 DRM_ERROR("copy %d cliprects failed: %d\n",
3606 args->num_cliprects, ret);
3607 ret = -EFAULT;
3608 goto pre_mutex_err;
3609 }
3610 }
3611
3612 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3613 &relocs);
3614 if (ret != 0)
3615 goto pre_mutex_err;
3616
3617 mutex_lock(&dev->struct_mutex);
3618
3619 i915_verify_inactive(dev, __FILE__, __LINE__);
3620
3621 if (atomic_read(&dev_priv->mm.wedged)) {
3622 mutex_unlock(&dev->struct_mutex);
3623 ret = -EIO;
3624 goto pre_mutex_err;
3625 }
3626
3627 if (dev_priv->mm.suspended) {
3628 mutex_unlock(&dev->struct_mutex);
3629 ret = -EBUSY;
3630 goto pre_mutex_err;
3631 }
3632
3633 /* Look up object handles */
3634 flips = 0;
3635 for (i = 0; i < args->buffer_count; i++) {
3636 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3637 exec_list[i].handle);
3638 if (object_list[i] == NULL) {
3639 DRM_ERROR("Invalid object handle %d at index %d\n",
3640 exec_list[i].handle, i);
3641 /* prevent error path from reading uninitialized data */
3642 args->buffer_count = i + 1;
3643 ret = -ENOENT;
3644 goto err;
3645 }
3646
3647 obj_priv = to_intel_bo(object_list[i]);
3648 if (obj_priv->in_execbuffer) {
3649 DRM_ERROR("Object %p appears more than once in object list\n",
3650 object_list[i]);
3651 /* prevent error path from reading uninitialized data */
3652 args->buffer_count = i + 1;
3653 ret = -EINVAL;
3654 goto err;
3655 }
3656 obj_priv->in_execbuffer = true;
3657 flips += atomic_read(&obj_priv->pending_flip);
3658 }
3659
3660 if (flips > 0) {
3661 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3662 args->buffer_count);
3663 if (ret)
3664 goto err;
3665 }
3666 3245
3667 /* Pin and relocate */ 3246 spin_lock(&file_priv->mm.lock);
3668 for (pin_tries = 0; ; pin_tries++) { 3247 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3669 ret = 0; 3248 if (time_after_eq(request->emitted_jiffies, recent_enough))
3670 reloc_index = 0;
3671
3672 for (i = 0; i < args->buffer_count; i++) {
3673 object_list[i]->pending_read_domains = 0;
3674 object_list[i]->pending_write_domain = 0;
3675 ret = i915_gem_object_pin_and_relocate(object_list[i],
3676 file_priv,
3677 &exec_list[i],
3678 &relocs[reloc_index]);
3679 if (ret)
3680 break;
3681 pinned = i + 1;
3682 reloc_index += exec_list[i].relocation_count;
3683 }
3684 /* success */
3685 if (ret == 0)
3686 break; 3249 break;
3687 3250
3688 /* error other than GTT full, or we've already tried again */ 3251 ring = request->ring;
3689 if (ret != -ENOSPC || pin_tries >= 1) { 3252 seqno = request->seqno;
3690 if (ret != -ERESTARTSYS) {
3691 unsigned long long total_size = 0;
3692 int num_fences = 0;
3693 for (i = 0; i < args->buffer_count; i++) {
3694 obj_priv = to_intel_bo(object_list[i]);
3695
3696 total_size += object_list[i]->size;
3697 num_fences +=
3698 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3699 obj_priv->tiling_mode != I915_TILING_NONE;
3700 }
3701 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3702 pinned+1, args->buffer_count,
3703 total_size, num_fences,
3704 ret);
3705 DRM_ERROR("%d objects [%d pinned], "
3706 "%d object bytes [%d pinned], "
3707 "%d/%d gtt bytes\n",
3708 atomic_read(&dev->object_count),
3709 atomic_read(&dev->pin_count),
3710 atomic_read(&dev->object_memory),
3711 atomic_read(&dev->pin_memory),
3712 atomic_read(&dev->gtt_memory),
3713 dev->gtt_total);
3714 }
3715 goto err;
3716 }
3717
3718 /* unpin all of our buffers */
3719 for (i = 0; i < pinned; i++)
3720 i915_gem_object_unpin(object_list[i]);
3721 pinned = 0;
3722
3723 /* evict everyone we can from the aperture */
3724 ret = i915_gem_evict_everything(dev);
3725 if (ret && ret != -ENOSPC)
3726 goto err;
3727 }
3728
3729 /* Set the pending read domains for the batch buffer to COMMAND */
3730 batch_obj = object_list[args->buffer_count-1];
3731 if (batch_obj->pending_write_domain) {
3732 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3733 ret = -EINVAL;
3734 goto err;
3735 }
3736 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3737
3738 /* Sanity check the batch buffer, prior to moving objects */
3739 exec_offset = exec_list[args->buffer_count - 1].offset;
3740 ret = i915_gem_check_execbuffer (args, exec_offset);
3741 if (ret != 0) {
3742 DRM_ERROR("execbuf with invalid offset/length\n");
3743 goto err;
3744 }
3745
3746 i915_verify_inactive(dev, __FILE__, __LINE__);
3747
3748 /* Zero the global flush/invalidate flags. These
3749 * will be modified as new domains are computed
3750 * for each object
3751 */
3752 dev->invalidate_domains = 0;
3753 dev->flush_domains = 0;
3754 dev_priv->flush_rings = 0;
3755
3756 for (i = 0; i < args->buffer_count; i++) {
3757 struct drm_gem_object *obj = object_list[i];
3758
3759 /* Compute new gpu domains and update invalidate/flush */
3760 i915_gem_object_set_to_gpu_domain(obj);
3761 } 3253 }
3254 spin_unlock(&file_priv->mm.lock);
3762 3255
3763 i915_verify_inactive(dev, __FILE__, __LINE__); 3256 if (seqno == 0)
3764 3257 return 0;
3765 if (dev->invalidate_domains | dev->flush_domains) {
3766#if WATCH_EXEC
3767 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3768 __func__,
3769 dev->invalidate_domains,
3770 dev->flush_domains);
3771#endif
3772 i915_gem_flush(dev,
3773 dev->invalidate_domains,
3774 dev->flush_domains);
3775 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3776 (void)i915_add_request(dev, file_priv,
3777 dev->flush_domains,
3778 &dev_priv->render_ring);
3779 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3780 (void)i915_add_request(dev, file_priv,
3781 dev->flush_domains,
3782 &dev_priv->bsd_ring);
3783 }
3784
3785 for (i = 0; i < args->buffer_count; i++) {
3786 struct drm_gem_object *obj = object_list[i];
3787 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3788 uint32_t old_write_domain = obj->write_domain;
3789
3790 obj->write_domain = obj->pending_write_domain;
3791 if (obj->write_domain)
3792 list_move_tail(&obj_priv->gpu_write_list,
3793 &dev_priv->mm.gpu_write_list);
3794 else
3795 list_del_init(&obj_priv->gpu_write_list);
3796
3797 trace_i915_gem_object_change_domain(obj,
3798 obj->read_domains,
3799 old_write_domain);
3800 }
3801
3802 i915_verify_inactive(dev, __FILE__, __LINE__);
3803
3804#if WATCH_COHERENCY
3805 for (i = 0; i < args->buffer_count; i++) {
3806 i915_gem_object_check_coherency(object_list[i],
3807 exec_list[i].handle);
3808 }
3809#endif
3810
3811#if WATCH_EXEC
3812 i915_gem_dump_object(batch_obj,
3813 args->batch_len,
3814 __func__,
3815 ~0);
3816#endif
3817
3818 /* Exec the batchbuffer */
3819 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3820 cliprects, exec_offset);
3821 if (ret) {
3822 DRM_ERROR("dispatch failed %d\n", ret);
3823 goto err;
3824 }
3825
3826 /*
3827 * Ensure that the commands in the batch buffer are
3828 * finished before the interrupt fires
3829 */
3830 flush_domains = i915_retire_commands(dev, ring);
3831
3832 i915_verify_inactive(dev, __FILE__, __LINE__);
3833
3834 /*
3835 * Get a seqno representing the execution of the current buffer,
3836 * which we can wait on. We would like to mitigate these interrupts,
3837 * likely by only creating seqnos occasionally (so that we have
3838 * *some* interrupts representing completion of buffers that we can
3839 * wait on when trying to clear up gtt space).
3840 */
3841 seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3842 BUG_ON(seqno == 0);
3843 for (i = 0; i < args->buffer_count; i++) {
3844 struct drm_gem_object *obj = object_list[i];
3845 obj_priv = to_intel_bo(obj);
3846
3847 i915_gem_object_move_to_active(obj, seqno, ring);
3848#if WATCH_LRU
3849 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3850#endif
3851 }
3852#if WATCH_LRU
3853 i915_dump_lru(dev, __func__);
3854#endif
3855
3856 i915_verify_inactive(dev, __FILE__, __LINE__);
3857
3858err:
3859 for (i = 0; i < pinned; i++)
3860 i915_gem_object_unpin(object_list[i]);
3861
3862 for (i = 0; i < args->buffer_count; i++) {
3863 if (object_list[i]) {
3864 obj_priv = to_intel_bo(object_list[i]);
3865 obj_priv->in_execbuffer = false;
3866 }
3867 drm_gem_object_unreference(object_list[i]);
3868 }
3869
3870 mutex_unlock(&dev->struct_mutex);
3871
3872pre_mutex_err:
3873 /* Copy the updated relocations out regardless of current error
3874 * state. Failure to update the relocs would mean that the next
3875 * time userland calls execbuf, it would do so with presumed offset
3876 * state that didn't match the actual object state.
3877 */
3878 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3879 relocs);
3880 if (ret2 != 0) {
3881 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3882
3883 if (ret == 0)
3884 ret = ret2;
3885 }
3886
3887 drm_free_large(object_list);
3888 kfree(cliprects);
3889
3890 return ret;
3891}
3892
3893/*
3894 * Legacy execbuffer just creates an exec2 list from the original exec object
3895 * list array and passes it to the real function.
3896 */
3897int
3898i915_gem_execbuffer(struct drm_device *dev, void *data,
3899 struct drm_file *file_priv)
3900{
3901 struct drm_i915_gem_execbuffer *args = data;
3902 struct drm_i915_gem_execbuffer2 exec2;
3903 struct drm_i915_gem_exec_object *exec_list = NULL;
3904 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3905 int ret, i;
3906
3907#if WATCH_EXEC
3908 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3909 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3910#endif
3911
3912 if (args->buffer_count < 1) {
3913 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3914 return -EINVAL;
3915 }
3916
3917 /* Copy in the exec list from userland */
3918 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3919 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3920 if (exec_list == NULL || exec2_list == NULL) {
3921 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3922 args->buffer_count);
3923 drm_free_large(exec_list);
3924 drm_free_large(exec2_list);
3925 return -ENOMEM;
3926 }
3927 ret = copy_from_user(exec_list,
3928 (struct drm_i915_relocation_entry __user *)
3929 (uintptr_t) args->buffers_ptr,
3930 sizeof(*exec_list) * args->buffer_count);
3931 if (ret != 0) {
3932 DRM_ERROR("copy %d exec entries failed %d\n",
3933 args->buffer_count, ret);
3934 drm_free_large(exec_list);
3935 drm_free_large(exec2_list);
3936 return -EFAULT;
3937 }
3938 3258
3939 for (i = 0; i < args->buffer_count; i++) { 3259 ret = 0;
3940 exec2_list[i].handle = exec_list[i].handle; 3260 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3941 exec2_list[i].relocation_count = exec_list[i].relocation_count; 3261 /* And wait for the seqno passing without holding any locks and
3942 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; 3262 * causing extra latency for others. This is safe as the irq
3943 exec2_list[i].alignment = exec_list[i].alignment; 3263 * generation is designed to be run atomically and so is
3944 exec2_list[i].offset = exec_list[i].offset; 3264 * lockless.
3945 if (!IS_I965G(dev)) 3265 */
3946 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; 3266 if (ring->irq_get(ring)) {
3947 else 3267 ret = wait_event_interruptible(ring->irq_queue,
3948 exec2_list[i].flags = 0; 3268 i915_seqno_passed(ring->get_seqno(ring), seqno)
3949 } 3269 || atomic_read(&dev_priv->mm.wedged));
3270 ring->irq_put(ring);
3950 3271
3951 exec2.buffers_ptr = args->buffers_ptr; 3272 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3952 exec2.buffer_count = args->buffer_count; 3273 ret = -EIO;
3953 exec2.batch_start_offset = args->batch_start_offset;
3954 exec2.batch_len = args->batch_len;
3955 exec2.DR1 = args->DR1;
3956 exec2.DR4 = args->DR4;
3957 exec2.num_cliprects = args->num_cliprects;
3958 exec2.cliprects_ptr = args->cliprects_ptr;
3959 exec2.flags = I915_EXEC_RENDER;
3960
3961 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3962 if (!ret) {
3963 /* Copy the new buffer offsets back to the user's exec list. */
3964 for (i = 0; i < args->buffer_count; i++)
3965 exec_list[i].offset = exec2_list[i].offset;
3966 /* ... and back out to userspace */
3967 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3968 (uintptr_t) args->buffers_ptr,
3969 exec_list,
3970 sizeof(*exec_list) * args->buffer_count);
3971 if (ret) {
3972 ret = -EFAULT;
3973 DRM_ERROR("failed to copy %d exec entries "
3974 "back to user (%d)\n",
3975 args->buffer_count, ret);
3976 } 3274 }
3977 } 3275 }
3978 3276
3979 drm_free_large(exec_list); 3277 if (ret == 0)
3980 drm_free_large(exec2_list); 3278 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3981 return ret;
3982}
3983
3984int
3985i915_gem_execbuffer2(struct drm_device *dev, void *data,
3986 struct drm_file *file_priv)
3987{
3988 struct drm_i915_gem_execbuffer2 *args = data;
3989 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3990 int ret;
3991
3992#if WATCH_EXEC
3993 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3994 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3995#endif
3996
3997 if (args->buffer_count < 1) {
3998 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3999 return -EINVAL;
4000 }
4001
4002 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4003 if (exec2_list == NULL) {
4004 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4005 args->buffer_count);
4006 return -ENOMEM;
4007 }
4008 ret = copy_from_user(exec2_list,
4009 (struct drm_i915_relocation_entry __user *)
4010 (uintptr_t) args->buffers_ptr,
4011 sizeof(*exec2_list) * args->buffer_count);
4012 if (ret != 0) {
4013 DRM_ERROR("copy %d exec entries failed %d\n",
4014 args->buffer_count, ret);
4015 drm_free_large(exec2_list);
4016 return -EFAULT;
4017 }
4018
4019 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4020 if (!ret) {
4021 /* Copy the new buffer offsets back to the user's exec list. */
4022 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4023 (uintptr_t) args->buffers_ptr,
4024 exec2_list,
4025 sizeof(*exec2_list) * args->buffer_count);
4026 if (ret) {
4027 ret = -EFAULT;
4028 DRM_ERROR("failed to copy %d exec entries "
4029 "back to user (%d)\n",
4030 args->buffer_count, ret);
4031 }
4032 }
4033 3279
4034 drm_free_large(exec2_list);
4035 return ret; 3280 return ret;
4036} 3281}
4037 3282
4038int 3283int
4039i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 3284i915_gem_object_pin(struct drm_i915_gem_object *obj,
3285 uint32_t alignment,
3286 bool map_and_fenceable)
4040{ 3287{
4041 struct drm_device *dev = obj->dev; 3288 struct drm_device *dev = obj->base.dev;
4042 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3289 struct drm_i915_private *dev_priv = dev->dev_private;
4043 int ret; 3290 int ret;
4044 3291
4045 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); 3292 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4046 3293 WARN_ON(i915_verify_lists(dev));
4047 i915_verify_inactive(dev, __FILE__, __LINE__);
4048 3294
4049 if (obj_priv->gtt_space != NULL) { 3295 if (obj->gtt_space != NULL) {
4050 if (alignment == 0) 3296 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4051 alignment = i915_gem_get_gtt_alignment(obj); 3297 (map_and_fenceable && !obj->map_and_fenceable)) {
4052 if (obj_priv->gtt_offset & (alignment - 1)) { 3298 WARN(obj->pin_count,
4053 WARN(obj_priv->pin_count,
4054 "bo is already pinned with incorrect alignment:" 3299 "bo is already pinned with incorrect alignment:"
4055 " offset=%x, req.alignment=%x\n", 3300 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4056 obj_priv->gtt_offset, alignment); 3301 " obj->map_and_fenceable=%d\n",
3302 obj->gtt_offset, alignment,
3303 map_and_fenceable,
3304 obj->map_and_fenceable);
4057 ret = i915_gem_object_unbind(obj); 3305 ret = i915_gem_object_unbind(obj);
4058 if (ret) 3306 if (ret)
4059 return ret; 3307 return ret;
4060 } 3308 }
4061 } 3309 }
4062 3310
4063 if (obj_priv->gtt_space == NULL) { 3311 if (obj->gtt_space == NULL) {
4064 ret = i915_gem_object_bind_to_gtt(obj, alignment); 3312 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3313 map_and_fenceable);
4065 if (ret) 3314 if (ret)
4066 return ret; 3315 return ret;
4067 } 3316 }
4068 3317
4069 obj_priv->pin_count++; 3318 if (obj->pin_count++ == 0) {
4070 3319 if (!obj->active)
4071 /* If the object is not active and not pending a flush, 3320 list_move_tail(&obj->mm_list,
4072 * remove it from the inactive list 3321 &dev_priv->mm.pinned_list);
4073 */
4074 if (obj_priv->pin_count == 1) {
4075 atomic_inc(&dev->pin_count);
4076 atomic_add(obj->size, &dev->pin_memory);
4077 if (!obj_priv->active &&
4078 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4079 list_del_init(&obj_priv->list);
4080 } 3322 }
4081 i915_verify_inactive(dev, __FILE__, __LINE__); 3323 obj->pin_mappable |= map_and_fenceable;
4082 3324
3325 WARN_ON(i915_verify_lists(dev));
4083 return 0; 3326 return 0;
4084} 3327}
4085 3328
4086void 3329void
4087i915_gem_object_unpin(struct drm_gem_object *obj) 3330i915_gem_object_unpin(struct drm_i915_gem_object *obj)
4088{ 3331{
4089 struct drm_device *dev = obj->dev; 3332 struct drm_device *dev = obj->base.dev;
4090 drm_i915_private_t *dev_priv = dev->dev_private; 3333 drm_i915_private_t *dev_priv = dev->dev_private;
4091 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4092 3334
4093 i915_verify_inactive(dev, __FILE__, __LINE__); 3335 WARN_ON(i915_verify_lists(dev));
4094 obj_priv->pin_count--; 3336 BUG_ON(obj->pin_count == 0);
4095 BUG_ON(obj_priv->pin_count < 0); 3337 BUG_ON(obj->gtt_space == NULL);
4096 BUG_ON(obj_priv->gtt_space == NULL);
4097 3338
4098 /* If the object is no longer pinned, and is 3339 if (--obj->pin_count == 0) {
4099 * neither active nor being flushed, then stick it on 3340 if (!obj->active)
4100 * the inactive list 3341 list_move_tail(&obj->mm_list,
4101 */
4102 if (obj_priv->pin_count == 0) {
4103 if (!obj_priv->active &&
4104 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4105 list_move_tail(&obj_priv->list,
4106 &dev_priv->mm.inactive_list); 3342 &dev_priv->mm.inactive_list);
4107 atomic_dec(&dev->pin_count); 3343 obj->pin_mappable = false;
4108 atomic_sub(obj->size, &dev->pin_memory);
4109 } 3344 }
4110 i915_verify_inactive(dev, __FILE__, __LINE__); 3345 WARN_ON(i915_verify_lists(dev));
4111} 3346}
4112 3347
4113int 3348int
4114i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3349i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4115 struct drm_file *file_priv) 3350 struct drm_file *file)
4116{ 3351{
4117 struct drm_i915_gem_pin *args = data; 3352 struct drm_i915_gem_pin *args = data;
4118 struct drm_gem_object *obj; 3353 struct drm_i915_gem_object *obj;
4119 struct drm_i915_gem_object *obj_priv;
4120 int ret; 3354 int ret;
4121 3355
4122 mutex_lock(&dev->struct_mutex); 3356 ret = i915_mutex_lock_interruptible(dev);
3357 if (ret)
3358 return ret;
4123 3359
4124 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3360 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4125 if (obj == NULL) { 3361 if (&obj->base == NULL) {
4126 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", 3362 ret = -ENOENT;
4127 args->handle); 3363 goto unlock;
4128 mutex_unlock(&dev->struct_mutex);
4129 return -ENOENT;
4130 } 3364 }
4131 obj_priv = to_intel_bo(obj);
4132 3365
4133 if (obj_priv->madv != I915_MADV_WILLNEED) { 3366 if (obj->madv != I915_MADV_WILLNEED) {
4134 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3367 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4135 drm_gem_object_unreference(obj); 3368 ret = -EINVAL;
4136 mutex_unlock(&dev->struct_mutex); 3369 goto out;
4137 return -EINVAL;
4138 } 3370 }
4139 3371
4140 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 3372 if (obj->pin_filp != NULL && obj->pin_filp != file) {
4141 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3373 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4142 args->handle); 3374 args->handle);
4143 drm_gem_object_unreference(obj); 3375 ret = -EINVAL;
4144 mutex_unlock(&dev->struct_mutex); 3376 goto out;
4145 return -EINVAL;
4146 } 3377 }
4147 3378
4148 obj_priv->user_pin_count++; 3379 obj->user_pin_count++;
4149 obj_priv->pin_filp = file_priv; 3380 obj->pin_filp = file;
4150 if (obj_priv->user_pin_count == 1) { 3381 if (obj->user_pin_count == 1) {
4151 ret = i915_gem_object_pin(obj, args->alignment); 3382 ret = i915_gem_object_pin(obj, args->alignment, true);
4152 if (ret != 0) { 3383 if (ret)
4153 drm_gem_object_unreference(obj); 3384 goto out;
4154 mutex_unlock(&dev->struct_mutex);
4155 return ret;
4156 }
4157 } 3385 }
4158 3386
4159 /* XXX - flush the CPU caches for pinned objects 3387 /* XXX - flush the CPU caches for pinned objects
4160 * as the X server doesn't manage domains yet 3388 * as the X server doesn't manage domains yet
4161 */ 3389 */
4162 i915_gem_object_flush_cpu_write_domain(obj); 3390 i915_gem_object_flush_cpu_write_domain(obj);
4163 args->offset = obj_priv->gtt_offset; 3391 args->offset = obj->gtt_offset;
4164 drm_gem_object_unreference(obj); 3392out:
3393 drm_gem_object_unreference(&obj->base);
3394unlock:
4165 mutex_unlock(&dev->struct_mutex); 3395 mutex_unlock(&dev->struct_mutex);
4166 3396 return ret;
4167 return 0;
4168} 3397}
4169 3398
4170int 3399int
4171i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3400i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4172 struct drm_file *file_priv) 3401 struct drm_file *file)
4173{ 3402{
4174 struct drm_i915_gem_pin *args = data; 3403 struct drm_i915_gem_pin *args = data;
4175 struct drm_gem_object *obj; 3404 struct drm_i915_gem_object *obj;
4176 struct drm_i915_gem_object *obj_priv; 3405 int ret;
4177 3406
4178 mutex_lock(&dev->struct_mutex); 3407 ret = i915_mutex_lock_interruptible(dev);
3408 if (ret)
3409 return ret;
4179 3410
4180 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3411 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4181 if (obj == NULL) { 3412 if (&obj->base == NULL) {
4182 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", 3413 ret = -ENOENT;
4183 args->handle); 3414 goto unlock;
4184 mutex_unlock(&dev->struct_mutex);
4185 return -ENOENT;
4186 } 3415 }
4187 3416
4188 obj_priv = to_intel_bo(obj); 3417 if (obj->pin_filp != file) {
4189 if (obj_priv->pin_filp != file_priv) {
4190 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3418 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4191 args->handle); 3419 args->handle);
4192 drm_gem_object_unreference(obj); 3420 ret = -EINVAL;
4193 mutex_unlock(&dev->struct_mutex); 3421 goto out;
4194 return -EINVAL;
4195 } 3422 }
4196 obj_priv->user_pin_count--; 3423 obj->user_pin_count--;
4197 if (obj_priv->user_pin_count == 0) { 3424 if (obj->user_pin_count == 0) {
4198 obj_priv->pin_filp = NULL; 3425 obj->pin_filp = NULL;
4199 i915_gem_object_unpin(obj); 3426 i915_gem_object_unpin(obj);
4200 } 3427 }
4201 3428
4202 drm_gem_object_unreference(obj); 3429out:
3430 drm_gem_object_unreference(&obj->base);
3431unlock:
4203 mutex_unlock(&dev->struct_mutex); 3432 mutex_unlock(&dev->struct_mutex);
4204 return 0; 3433 return ret;
4205} 3434}
4206 3435
4207int 3436int
4208i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3437i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4209 struct drm_file *file_priv) 3438 struct drm_file *file)
4210{ 3439{
4211 struct drm_i915_gem_busy *args = data; 3440 struct drm_i915_gem_busy *args = data;
4212 struct drm_gem_object *obj; 3441 struct drm_i915_gem_object *obj;
4213 struct drm_i915_gem_object *obj_priv; 3442 int ret;
4214 3443
4215 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3444 ret = i915_mutex_lock_interruptible(dev);
4216 if (obj == NULL) { 3445 if (ret)
4217 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n", 3446 return ret;
4218 args->handle);
4219 return -ENOENT;
4220 }
4221 3447
4222 mutex_lock(&dev->struct_mutex); 3448 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3449 if (&obj->base == NULL) {
3450 ret = -ENOENT;
3451 goto unlock;
3452 }
4223 3453
4224 /* Count all active objects as busy, even if they are currently not used 3454 /* Count all active objects as busy, even if they are currently not used
4225 * by the gpu. Users of this interface expect objects to eventually 3455 * by the gpu. Users of this interface expect objects to eventually
4226 * become non-busy without any further actions, therefore emit any 3456 * become non-busy without any further actions, therefore emit any
4227 * necessary flushes here. 3457 * necessary flushes here.
4228 */ 3458 */
4229 obj_priv = to_intel_bo(obj); 3459 args->busy = obj->active;
4230 args->busy = obj_priv->active;
4231 if (args->busy) { 3460 if (args->busy) {
4232 /* Unconditionally flush objects, even when the gpu still uses this 3461 /* Unconditionally flush objects, even when the gpu still uses this
4233 * object. Userspace calling this function indicates that it wants to 3462 * object. Userspace calling this function indicates that it wants to
4234 * use this buffer rather sooner than later, so issuing the required 3463 * use this buffer rather sooner than later, so issuing the required
4235 * flush earlier is beneficial. 3464 * flush earlier is beneficial.
4236 */ 3465 */
4237 if (obj->write_domain) { 3466 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
4238 i915_gem_flush(dev, 0, obj->write_domain); 3467 ret = i915_gem_flush_ring(obj->ring,
4239 (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring); 3468 0, obj->base.write_domain);
3469 } else if (obj->ring->outstanding_lazy_request ==
3470 obj->last_rendering_seqno) {
3471 struct drm_i915_gem_request *request;
3472
3473 /* This ring is not being cleared by active usage,
3474 * so emit a request to do so.
3475 */
3476 request = kzalloc(sizeof(*request), GFP_KERNEL);
3477 if (request)
3478 ret = i915_add_request(obj->ring, NULL,request);
3479 else
3480 ret = -ENOMEM;
4240 } 3481 }
4241 3482
4242 /* Update the active list for the hardware's current position. 3483 /* Update the active list for the hardware's current position.
@@ -4244,14 +3485,15 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4244 * are actually unmasked, and our working set ends up being 3485 * are actually unmasked, and our working set ends up being
4245 * larger than required. 3486 * larger than required.
4246 */ 3487 */
4247 i915_gem_retire_requests_ring(dev, obj_priv->ring); 3488 i915_gem_retire_requests_ring(obj->ring);
4248 3489
4249 args->busy = obj_priv->active; 3490 args->busy = obj->active;
4250 } 3491 }
4251 3492
4252 drm_gem_object_unreference(obj); 3493 drm_gem_object_unreference(&obj->base);
3494unlock:
4253 mutex_unlock(&dev->struct_mutex); 3495 mutex_unlock(&dev->struct_mutex);
4254 return 0; 3496 return ret;
4255} 3497}
4256 3498
4257int 3499int
@@ -4266,8 +3508,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4266 struct drm_file *file_priv) 3508 struct drm_file *file_priv)
4267{ 3509{
4268 struct drm_i915_gem_madvise *args = data; 3510 struct drm_i915_gem_madvise *args = data;
4269 struct drm_gem_object *obj; 3511 struct drm_i915_gem_object *obj;
4270 struct drm_i915_gem_object *obj_priv; 3512 int ret;
4271 3513
4272 switch (args->madv) { 3514 switch (args->madv) {
4273 case I915_MADV_DONTNEED: 3515 case I915_MADV_DONTNEED:
@@ -4277,44 +3519,44 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4277 return -EINVAL; 3519 return -EINVAL;
4278 } 3520 }
4279 3521
4280 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3522 ret = i915_mutex_lock_interruptible(dev);
4281 if (obj == NULL) { 3523 if (ret)
4282 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n", 3524 return ret;
4283 args->handle);
4284 return -ENOENT;
4285 }
4286
4287 mutex_lock(&dev->struct_mutex);
4288 obj_priv = to_intel_bo(obj);
4289 3525
4290 if (obj_priv->pin_count) { 3526 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4291 drm_gem_object_unreference(obj); 3527 if (&obj->base == NULL) {
4292 mutex_unlock(&dev->struct_mutex); 3528 ret = -ENOENT;
3529 goto unlock;
3530 }
4293 3531
4294 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n"); 3532 if (obj->pin_count) {
4295 return -EINVAL; 3533 ret = -EINVAL;
3534 goto out;
4296 } 3535 }
4297 3536
4298 if (obj_priv->madv != __I915_MADV_PURGED) 3537 if (obj->madv != __I915_MADV_PURGED)
4299 obj_priv->madv = args->madv; 3538 obj->madv = args->madv;
4300 3539
4301 /* if the object is no longer bound, discard its backing storage */ 3540 /* if the object is no longer bound, discard its backing storage */
4302 if (i915_gem_object_is_purgeable(obj_priv) && 3541 if (i915_gem_object_is_purgeable(obj) &&
4303 obj_priv->gtt_space == NULL) 3542 obj->gtt_space == NULL)
4304 i915_gem_object_truncate(obj); 3543 i915_gem_object_truncate(obj);
4305 3544
4306 args->retained = obj_priv->madv != __I915_MADV_PURGED; 3545 args->retained = obj->madv != __I915_MADV_PURGED;
4307 3546
4308 drm_gem_object_unreference(obj); 3547out:
3548 drm_gem_object_unreference(&obj->base);
3549unlock:
4309 mutex_unlock(&dev->struct_mutex); 3550 mutex_unlock(&dev->struct_mutex);
4310 3551 return ret;
4311 return 0;
4312} 3552}
4313 3553
4314struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 3554struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4315 size_t size) 3555 size_t size)
4316{ 3556{
3557 struct drm_i915_private *dev_priv = dev->dev_private;
4317 struct drm_i915_gem_object *obj; 3558 struct drm_i915_gem_object *obj;
3559 struct address_space *mapping;
4318 3560
4319 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 3561 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4320 if (obj == NULL) 3562 if (obj == NULL)
@@ -4325,19 +3567,27 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4325 return NULL; 3567 return NULL;
4326 } 3568 }
4327 3569
3570 mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3571 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
3572
3573 i915_gem_info_add_obj(dev_priv, size);
3574
4328 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 3575 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4329 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 3576 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4330 3577
4331 obj->agp_type = AGP_USER_MEMORY; 3578 obj->cache_level = I915_CACHE_NONE;
4332 obj->base.driver_private = NULL; 3579 obj->base.driver_private = NULL;
4333 obj->fence_reg = I915_FENCE_REG_NONE; 3580 obj->fence_reg = I915_FENCE_REG_NONE;
4334 INIT_LIST_HEAD(&obj->list); 3581 INIT_LIST_HEAD(&obj->mm_list);
3582 INIT_LIST_HEAD(&obj->gtt_list);
3583 INIT_LIST_HEAD(&obj->ring_list);
3584 INIT_LIST_HEAD(&obj->exec_list);
4335 INIT_LIST_HEAD(&obj->gpu_write_list); 3585 INIT_LIST_HEAD(&obj->gpu_write_list);
4336 obj->madv = I915_MADV_WILLNEED; 3586 obj->madv = I915_MADV_WILLNEED;
3587 /* Avoid an unnecessary call to unbind on the first bind. */
3588 obj->map_and_fenceable = true;
4337 3589
4338 trace_i915_gem_object_create(&obj->base); 3590 return obj;
4339
4340 return &obj->base;
4341} 3591}
4342 3592
4343int i915_gem_init_object(struct drm_gem_object *obj) 3593int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4347,41 +3597,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
4347 return 0; 3597 return 0;
4348} 3598}
4349 3599
4350static void i915_gem_free_object_tail(struct drm_gem_object *obj) 3600static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
4351{ 3601{
4352 struct drm_device *dev = obj->dev; 3602 struct drm_device *dev = obj->base.dev;
4353 drm_i915_private_t *dev_priv = dev->dev_private; 3603 drm_i915_private_t *dev_priv = dev->dev_private;
4354 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4355 int ret; 3604 int ret;
4356 3605
4357 ret = i915_gem_object_unbind(obj); 3606 ret = i915_gem_object_unbind(obj);
4358 if (ret == -ERESTARTSYS) { 3607 if (ret == -ERESTARTSYS) {
4359 list_move(&obj_priv->list, 3608 list_move(&obj->mm_list,
4360 &dev_priv->mm.deferred_free_list); 3609 &dev_priv->mm.deferred_free_list);
4361 return; 3610 return;
4362 } 3611 }
4363 3612
4364 if (obj_priv->mmap_offset) 3613 trace_i915_gem_object_destroy(obj);
3614
3615 if (obj->base.map_list.map)
4365 i915_gem_free_mmap_offset(obj); 3616 i915_gem_free_mmap_offset(obj);
4366 3617
4367 drm_gem_object_release(obj); 3618 drm_gem_object_release(&obj->base);
3619 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4368 3620
4369 kfree(obj_priv->page_cpu_valid); 3621 kfree(obj->page_cpu_valid);
4370 kfree(obj_priv->bit_17); 3622 kfree(obj->bit_17);
4371 kfree(obj_priv); 3623 kfree(obj);
4372} 3624}
4373 3625
4374void i915_gem_free_object(struct drm_gem_object *obj) 3626void i915_gem_free_object(struct drm_gem_object *gem_obj)
4375{ 3627{
4376 struct drm_device *dev = obj->dev; 3628 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4377 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3629 struct drm_device *dev = obj->base.dev;
4378
4379 trace_i915_gem_object_destroy(obj);
4380 3630
4381 while (obj_priv->pin_count > 0) 3631 while (obj->pin_count > 0)
4382 i915_gem_object_unpin(obj); 3632 i915_gem_object_unpin(obj);
4383 3633
4384 if (obj_priv->phys_obj) 3634 if (obj->phys_obj)
4385 i915_gem_detach_phys_object(dev, obj); 3635 i915_gem_detach_phys_object(dev, obj);
4386 3636
4387 i915_gem_free_object_tail(obj); 3637 i915_gem_free_object_tail(obj);
@@ -4395,10 +3645,7 @@ i915_gem_idle(struct drm_device *dev)
4395 3645
4396 mutex_lock(&dev->struct_mutex); 3646 mutex_lock(&dev->struct_mutex);
4397 3647
4398 if (dev_priv->mm.suspended || 3648 if (dev_priv->mm.suspended) {
4399 (dev_priv->render_ring.gem_object == NULL) ||
4400 (HAS_BSD(dev) &&
4401 dev_priv->bsd_ring.gem_object == NULL)) {
4402 mutex_unlock(&dev->struct_mutex); 3649 mutex_unlock(&dev->struct_mutex);
4403 return 0; 3650 return 0;
4404 } 3651 }
@@ -4411,19 +3658,21 @@ i915_gem_idle(struct drm_device *dev)
4411 3658
4412 /* Under UMS, be paranoid and evict. */ 3659 /* Under UMS, be paranoid and evict. */
4413 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 3660 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4414 ret = i915_gem_evict_inactive(dev); 3661 ret = i915_gem_evict_inactive(dev, false);
4415 if (ret) { 3662 if (ret) {
4416 mutex_unlock(&dev->struct_mutex); 3663 mutex_unlock(&dev->struct_mutex);
4417 return ret; 3664 return ret;
4418 } 3665 }
4419 } 3666 }
4420 3667
3668 i915_gem_reset_fences(dev);
3669
4421 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3670 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4422 * We need to replace this with a semaphore, or something. 3671 * We need to replace this with a semaphore, or something.
4423 * And not confound mm.suspended! 3672 * And not confound mm.suspended!
4424 */ 3673 */
4425 dev_priv->mm.suspended = 1; 3674 dev_priv->mm.suspended = 1;
4426 del_timer(&dev_priv->hangcheck_timer); 3675 del_timer_sync(&dev_priv->hangcheck_timer);
4427 3676
4428 i915_kernel_lost_context(dev); 3677 i915_kernel_lost_context(dev);
4429 i915_gem_cleanup_ringbuffer(dev); 3678 i915_gem_cleanup_ringbuffer(dev);
@@ -4436,108 +3685,36 @@ i915_gem_idle(struct drm_device *dev)
4436 return 0; 3685 return 0;
4437} 3686}
4438 3687
4439/*
4440 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4441 * over cache flushing.
4442 */
4443static int
4444i915_gem_init_pipe_control(struct drm_device *dev)
4445{
4446 drm_i915_private_t *dev_priv = dev->dev_private;
4447 struct drm_gem_object *obj;
4448 struct drm_i915_gem_object *obj_priv;
4449 int ret;
4450
4451 obj = i915_gem_alloc_object(dev, 4096);
4452 if (obj == NULL) {
4453 DRM_ERROR("Failed to allocate seqno page\n");
4454 ret = -ENOMEM;
4455 goto err;
4456 }
4457 obj_priv = to_intel_bo(obj);
4458 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4459
4460 ret = i915_gem_object_pin(obj, 4096);
4461 if (ret)
4462 goto err_unref;
4463
4464 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4465 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4466 if (dev_priv->seqno_page == NULL)
4467 goto err_unpin;
4468
4469 dev_priv->seqno_obj = obj;
4470 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4471
4472 return 0;
4473
4474err_unpin:
4475 i915_gem_object_unpin(obj);
4476err_unref:
4477 drm_gem_object_unreference(obj);
4478err:
4479 return ret;
4480}
4481
4482
4483static void
4484i915_gem_cleanup_pipe_control(struct drm_device *dev)
4485{
4486 drm_i915_private_t *dev_priv = dev->dev_private;
4487 struct drm_gem_object *obj;
4488 struct drm_i915_gem_object *obj_priv;
4489
4490 obj = dev_priv->seqno_obj;
4491 obj_priv = to_intel_bo(obj);
4492 kunmap(obj_priv->pages[0]);
4493 i915_gem_object_unpin(obj);
4494 drm_gem_object_unreference(obj);
4495 dev_priv->seqno_obj = NULL;
4496
4497 dev_priv->seqno_page = NULL;
4498}
4499
4500int 3688int
4501i915_gem_init_ringbuffer(struct drm_device *dev) 3689i915_gem_init_ringbuffer(struct drm_device *dev)
4502{ 3690{
4503 drm_i915_private_t *dev_priv = dev->dev_private; 3691 drm_i915_private_t *dev_priv = dev->dev_private;
4504 int ret; 3692 int ret;
4505 3693
4506 dev_priv->render_ring = render_ring; 3694 ret = intel_init_render_ring_buffer(dev);
4507
4508 if (!I915_NEED_GFX_HWS(dev)) {
4509 dev_priv->render_ring.status_page.page_addr
4510 = dev_priv->status_page_dmah->vaddr;
4511 memset(dev_priv->render_ring.status_page.page_addr,
4512 0, PAGE_SIZE);
4513 }
4514
4515 if (HAS_PIPE_CONTROL(dev)) {
4516 ret = i915_gem_init_pipe_control(dev);
4517 if (ret)
4518 return ret;
4519 }
4520
4521 ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4522 if (ret) 3695 if (ret)
4523 goto cleanup_pipe_control; 3696 return ret;
4524 3697
4525 if (HAS_BSD(dev)) { 3698 if (HAS_BSD(dev)) {
4526 dev_priv->bsd_ring = bsd_ring; 3699 ret = intel_init_bsd_ring_buffer(dev);
4527 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4528 if (ret) 3700 if (ret)
4529 goto cleanup_render_ring; 3701 goto cleanup_render_ring;
4530 } 3702 }
4531 3703
3704 if (HAS_BLT(dev)) {
3705 ret = intel_init_blt_ring_buffer(dev);
3706 if (ret)
3707 goto cleanup_bsd_ring;
3708 }
3709
4532 dev_priv->next_seqno = 1; 3710 dev_priv->next_seqno = 1;
4533 3711
4534 return 0; 3712 return 0;
4535 3713
3714cleanup_bsd_ring:
3715 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4536cleanup_render_ring: 3716cleanup_render_ring:
4537 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3717 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4538cleanup_pipe_control:
4539 if (HAS_PIPE_CONTROL(dev))
4540 i915_gem_cleanup_pipe_control(dev);
4541 return ret; 3718 return ret;
4542} 3719}
4543 3720
@@ -4545,12 +3722,10 @@ void
4545i915_gem_cleanup_ringbuffer(struct drm_device *dev) 3722i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4546{ 3723{
4547 drm_i915_private_t *dev_priv = dev->dev_private; 3724 drm_i915_private_t *dev_priv = dev->dev_private;
3725 int i;
4548 3726
4549 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3727 for (i = 0; i < I915_NUM_RINGS; i++)
4550 if (HAS_BSD(dev)) 3728 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
4551 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4552 if (HAS_PIPE_CONTROL(dev))
4553 i915_gem_cleanup_pipe_control(dev);
4554} 3729}
4555 3730
4556int 3731int
@@ -4558,7 +3733,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4558 struct drm_file *file_priv) 3733 struct drm_file *file_priv)
4559{ 3734{
4560 drm_i915_private_t *dev_priv = dev->dev_private; 3735 drm_i915_private_t *dev_priv = dev->dev_private;
4561 int ret; 3736 int ret, i;
4562 3737
4563 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3738 if (drm_core_check_feature(dev, DRIVER_MODESET))
4564 return 0; 3739 return 0;
@@ -4577,15 +3752,13 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4577 return ret; 3752 return ret;
4578 } 3753 }
4579 3754
4580 spin_lock(&dev_priv->mm.active_list_lock); 3755 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4581 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4582 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4583 spin_unlock(&dev_priv->mm.active_list_lock);
4584
4585 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 3756 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4586 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 3757 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4587 BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); 3758 for (i = 0; i < I915_NUM_RINGS; i++) {
4588 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list)); 3759 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3760 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3761 }
4589 mutex_unlock(&dev->struct_mutex); 3762 mutex_unlock(&dev->struct_mutex);
4590 3763
4591 ret = drm_irq_install(dev); 3764 ret = drm_irq_install(dev);
@@ -4627,31 +3800,34 @@ i915_gem_lastclose(struct drm_device *dev)
4627 DRM_ERROR("failed to idle hardware: %d\n", ret); 3800 DRM_ERROR("failed to idle hardware: %d\n", ret);
4628} 3801}
4629 3802
3803static void
3804init_ring_lists(struct intel_ring_buffer *ring)
3805{
3806 INIT_LIST_HEAD(&ring->active_list);
3807 INIT_LIST_HEAD(&ring->request_list);
3808 INIT_LIST_HEAD(&ring->gpu_write_list);
3809}
3810
4630void 3811void
4631i915_gem_load(struct drm_device *dev) 3812i915_gem_load(struct drm_device *dev)
4632{ 3813{
4633 int i; 3814 int i;
4634 drm_i915_private_t *dev_priv = dev->dev_private; 3815 drm_i915_private_t *dev_priv = dev->dev_private;
4635 3816
4636 spin_lock_init(&dev_priv->mm.active_list_lock); 3817 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4637 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 3818 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4638 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4639 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 3819 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3820 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4640 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3821 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4641 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 3822 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4642 INIT_LIST_HEAD(&dev_priv->render_ring.active_list); 3823 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
4643 INIT_LIST_HEAD(&dev_priv->render_ring.request_list); 3824 for (i = 0; i < I915_NUM_RINGS; i++)
4644 if (HAS_BSD(dev)) { 3825 init_ring_lists(&dev_priv->ring[i]);
4645 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4646 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4647 }
4648 for (i = 0; i < 16; i++) 3826 for (i = 0; i < 16; i++)
4649 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 3827 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4650 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 3828 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4651 i915_gem_retire_work_handler); 3829 i915_gem_retire_work_handler);
4652 spin_lock(&shrink_list_lock); 3830 init_completion(&dev_priv->error_completion);
4653 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4654 spin_unlock(&shrink_list_lock);
4655 3831
4656 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 3832 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4657 if (IS_GEN3(dev)) { 3833 if (IS_GEN3(dev)) {
@@ -4663,36 +3839,38 @@ i915_gem_load(struct drm_device *dev)
4663 } 3839 }
4664 } 3840 }
4665 3841
3842 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3843
4666 /* Old X drivers will take 0-2 for front, back, depth buffers */ 3844 /* Old X drivers will take 0-2 for front, back, depth buffers */
4667 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 3845 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4668 dev_priv->fence_reg_start = 3; 3846 dev_priv->fence_reg_start = 3;
4669 3847
4670 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 3848 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4671 dev_priv->num_fence_regs = 16; 3849 dev_priv->num_fence_regs = 16;
4672 else 3850 else
4673 dev_priv->num_fence_regs = 8; 3851 dev_priv->num_fence_regs = 8;
4674 3852
4675 /* Initialize fence registers to zero */ 3853 /* Initialize fence registers to zero */
4676 if (IS_I965G(dev)) { 3854 for (i = 0; i < dev_priv->num_fence_regs; i++) {
4677 for (i = 0; i < 16; i++) 3855 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
4678 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4679 } else {
4680 for (i = 0; i < 8; i++)
4681 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4682 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4683 for (i = 0; i < 8; i++)
4684 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4685 } 3856 }
3857
4686 i915_gem_detect_bit_6_swizzle(dev); 3858 i915_gem_detect_bit_6_swizzle(dev);
4687 init_waitqueue_head(&dev_priv->pending_flip_queue); 3859 init_waitqueue_head(&dev_priv->pending_flip_queue);
3860
3861 dev_priv->mm.interruptible = true;
3862
3863 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3864 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3865 register_shrinker(&dev_priv->mm.inactive_shrinker);
4688} 3866}
4689 3867
4690/* 3868/*
4691 * Create a physically contiguous memory object for this object 3869 * Create a physically contiguous memory object for this object
4692 * e.g. for cursor + overlay regs 3870 * e.g. for cursor + overlay regs
4693 */ 3871 */
4694int i915_gem_init_phys_object(struct drm_device *dev, 3872static int i915_gem_init_phys_object(struct drm_device *dev,
4695 int id, int size, int align) 3873 int id, int size, int align)
4696{ 3874{
4697 drm_i915_private_t *dev_priv = dev->dev_private; 3875 drm_i915_private_t *dev_priv = dev->dev_private;
4698 struct drm_i915_gem_phys_object *phys_obj; 3876 struct drm_i915_gem_phys_object *phys_obj;
@@ -4724,7 +3902,7 @@ kfree_obj:
4724 return ret; 3902 return ret;
4725} 3903}
4726 3904
4727void i915_gem_free_phys_object(struct drm_device *dev, int id) 3905static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4728{ 3906{
4729 drm_i915_private_t *dev_priv = dev->dev_private; 3907 drm_i915_private_t *dev_priv = dev->dev_private;
4730 struct drm_i915_gem_phys_object *phys_obj; 3908 struct drm_i915_gem_phys_object *phys_obj;
@@ -4754,47 +3932,46 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
4754} 3932}
4755 3933
4756void i915_gem_detach_phys_object(struct drm_device *dev, 3934void i915_gem_detach_phys_object(struct drm_device *dev,
4757 struct drm_gem_object *obj) 3935 struct drm_i915_gem_object *obj)
4758{ 3936{
4759 struct drm_i915_gem_object *obj_priv; 3937 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3938 char *vaddr;
4760 int i; 3939 int i;
4761 int ret;
4762 int page_count; 3940 int page_count;
4763 3941
4764 obj_priv = to_intel_bo(obj); 3942 if (!obj->phys_obj)
4765 if (!obj_priv->phys_obj)
4766 return; 3943 return;
3944 vaddr = obj->phys_obj->handle->vaddr;
4767 3945
4768 ret = i915_gem_object_get_pages(obj, 0); 3946 page_count = obj->base.size / PAGE_SIZE;
4769 if (ret)
4770 goto out;
4771
4772 page_count = obj->size / PAGE_SIZE;
4773
4774 for (i = 0; i < page_count; i++) { 3947 for (i = 0; i < page_count; i++) {
4775 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0); 3948 struct page *page = shmem_read_mapping_page(mapping, i);
4776 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3949 if (!IS_ERR(page)) {
3950 char *dst = kmap_atomic(page);
3951 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3952 kunmap_atomic(dst);
4777 3953
4778 memcpy(dst, src, PAGE_SIZE); 3954 drm_clflush_pages(&page, 1);
4779 kunmap_atomic(dst, KM_USER0); 3955
3956 set_page_dirty(page);
3957 mark_page_accessed(page);
3958 page_cache_release(page);
3959 }
4780 } 3960 }
4781 drm_clflush_pages(obj_priv->pages, page_count); 3961 intel_gtt_chipset_flush();
4782 drm_agp_chipset_flush(dev);
4783 3962
4784 i915_gem_object_put_pages(obj); 3963 obj->phys_obj->cur_obj = NULL;
4785out: 3964 obj->phys_obj = NULL;
4786 obj_priv->phys_obj->cur_obj = NULL;
4787 obj_priv->phys_obj = NULL;
4788} 3965}
4789 3966
4790int 3967int
4791i915_gem_attach_phys_object(struct drm_device *dev, 3968i915_gem_attach_phys_object(struct drm_device *dev,
4792 struct drm_gem_object *obj, 3969 struct drm_i915_gem_object *obj,
4793 int id, 3970 int id,
4794 int align) 3971 int align)
4795{ 3972{
3973 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4796 drm_i915_private_t *dev_priv = dev->dev_private; 3974 drm_i915_private_t *dev_priv = dev->dev_private;
4797 struct drm_i915_gem_object *obj_priv;
4798 int ret = 0; 3975 int ret = 0;
4799 int page_count; 3976 int page_count;
4800 int i; 3977 int i;
@@ -4802,10 +3979,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4802 if (id > I915_MAX_PHYS_OBJECT) 3979 if (id > I915_MAX_PHYS_OBJECT)
4803 return -EINVAL; 3980 return -EINVAL;
4804 3981
4805 obj_priv = to_intel_bo(obj); 3982 if (obj->phys_obj) {
4806 3983 if (obj->phys_obj->id == id)
4807 if (obj_priv->phys_obj) {
4808 if (obj_priv->phys_obj->id == id)
4809 return 0; 3984 return 0;
4810 i915_gem_detach_phys_object(dev, obj); 3985 i915_gem_detach_phys_object(dev, obj);
4811 } 3986 }
@@ -4813,74 +3988,86 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4813 /* create a new object */ 3988 /* create a new object */
4814 if (!dev_priv->mm.phys_objs[id - 1]) { 3989 if (!dev_priv->mm.phys_objs[id - 1]) {
4815 ret = i915_gem_init_phys_object(dev, id, 3990 ret = i915_gem_init_phys_object(dev, id,
4816 obj->size, align); 3991 obj->base.size, align);
4817 if (ret) { 3992 if (ret) {
4818 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 3993 DRM_ERROR("failed to init phys object %d size: %zu\n",
4819 goto out; 3994 id, obj->base.size);
3995 return ret;
4820 } 3996 }
4821 } 3997 }
4822 3998
4823 /* bind to the object */ 3999 /* bind to the object */
4824 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; 4000 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4825 obj_priv->phys_obj->cur_obj = obj; 4001 obj->phys_obj->cur_obj = obj;
4826
4827 ret = i915_gem_object_get_pages(obj, 0);
4828 if (ret) {
4829 DRM_ERROR("failed to get page list\n");
4830 goto out;
4831 }
4832 4002
4833 page_count = obj->size / PAGE_SIZE; 4003 page_count = obj->base.size / PAGE_SIZE;
4834 4004
4835 for (i = 0; i < page_count; i++) { 4005 for (i = 0; i < page_count; i++) {
4836 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0); 4006 struct page *page;
4837 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 4007 char *dst, *src;
4838 4008
4009 page = shmem_read_mapping_page(mapping, i);
4010 if (IS_ERR(page))
4011 return PTR_ERR(page);
4012
4013 src = kmap_atomic(page);
4014 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4839 memcpy(dst, src, PAGE_SIZE); 4015 memcpy(dst, src, PAGE_SIZE);
4840 kunmap_atomic(src, KM_USER0); 4016 kunmap_atomic(src);
4841 }
4842 4017
4843 i915_gem_object_put_pages(obj); 4018 mark_page_accessed(page);
4019 page_cache_release(page);
4020 }
4844 4021
4845 return 0; 4022 return 0;
4846out:
4847 return ret;
4848} 4023}
4849 4024
4850static int 4025static int
4851i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 4026i915_gem_phys_pwrite(struct drm_device *dev,
4027 struct drm_i915_gem_object *obj,
4852 struct drm_i915_gem_pwrite *args, 4028 struct drm_i915_gem_pwrite *args,
4853 struct drm_file *file_priv) 4029 struct drm_file *file_priv)
4854{ 4030{
4855 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 4031 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4856 void *obj_addr; 4032 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
4857 int ret;
4858 char __user *user_data;
4859 4033
4860 user_data = (char __user *) (uintptr_t) args->data_ptr; 4034 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4861 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset; 4035 unsigned long unwritten;
4862 4036
4863 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size); 4037 /* The physical object once assigned is fixed for the lifetime
4864 ret = copy_from_user(obj_addr, user_data, args->size); 4038 * of the obj, so we can safely drop the lock and continue
4865 if (ret) 4039 * to access vaddr.
4866 return -EFAULT; 4040 */
4041 mutex_unlock(&dev->struct_mutex);
4042 unwritten = copy_from_user(vaddr, user_data, args->size);
4043 mutex_lock(&dev->struct_mutex);
4044 if (unwritten)
4045 return -EFAULT;
4046 }
4867 4047
4868 drm_agp_chipset_flush(dev); 4048 intel_gtt_chipset_flush();
4869 return 0; 4049 return 0;
4870} 4050}
4871 4051
4872void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv) 4052void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4873{ 4053{
4874 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv; 4054 struct drm_i915_file_private *file_priv = file->driver_priv;
4875 4055
4876 /* Clean up our request list when the client is going away, so that 4056 /* Clean up our request list when the client is going away, so that
4877 * later retire_requests won't dereference our soon-to-be-gone 4057 * later retire_requests won't dereference our soon-to-be-gone
4878 * file_priv. 4058 * file_priv.
4879 */ 4059 */
4880 mutex_lock(&dev->struct_mutex); 4060 spin_lock(&file_priv->mm.lock);
4881 while (!list_empty(&i915_file_priv->mm.request_list)) 4061 while (!list_empty(&file_priv->mm.request_list)) {
4882 list_del_init(i915_file_priv->mm.request_list.next); 4062 struct drm_i915_gem_request *request;
4883 mutex_unlock(&dev->struct_mutex); 4063
4064 request = list_first_entry(&file_priv->mm.request_list,
4065 struct drm_i915_gem_request,
4066 client_list);
4067 list_del(&request->client_list);
4068 request->file_priv = NULL;
4069 }
4070 spin_unlock(&file_priv->mm.lock);
4884} 4071}
4885 4072
4886static int 4073static int
@@ -4889,155 +4076,74 @@ i915_gpu_is_active(struct drm_device *dev)
4889 drm_i915_private_t *dev_priv = dev->dev_private; 4076 drm_i915_private_t *dev_priv = dev->dev_private;
4890 int lists_empty; 4077 int lists_empty;
4891 4078
4892 spin_lock(&dev_priv->mm.active_list_lock);
4893 lists_empty = list_empty(&dev_priv->mm.flushing_list) && 4079 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4894 list_empty(&dev_priv->render_ring.active_list); 4080 list_empty(&dev_priv->mm.active_list);
4895 if (HAS_BSD(dev))
4896 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4897 spin_unlock(&dev_priv->mm.active_list_lock);
4898 4081
4899 return !lists_empty; 4082 return !lists_empty;
4900} 4083}
4901 4084
4902static int 4085static int
4903i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) 4086i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4904{ 4087{
4905 drm_i915_private_t *dev_priv, *next_dev; 4088 struct drm_i915_private *dev_priv =
4906 struct drm_i915_gem_object *obj_priv, *next_obj; 4089 container_of(shrinker,
4907 int cnt = 0; 4090 struct drm_i915_private,
4908 int would_deadlock = 1; 4091 mm.inactive_shrinker);
4092 struct drm_device *dev = dev_priv->dev;
4093 struct drm_i915_gem_object *obj, *next;
4094 int nr_to_scan = sc->nr_to_scan;
4095 int cnt;
4096
4097 if (!mutex_trylock(&dev->struct_mutex))
4098 return 0;
4909 4099
4910 /* "fast-path" to count number of available objects */ 4100 /* "fast-path" to count number of available objects */
4911 if (nr_to_scan == 0) { 4101 if (nr_to_scan == 0) {
4912 spin_lock(&shrink_list_lock); 4102 cnt = 0;
4913 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4103 list_for_each_entry(obj,
4914 struct drm_device *dev = dev_priv->dev; 4104 &dev_priv->mm.inactive_list,
4915 4105 mm_list)
4916 if (mutex_trylock(&dev->struct_mutex)) { 4106 cnt++;
4917 list_for_each_entry(obj_priv, 4107 mutex_unlock(&dev->struct_mutex);
4918 &dev_priv->mm.inactive_list, 4108 return cnt / 100 * sysctl_vfs_cache_pressure;
4919 list)
4920 cnt++;
4921 mutex_unlock(&dev->struct_mutex);
4922 }
4923 }
4924 spin_unlock(&shrink_list_lock);
4925
4926 return (cnt / 100) * sysctl_vfs_cache_pressure;
4927 } 4109 }
4928 4110
4929 spin_lock(&shrink_list_lock);
4930
4931rescan: 4111rescan:
4932 /* first scan for clean buffers */ 4112 /* first scan for clean buffers */
4933 list_for_each_entry_safe(dev_priv, next_dev, 4113 i915_gem_retire_requests(dev);
4934 &shrink_list, mm.shrink_list) {
4935 struct drm_device *dev = dev_priv->dev;
4936
4937 if (! mutex_trylock(&dev->struct_mutex))
4938 continue;
4939
4940 spin_unlock(&shrink_list_lock);
4941 i915_gem_retire_requests(dev);
4942 4114
4943 list_for_each_entry_safe(obj_priv, next_obj, 4115 list_for_each_entry_safe(obj, next,
4944 &dev_priv->mm.inactive_list, 4116 &dev_priv->mm.inactive_list,
4945 list) { 4117 mm_list) {
4946 if (i915_gem_object_is_purgeable(obj_priv)) { 4118 if (i915_gem_object_is_purgeable(obj)) {
4947 i915_gem_object_unbind(&obj_priv->base); 4119 if (i915_gem_object_unbind(obj) == 0 &&
4948 if (--nr_to_scan <= 0) 4120 --nr_to_scan == 0)
4949 break; 4121 break;
4950 }
4951 } 4122 }
4952
4953 spin_lock(&shrink_list_lock);
4954 mutex_unlock(&dev->struct_mutex);
4955
4956 would_deadlock = 0;
4957
4958 if (nr_to_scan <= 0)
4959 break;
4960 } 4123 }
4961 4124
4962 /* second pass, evict/count anything still on the inactive list */ 4125 /* second pass, evict/count anything still on the inactive list */
4963 list_for_each_entry_safe(dev_priv, next_dev, 4126 cnt = 0;
4964 &shrink_list, mm.shrink_list) { 4127 list_for_each_entry_safe(obj, next,
4965 struct drm_device *dev = dev_priv->dev; 4128 &dev_priv->mm.inactive_list,
4966 4129 mm_list) {
4967 if (! mutex_trylock(&dev->struct_mutex)) 4130 if (nr_to_scan &&
4968 continue; 4131 i915_gem_object_unbind(obj) == 0)
4969 4132 nr_to_scan--;
4970 spin_unlock(&shrink_list_lock); 4133 else
4971 4134 cnt++;
4972 list_for_each_entry_safe(obj_priv, next_obj,
4973 &dev_priv->mm.inactive_list,
4974 list) {
4975 if (nr_to_scan > 0) {
4976 i915_gem_object_unbind(&obj_priv->base);
4977 nr_to_scan--;
4978 } else
4979 cnt++;
4980 }
4981
4982 spin_lock(&shrink_list_lock);
4983 mutex_unlock(&dev->struct_mutex);
4984
4985 would_deadlock = 0;
4986 } 4135 }
4987 4136
4988 if (nr_to_scan) { 4137 if (nr_to_scan && i915_gpu_is_active(dev)) {
4989 int active = 0;
4990
4991 /* 4138 /*
4992 * We are desperate for pages, so as a last resort, wait 4139 * We are desperate for pages, so as a last resort, wait
4993 * for the GPU to finish and discard whatever we can. 4140 * for the GPU to finish and discard whatever we can.
4994 * This has a dramatic impact to reduce the number of 4141 * This has a dramatic impact to reduce the number of
4995 * OOM-killer events whilst running the GPU aggressively. 4142 * OOM-killer events whilst running the GPU aggressively.
4996 */ 4143 */
4997 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4144 if (i915_gpu_idle(dev) == 0)
4998 struct drm_device *dev = dev_priv->dev;
4999
5000 if (!mutex_trylock(&dev->struct_mutex))
5001 continue;
5002
5003 spin_unlock(&shrink_list_lock);
5004
5005 if (i915_gpu_is_active(dev)) {
5006 i915_gpu_idle(dev);
5007 active++;
5008 }
5009
5010 spin_lock(&shrink_list_lock);
5011 mutex_unlock(&dev->struct_mutex);
5012 }
5013
5014 if (active)
5015 goto rescan; 4145 goto rescan;
5016 } 4146 }
5017 4147 mutex_unlock(&dev->struct_mutex);
5018 spin_unlock(&shrink_list_lock); 4148 return cnt / 100 * sysctl_vfs_cache_pressure;
5019
5020 if (would_deadlock)
5021 return -1;
5022 else if (cnt > 0)
5023 return (cnt / 100) * sysctl_vfs_cache_pressure;
5024 else
5025 return 0;
5026}
5027
5028static struct shrinker shrinker = {
5029 .shrink = i915_gem_shrink,
5030 .seeks = DEFAULT_SEEKS,
5031};
5032
5033__init void
5034i915_gem_shrinker_init(void)
5035{
5036 register_shrinker(&shrinker);
5037}
5038
5039__exit void
5040i915_gem_shrinker_exit(void)
5041{
5042 unregister_shrinker(&shrinker);
5043} 4149}
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-20 18:15:14 -0400