aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c3645
1 files changed, 1312 insertions, 2333 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 17b1cba3b5f1..c79c0b62ef60 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -34,39 +34,31 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/swap.h> 35#include <linux/swap.h>
36#include <linux/pci.h> 36#include <linux/pci.h>
37#include <linux/intel-gtt.h>
38 37
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); 38static void i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40 39static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, 40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42 bool pipelined); 41static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 42 bool write);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 43static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset, 44 uint64_t offset,
49 uint64_t size); 45 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); 46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj, 47static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
52 bool interruptible); 48 unsigned alignment,
53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, 49 bool map_and_fenceable);
54 unsigned alignment); 50static void i915_gem_clear_fence_reg(struct drm_device *dev,
55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); 51 struct drm_i915_fence_reg *reg);
56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 52static int i915_gem_phys_pwrite(struct drm_device *dev,
53 struct drm_i915_gem_object *obj,
57 struct drm_i915_gem_pwrite *args, 54 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv); 55 struct drm_file *file);
59static void i915_gem_free_object_tail(struct drm_gem_object *obj); 56static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
60 57
61static int 58static int i915_gem_inactive_shrink(struct shrinker *shrinker,
62i915_gem_object_get_pages(struct drm_gem_object *obj, 59 int nr_to_scan,
63 gfp_t gfpmask); 60 gfp_t gfp_mask);
64 61
65static void
66i915_gem_object_put_pages(struct drm_gem_object *obj);
67
68static LIST_HEAD(shrink_list);
69static DEFINE_SPINLOCK(shrink_list_lock);
70 62
71/* some bookkeeping */ 63/* some bookkeeping */
72static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 64static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
@@ -83,34 +75,6 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
83 dev_priv->mm.object_memory -= size; 75 dev_priv->mm.object_memory -= size;
84} 76}
85 77
86static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
87 size_t size)
88{
89 dev_priv->mm.gtt_count++;
90 dev_priv->mm.gtt_memory += size;
91}
92
93static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
94 size_t size)
95{
96 dev_priv->mm.gtt_count--;
97 dev_priv->mm.gtt_memory -= size;
98}
99
100static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
101 size_t size)
102{
103 dev_priv->mm.pin_count++;
104 dev_priv->mm.pin_memory += size;
105}
106
107static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
108 size_t size)
109{
110 dev_priv->mm.pin_count--;
111 dev_priv->mm.pin_memory -= size;
112}
113
114int 78int
115i915_gem_check_is_wedged(struct drm_device *dev) 79i915_gem_check_is_wedged(struct drm_device *dev)
116{ 80{
@@ -141,7 +105,7 @@ i915_gem_check_is_wedged(struct drm_device *dev)
141 return -EIO; 105 return -EIO;
142} 106}
143 107
144static int i915_mutex_lock_interruptible(struct drm_device *dev) 108int i915_mutex_lock_interruptible(struct drm_device *dev)
145{ 109{
146 struct drm_i915_private *dev_priv = dev->dev_private; 110 struct drm_i915_private *dev_priv = dev->dev_private;
147 int ret; 111 int ret;
@@ -164,75 +128,76 @@ static int i915_mutex_lock_interruptible(struct drm_device *dev)
164} 128}
165 129
166static inline bool 130static inline bool
167i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) 131i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
168{ 132{
169 return obj_priv->gtt_space && 133 return obj->gtt_space && !obj->active && obj->pin_count == 0;
170 !obj_priv->active &&
171 obj_priv->pin_count == 0;
172} 134}
173 135
174int i915_gem_do_init(struct drm_device *dev, 136void i915_gem_do_init(struct drm_device *dev,
175 unsigned long start, 137 unsigned long start,
176 unsigned long end) 138 unsigned long mappable_end,
139 unsigned long end)
177{ 140{
178 drm_i915_private_t *dev_priv = dev->dev_private; 141 drm_i915_private_t *dev_priv = dev->dev_private;
179 142
180 if (start >= end ||
181 (start & (PAGE_SIZE - 1)) != 0 ||
182 (end & (PAGE_SIZE - 1)) != 0) {
183 return -EINVAL;
184 }
185
186 drm_mm_init(&dev_priv->mm.gtt_space, start, 143 drm_mm_init(&dev_priv->mm.gtt_space, start,
187 end - start); 144 end - start);
188 145
189 dev_priv->mm.gtt_total = end - start; 146 dev_priv->mm.gtt_total = end - start;
190 147 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
191 return 0; 148 dev_priv->mm.gtt_mappable_end = mappable_end;
192} 149}
193 150
194int 151int
195i915_gem_init_ioctl(struct drm_device *dev, void *data, 152i915_gem_init_ioctl(struct drm_device *dev, void *data,
196 struct drm_file *file_priv) 153 struct drm_file *file)
197{ 154{
198 struct drm_i915_gem_init *args = data; 155 struct drm_i915_gem_init *args = data;
199 int ret; 156
157 if (args->gtt_start >= args->gtt_end ||
158 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
159 return -EINVAL;
200 160
201 mutex_lock(&dev->struct_mutex); 161 mutex_lock(&dev->struct_mutex);
202 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); 162 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
203 mutex_unlock(&dev->struct_mutex); 163 mutex_unlock(&dev->struct_mutex);
204 164
205 return ret; 165 return 0;
206} 166}
207 167
208int 168int
209i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 169i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
210 struct drm_file *file_priv) 170 struct drm_file *file)
211{ 171{
212 struct drm_i915_private *dev_priv = dev->dev_private; 172 struct drm_i915_private *dev_priv = dev->dev_private;
213 struct drm_i915_gem_get_aperture *args = data; 173 struct drm_i915_gem_get_aperture *args = data;
174 struct drm_i915_gem_object *obj;
175 size_t pinned;
214 176
215 if (!(dev->driver->driver_features & DRIVER_GEM)) 177 if (!(dev->driver->driver_features & DRIVER_GEM))
216 return -ENODEV; 178 return -ENODEV;
217 179
180 pinned = 0;
218 mutex_lock(&dev->struct_mutex); 181 mutex_lock(&dev->struct_mutex);
219 args->aper_size = dev_priv->mm.gtt_total; 182 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
220 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory; 183 pinned += obj->gtt_space->size;
221 mutex_unlock(&dev->struct_mutex); 184 mutex_unlock(&dev->struct_mutex);
222 185
186 args->aper_size = dev_priv->mm.gtt_total;
187 args->aper_available_size = args->aper_size -pinned;
188
223 return 0; 189 return 0;
224} 190}
225 191
226
227/** 192/**
228 * Creates a new mm object and returns a handle to it. 193 * Creates a new mm object and returns a handle to it.
229 */ 194 */
230int 195int
231i915_gem_create_ioctl(struct drm_device *dev, void *data, 196i915_gem_create_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file_priv) 197 struct drm_file *file)
233{ 198{
234 struct drm_i915_gem_create *args = data; 199 struct drm_i915_gem_create *args = data;
235 struct drm_gem_object *obj; 200 struct drm_i915_gem_object *obj;
236 int ret; 201 int ret;
237 u32 handle; 202 u32 handle;
238 203
@@ -243,45 +208,28 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
243 if (obj == NULL) 208 if (obj == NULL)
244 return -ENOMEM; 209 return -ENOMEM;
245 210
246 ret = drm_gem_handle_create(file_priv, obj, &handle); 211 ret = drm_gem_handle_create(file, &obj->base, &handle);
247 if (ret) { 212 if (ret) {
248 drm_gem_object_release(obj); 213 drm_gem_object_release(&obj->base);
249 i915_gem_info_remove_obj(dev->dev_private, obj->size); 214 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
250 kfree(obj); 215 kfree(obj);
251 return ret; 216 return ret;
252 } 217 }
253 218
254 /* drop reference from allocate - handle holds it now */ 219 /* drop reference from allocate - handle holds it now */
255 drm_gem_object_unreference(obj); 220 drm_gem_object_unreference(&obj->base);
256 trace_i915_gem_object_create(obj); 221 trace_i915_gem_object_create(obj);
257 222
258 args->handle = handle; 223 args->handle = handle;
259 return 0; 224 return 0;
260} 225}
261 226
262static inline int 227static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
263fast_shmem_read(struct page **pages,
264 loff_t page_base, int page_offset,
265 char __user *data,
266 int length)
267{ 228{
268 char *vaddr; 229 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
269 int ret;
270
271 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
272 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
273 kunmap_atomic(vaddr);
274
275 return ret;
276}
277
278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
279{
280 drm_i915_private_t *dev_priv = obj->dev->dev_private;
281 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
282 230
283 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 231 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
284 obj_priv->tiling_mode != I915_TILING_NONE; 232 obj->tiling_mode != I915_TILING_NONE;
285} 233}
286 234
287static inline void 235static inline void
@@ -357,38 +305,51 @@ slow_shmem_bit17_copy(struct page *gpu_page,
357 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). 305 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
358 */ 306 */
359static int 307static int
360i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, 308i915_gem_shmem_pread_fast(struct drm_device *dev,
309 struct drm_i915_gem_object *obj,
361 struct drm_i915_gem_pread *args, 310 struct drm_i915_gem_pread *args,
362 struct drm_file *file_priv) 311 struct drm_file *file)
363{ 312{
364 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 313 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
365 ssize_t remain; 314 ssize_t remain;
366 loff_t offset, page_base; 315 loff_t offset;
367 char __user *user_data; 316 char __user *user_data;
368 int page_offset, page_length; 317 int page_offset, page_length;
369 318
370 user_data = (char __user *) (uintptr_t) args->data_ptr; 319 user_data = (char __user *) (uintptr_t) args->data_ptr;
371 remain = args->size; 320 remain = args->size;
372 321
373 obj_priv = to_intel_bo(obj);
374 offset = args->offset; 322 offset = args->offset;
375 323
376 while (remain > 0) { 324 while (remain > 0) {
325 struct page *page;
326 char *vaddr;
327 int ret;
328
377 /* Operation in this page 329 /* Operation in this page
378 * 330 *
379 * page_base = page offset within aperture
380 * page_offset = offset within page 331 * page_offset = offset within page
381 * page_length = bytes to copy for this page 332 * page_length = bytes to copy for this page
382 */ 333 */
383 page_base = (offset & ~(PAGE_SIZE-1));
384 page_offset = offset & (PAGE_SIZE-1); 334 page_offset = offset & (PAGE_SIZE-1);
385 page_length = remain; 335 page_length = remain;
386 if ((page_offset + remain) > PAGE_SIZE) 336 if ((page_offset + remain) > PAGE_SIZE)
387 page_length = PAGE_SIZE - page_offset; 337 page_length = PAGE_SIZE - page_offset;
388 338
389 if (fast_shmem_read(obj_priv->pages, 339 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
390 page_base, page_offset, 340 GFP_HIGHUSER | __GFP_RECLAIMABLE);
391 user_data, page_length)) 341 if (IS_ERR(page))
342 return PTR_ERR(page);
343
344 vaddr = kmap_atomic(page);
345 ret = __copy_to_user_inatomic(user_data,
346 vaddr + page_offset,
347 page_length);
348 kunmap_atomic(vaddr);
349
350 mark_page_accessed(page);
351 page_cache_release(page);
352 if (ret)
392 return -EFAULT; 353 return -EFAULT;
393 354
394 remain -= page_length; 355 remain -= page_length;
@@ -399,30 +360,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
399 return 0; 360 return 0;
400} 361}
401 362
402static int
403i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
404{
405 int ret;
406
407 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
408
409 /* If we've insufficient memory to map in the pages, attempt
410 * to make some space by throwing out some old buffers.
411 */
412 if (ret == -ENOMEM) {
413 struct drm_device *dev = obj->dev;
414
415 ret = i915_gem_evict_something(dev, obj->size,
416 i915_gem_get_gtt_alignment(obj));
417 if (ret)
418 return ret;
419
420 ret = i915_gem_object_get_pages(obj, 0);
421 }
422
423 return ret;
424}
425
426/** 363/**
427 * This is the fallback shmem pread path, which allocates temporary storage 364 * This is the fallback shmem pread path, which allocates temporary storage
428 * in kernel space to copy_to_user into outside of the struct_mutex, so we 365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
@@ -430,18 +367,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
430 * and not take page faults. 367 * and not take page faults.
431 */ 368 */
432static int 369static int
433i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, 370i915_gem_shmem_pread_slow(struct drm_device *dev,
371 struct drm_i915_gem_object *obj,
434 struct drm_i915_gem_pread *args, 372 struct drm_i915_gem_pread *args,
435 struct drm_file *file_priv) 373 struct drm_file *file)
436{ 374{
437 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 375 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
438 struct mm_struct *mm = current->mm; 376 struct mm_struct *mm = current->mm;
439 struct page **user_pages; 377 struct page **user_pages;
440 ssize_t remain; 378 ssize_t remain;
441 loff_t offset, pinned_pages, i; 379 loff_t offset, pinned_pages, i;
442 loff_t first_data_page, last_data_page, num_pages; 380 loff_t first_data_page, last_data_page, num_pages;
443 int shmem_page_index, shmem_page_offset; 381 int shmem_page_offset;
444 int data_page_index, data_page_offset; 382 int data_page_index, data_page_offset;
445 int page_length; 383 int page_length;
446 int ret; 384 int ret;
447 uint64_t data_ptr = args->data_ptr; 385 uint64_t data_ptr = args->data_ptr;
@@ -480,19 +418,18 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
480 418
481 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 419 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
482 420
483 obj_priv = to_intel_bo(obj);
484 offset = args->offset; 421 offset = args->offset;
485 422
486 while (remain > 0) { 423 while (remain > 0) {
424 struct page *page;
425
487 /* Operation in this page 426 /* Operation in this page
488 * 427 *
489 * shmem_page_index = page number within shmem file
490 * shmem_page_offset = offset within page in shmem file 428 * shmem_page_offset = offset within page in shmem file
491 * data_page_index = page number in get_user_pages return 429 * data_page_index = page number in get_user_pages return
492 * data_page_offset = offset with data_page_index page. 430 * data_page_offset = offset with data_page_index page.
493 * page_length = bytes to copy for this page 431 * page_length = bytes to copy for this page
494 */ 432 */
495 shmem_page_index = offset / PAGE_SIZE;
496 shmem_page_offset = offset & ~PAGE_MASK; 433 shmem_page_offset = offset & ~PAGE_MASK;
497 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 434 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
498 data_page_offset = data_ptr & ~PAGE_MASK; 435 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -503,8 +440,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
503 if ((data_page_offset + page_length) > PAGE_SIZE) 440 if ((data_page_offset + page_length) > PAGE_SIZE)
504 page_length = PAGE_SIZE - data_page_offset; 441 page_length = PAGE_SIZE - data_page_offset;
505 442
443 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
444 GFP_HIGHUSER | __GFP_RECLAIMABLE);
445 if (IS_ERR(page))
446 return PTR_ERR(page);
447
506 if (do_bit17_swizzling) { 448 if (do_bit17_swizzling) {
507 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 449 slow_shmem_bit17_copy(page,
508 shmem_page_offset, 450 shmem_page_offset,
509 user_pages[data_page_index], 451 user_pages[data_page_index],
510 data_page_offset, 452 data_page_offset,
@@ -513,11 +455,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
513 } else { 455 } else {
514 slow_shmem_copy(user_pages[data_page_index], 456 slow_shmem_copy(user_pages[data_page_index],
515 data_page_offset, 457 data_page_offset,
516 obj_priv->pages[shmem_page_index], 458 page,
517 shmem_page_offset, 459 shmem_page_offset,
518 page_length); 460 page_length);
519 } 461 }
520 462
463 mark_page_accessed(page);
464 page_cache_release(page);
465
521 remain -= page_length; 466 remain -= page_length;
522 data_ptr += page_length; 467 data_ptr += page_length;
523 offset += page_length; 468 offset += page_length;
@@ -526,6 +471,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
526out: 471out:
527 for (i = 0; i < pinned_pages; i++) { 472 for (i = 0; i < pinned_pages; i++) {
528 SetPageDirty(user_pages[i]); 473 SetPageDirty(user_pages[i]);
474 mark_page_accessed(user_pages[i]);
529 page_cache_release(user_pages[i]); 475 page_cache_release(user_pages[i]);
530 } 476 }
531 drm_free_large(user_pages); 477 drm_free_large(user_pages);
@@ -540,11 +486,10 @@ out:
540 */ 486 */
541int 487int
542i915_gem_pread_ioctl(struct drm_device *dev, void *data, 488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
543 struct drm_file *file_priv) 489 struct drm_file *file)
544{ 490{
545 struct drm_i915_gem_pread *args = data; 491 struct drm_i915_gem_pread *args = data;
546 struct drm_gem_object *obj; 492 struct drm_i915_gem_object *obj;
547 struct drm_i915_gem_object *obj_priv;
548 int ret = 0; 493 int ret = 0;
549 494
550 if (args->size == 0) 495 if (args->size == 0)
@@ -564,39 +509,33 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
564 if (ret) 509 if (ret)
565 return ret; 510 return ret;
566 511
567 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 512 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
568 if (obj == NULL) { 513 if (obj == NULL) {
569 ret = -ENOENT; 514 ret = -ENOENT;
570 goto unlock; 515 goto unlock;
571 } 516 }
572 obj_priv = to_intel_bo(obj);
573 517
574 /* Bounds check source. */ 518 /* Bounds check source. */
575 if (args->offset > obj->size || args->size > obj->size - args->offset) { 519 if (args->offset > obj->base.size ||
520 args->size > obj->base.size - args->offset) {
576 ret = -EINVAL; 521 ret = -EINVAL;
577 goto out; 522 goto out;
578 } 523 }
579 524
580 ret = i915_gem_object_get_pages_or_evict(obj);
581 if (ret)
582 goto out;
583
584 ret = i915_gem_object_set_cpu_read_domain_range(obj, 525 ret = i915_gem_object_set_cpu_read_domain_range(obj,
585 args->offset, 526 args->offset,
586 args->size); 527 args->size);
587 if (ret) 528 if (ret)
588 goto out_put; 529 goto out;
589 530
590 ret = -EFAULT; 531 ret = -EFAULT;
591 if (!i915_gem_object_needs_bit17_swizzle(obj)) 532 if (!i915_gem_object_needs_bit17_swizzle(obj))
592 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 533 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
593 if (ret == -EFAULT) 534 if (ret == -EFAULT)
594 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 535 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
595 536
596out_put:
597 i915_gem_object_put_pages(obj);
598out: 537out:
599 drm_gem_object_unreference(obj); 538 drm_gem_object_unreference(&obj->base);
600unlock: 539unlock:
601 mutex_unlock(&dev->struct_mutex); 540 mutex_unlock(&dev->struct_mutex);
602 return ret; 541 return ret;
@@ -646,32 +585,16 @@ slow_kernel_write(struct io_mapping *mapping,
646 io_mapping_unmap(dst_vaddr); 585 io_mapping_unmap(dst_vaddr);
647} 586}
648 587
649static inline int
650fast_shmem_write(struct page **pages,
651 loff_t page_base, int page_offset,
652 char __user *data,
653 int length)
654{
655 char *vaddr;
656 int ret;
657
658 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
659 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
660 kunmap_atomic(vaddr);
661
662 return ret;
663}
664
665/** 588/**
666 * This is the fast pwrite path, where we copy the data directly from the 589 * This is the fast pwrite path, where we copy the data directly from the
667 * user into the GTT, uncached. 590 * user into the GTT, uncached.
668 */ 591 */
669static int 592static int
670i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 593i915_gem_gtt_pwrite_fast(struct drm_device *dev,
594 struct drm_i915_gem_object *obj,
671 struct drm_i915_gem_pwrite *args, 595 struct drm_i915_gem_pwrite *args,
672 struct drm_file *file_priv) 596 struct drm_file *file)
673{ 597{
674 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
675 drm_i915_private_t *dev_priv = dev->dev_private; 598 drm_i915_private_t *dev_priv = dev->dev_private;
676 ssize_t remain; 599 ssize_t remain;
677 loff_t offset, page_base; 600 loff_t offset, page_base;
@@ -681,8 +604,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
681 user_data = (char __user *) (uintptr_t) args->data_ptr; 604 user_data = (char __user *) (uintptr_t) args->data_ptr;
682 remain = args->size; 605 remain = args->size;
683 606
684 obj_priv = to_intel_bo(obj); 607 offset = obj->gtt_offset + args->offset;
685 offset = obj_priv->gtt_offset + args->offset;
686 608
687 while (remain > 0) { 609 while (remain > 0) {
688 /* Operation in this page 610 /* Operation in this page
@@ -722,11 +644,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
722 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). 644 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
723 */ 645 */
724static int 646static int
725i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 647i915_gem_gtt_pwrite_slow(struct drm_device *dev,
648 struct drm_i915_gem_object *obj,
726 struct drm_i915_gem_pwrite *args, 649 struct drm_i915_gem_pwrite *args,
727 struct drm_file *file_priv) 650 struct drm_file *file)
728{ 651{
729 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
730 drm_i915_private_t *dev_priv = dev->dev_private; 652 drm_i915_private_t *dev_priv = dev->dev_private;
731 ssize_t remain; 653 ssize_t remain;
732 loff_t gtt_page_base, offset; 654 loff_t gtt_page_base, offset;
@@ -763,12 +685,15 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
763 goto out_unpin_pages; 685 goto out_unpin_pages;
764 } 686 }
765 687
766 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 688 ret = i915_gem_object_set_to_gtt_domain(obj, true);
689 if (ret)
690 goto out_unpin_pages;
691
692 ret = i915_gem_object_put_fence(obj);
767 if (ret) 693 if (ret)
768 goto out_unpin_pages; 694 goto out_unpin_pages;
769 695
770 obj_priv = to_intel_bo(obj); 696 offset = obj->gtt_offset + args->offset;
771 offset = obj_priv->gtt_offset + args->offset;
772 697
773 while (remain > 0) { 698 while (remain > 0) {
774 /* Operation in this page 699 /* Operation in this page
@@ -814,39 +739,58 @@ out_unpin_pages:
814 * copy_from_user into the kmapped pages backing the object. 739 * copy_from_user into the kmapped pages backing the object.
815 */ 740 */
816static int 741static int
817i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, 742i915_gem_shmem_pwrite_fast(struct drm_device *dev,
743 struct drm_i915_gem_object *obj,
818 struct drm_i915_gem_pwrite *args, 744 struct drm_i915_gem_pwrite *args,
819 struct drm_file *file_priv) 745 struct drm_file *file)
820{ 746{
821 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 747 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
822 ssize_t remain; 748 ssize_t remain;
823 loff_t offset, page_base; 749 loff_t offset;
824 char __user *user_data; 750 char __user *user_data;
825 int page_offset, page_length; 751 int page_offset, page_length;
826 752
827 user_data = (char __user *) (uintptr_t) args->data_ptr; 753 user_data = (char __user *) (uintptr_t) args->data_ptr;
828 remain = args->size; 754 remain = args->size;
829 755
830 obj_priv = to_intel_bo(obj);
831 offset = args->offset; 756 offset = args->offset;
832 obj_priv->dirty = 1; 757 obj->dirty = 1;
833 758
834 while (remain > 0) { 759 while (remain > 0) {
760 struct page *page;
761 char *vaddr;
762 int ret;
763
835 /* Operation in this page 764 /* Operation in this page
836 * 765 *
837 * page_base = page offset within aperture
838 * page_offset = offset within page 766 * page_offset = offset within page
839 * page_length = bytes to copy for this page 767 * page_length = bytes to copy for this page
840 */ 768 */
841 page_base = (offset & ~(PAGE_SIZE-1));
842 page_offset = offset & (PAGE_SIZE-1); 769 page_offset = offset & (PAGE_SIZE-1);
843 page_length = remain; 770 page_length = remain;
844 if ((page_offset + remain) > PAGE_SIZE) 771 if ((page_offset + remain) > PAGE_SIZE)
845 page_length = PAGE_SIZE - page_offset; 772 page_length = PAGE_SIZE - page_offset;
846 773
847 if (fast_shmem_write(obj_priv->pages, 774 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
848 page_base, page_offset, 775 GFP_HIGHUSER | __GFP_RECLAIMABLE);
849 user_data, page_length)) 776 if (IS_ERR(page))
777 return PTR_ERR(page);
778
779 vaddr = kmap_atomic(page, KM_USER0);
780 ret = __copy_from_user_inatomic(vaddr + page_offset,
781 user_data,
782 page_length);
783 kunmap_atomic(vaddr, KM_USER0);
784
785 set_page_dirty(page);
786 mark_page_accessed(page);
787 page_cache_release(page);
788
789 /* If we get a fault while copying data, then (presumably) our
790 * source page isn't available. Return the error and we'll
791 * retry in the slow path.
792 */
793 if (ret)
850 return -EFAULT; 794 return -EFAULT;
851 795
852 remain -= page_length; 796 remain -= page_length;
@@ -865,17 +809,18 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
865 * struct_mutex is held. 809 * struct_mutex is held.
866 */ 810 */
867static int 811static int
868i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, 812i915_gem_shmem_pwrite_slow(struct drm_device *dev,
813 struct drm_i915_gem_object *obj,
869 struct drm_i915_gem_pwrite *args, 814 struct drm_i915_gem_pwrite *args,
870 struct drm_file *file_priv) 815 struct drm_file *file)
871{ 816{
872 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 817 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
873 struct mm_struct *mm = current->mm; 818 struct mm_struct *mm = current->mm;
874 struct page **user_pages; 819 struct page **user_pages;
875 ssize_t remain; 820 ssize_t remain;
876 loff_t offset, pinned_pages, i; 821 loff_t offset, pinned_pages, i;
877 loff_t first_data_page, last_data_page, num_pages; 822 loff_t first_data_page, last_data_page, num_pages;
878 int shmem_page_index, shmem_page_offset; 823 int shmem_page_offset;
879 int data_page_index, data_page_offset; 824 int data_page_index, data_page_offset;
880 int page_length; 825 int page_length;
881 int ret; 826 int ret;
@@ -913,20 +858,19 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
913 858
914 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 859 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
915 860
916 obj_priv = to_intel_bo(obj);
917 offset = args->offset; 861 offset = args->offset;
918 obj_priv->dirty = 1; 862 obj->dirty = 1;
919 863
920 while (remain > 0) { 864 while (remain > 0) {
865 struct page *page;
866
921 /* Operation in this page 867 /* Operation in this page
922 * 868 *
923 * shmem_page_index = page number within shmem file
924 * shmem_page_offset = offset within page in shmem file 869 * shmem_page_offset = offset within page in shmem file
925 * data_page_index = page number in get_user_pages return 870 * data_page_index = page number in get_user_pages return
926 * data_page_offset = offset with data_page_index page. 871 * data_page_offset = offset with data_page_index page.
927 * page_length = bytes to copy for this page 872 * page_length = bytes to copy for this page
928 */ 873 */
929 shmem_page_index = offset / PAGE_SIZE;
930 shmem_page_offset = offset & ~PAGE_MASK; 874 shmem_page_offset = offset & ~PAGE_MASK;
931 data_page_index = data_ptr / PAGE_SIZE - first_data_page; 875 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
932 data_page_offset = data_ptr & ~PAGE_MASK; 876 data_page_offset = data_ptr & ~PAGE_MASK;
@@ -937,21 +881,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
937 if ((data_page_offset + page_length) > PAGE_SIZE) 881 if ((data_page_offset + page_length) > PAGE_SIZE)
938 page_length = PAGE_SIZE - data_page_offset; 882 page_length = PAGE_SIZE - data_page_offset;
939 883
884 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
885 GFP_HIGHUSER | __GFP_RECLAIMABLE);
886 if (IS_ERR(page)) {
887 ret = PTR_ERR(page);
888 goto out;
889 }
890
940 if (do_bit17_swizzling) { 891 if (do_bit17_swizzling) {
941 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], 892 slow_shmem_bit17_copy(page,
942 shmem_page_offset, 893 shmem_page_offset,
943 user_pages[data_page_index], 894 user_pages[data_page_index],
944 data_page_offset, 895 data_page_offset,
945 page_length, 896 page_length,
946 0); 897 0);
947 } else { 898 } else {
948 slow_shmem_copy(obj_priv->pages[shmem_page_index], 899 slow_shmem_copy(page,
949 shmem_page_offset, 900 shmem_page_offset,
950 user_pages[data_page_index], 901 user_pages[data_page_index],
951 data_page_offset, 902 data_page_offset,
952 page_length); 903 page_length);
953 } 904 }
954 905
906 set_page_dirty(page);
907 mark_page_accessed(page);
908 page_cache_release(page);
909
955 remain -= page_length; 910 remain -= page_length;
956 data_ptr += page_length; 911 data_ptr += page_length;
957 offset += page_length; 912 offset += page_length;
@@ -975,8 +930,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
975 struct drm_file *file) 930 struct drm_file *file)
976{ 931{
977 struct drm_i915_gem_pwrite *args = data; 932 struct drm_i915_gem_pwrite *args = data;
978 struct drm_gem_object *obj; 933 struct drm_i915_gem_object *obj;
979 struct drm_i915_gem_object *obj_priv;
980 int ret; 934 int ret;
981 935
982 if (args->size == 0) 936 if (args->size == 0)
@@ -996,15 +950,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
996 if (ret) 950 if (ret)
997 return ret; 951 return ret;
998 952
999 obj = drm_gem_object_lookup(dev, file, args->handle); 953 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1000 if (obj == NULL) { 954 if (obj == NULL) {
1001 ret = -ENOENT; 955 ret = -ENOENT;
1002 goto unlock; 956 goto unlock;
1003 } 957 }
1004 obj_priv = to_intel_bo(obj);
1005 958
1006 /* Bounds check destination. */ 959 /* Bounds check destination. */
1007 if (args->offset > obj->size || args->size > obj->size - args->offset) { 960 if (args->offset > obj->base.size ||
961 args->size > obj->base.size - args->offset) {
1008 ret = -EINVAL; 962 ret = -EINVAL;
1009 goto out; 963 goto out;
1010 } 964 }
@@ -1015,16 +969,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1015 * pread/pwrite currently are reading and writing from the CPU 969 * pread/pwrite currently are reading and writing from the CPU
1016 * perspective, requiring manual detiling by the client. 970 * perspective, requiring manual detiling by the client.
1017 */ 971 */
1018 if (obj_priv->phys_obj) 972 if (obj->phys_obj)
1019 ret = i915_gem_phys_pwrite(dev, obj, args, file); 973 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1020 else if (obj_priv->tiling_mode == I915_TILING_NONE && 974 else if (obj->gtt_space &&
1021 obj_priv->gtt_space && 975 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1022 obj->write_domain != I915_GEM_DOMAIN_CPU) { 976 ret = i915_gem_object_pin(obj, 0, true);
1023 ret = i915_gem_object_pin(obj, 0);
1024 if (ret) 977 if (ret)
1025 goto out; 978 goto out;
1026 979
1027 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 980 ret = i915_gem_object_set_to_gtt_domain(obj, true);
981 if (ret)
982 goto out_unpin;
983
984 ret = i915_gem_object_put_fence(obj);
1028 if (ret) 985 if (ret)
1029 goto out_unpin; 986 goto out_unpin;
1030 987
@@ -1035,26 +992,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1035out_unpin: 992out_unpin:
1036 i915_gem_object_unpin(obj); 993 i915_gem_object_unpin(obj);
1037 } else { 994 } else {
1038 ret = i915_gem_object_get_pages_or_evict(obj);
1039 if (ret)
1040 goto out;
1041
1042 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 995 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1043 if (ret) 996 if (ret)
1044 goto out_put; 997 goto out;
1045 998
1046 ret = -EFAULT; 999 ret = -EFAULT;
1047 if (!i915_gem_object_needs_bit17_swizzle(obj)) 1000 if (!i915_gem_object_needs_bit17_swizzle(obj))
1048 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); 1001 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1049 if (ret == -EFAULT) 1002 if (ret == -EFAULT)
1050 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); 1003 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1051
1052out_put:
1053 i915_gem_object_put_pages(obj);
1054 } 1004 }
1055 1005
1056out: 1006out:
1057 drm_gem_object_unreference(obj); 1007 drm_gem_object_unreference(&obj->base);
1058unlock: 1008unlock:
1059 mutex_unlock(&dev->struct_mutex); 1009 mutex_unlock(&dev->struct_mutex);
1060 return ret; 1010 return ret;
@@ -1066,12 +1016,10 @@ unlock:
1066 */ 1016 */
1067int 1017int
1068i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1018i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1069 struct drm_file *file_priv) 1019 struct drm_file *file)
1070{ 1020{
1071 struct drm_i915_private *dev_priv = dev->dev_private;
1072 struct drm_i915_gem_set_domain *args = data; 1021 struct drm_i915_gem_set_domain *args = data;
1073 struct drm_gem_object *obj; 1022 struct drm_i915_gem_object *obj;
1074 struct drm_i915_gem_object *obj_priv;
1075 uint32_t read_domains = args->read_domains; 1023 uint32_t read_domains = args->read_domains;
1076 uint32_t write_domain = args->write_domain; 1024 uint32_t write_domain = args->write_domain;
1077 int ret; 1025 int ret;
@@ -1096,28 +1044,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1096 if (ret) 1044 if (ret)
1097 return ret; 1045 return ret;
1098 1046
1099 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1047 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1100 if (obj == NULL) { 1048 if (obj == NULL) {
1101 ret = -ENOENT; 1049 ret = -ENOENT;
1102 goto unlock; 1050 goto unlock;
1103 } 1051 }
1104 obj_priv = to_intel_bo(obj);
1105
1106 intel_mark_busy(dev, obj);
1107 1052
1108 if (read_domains & I915_GEM_DOMAIN_GTT) { 1053 if (read_domains & I915_GEM_DOMAIN_GTT) {
1109 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1054 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1110 1055
1111 /* Update the LRU on the fence for the CPU access that's
1112 * about to occur.
1113 */
1114 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1115 struct drm_i915_fence_reg *reg =
1116 &dev_priv->fence_regs[obj_priv->fence_reg];
1117 list_move_tail(&reg->lru_list,
1118 &dev_priv->mm.fence_list);
1119 }
1120
1121 /* Silently promote "you're not bound, there was nothing to do" 1056 /* Silently promote "you're not bound, there was nothing to do"
1122 * to success, since the client was just asking us to 1057 * to success, since the client was just asking us to
1123 * make sure everything was done. 1058 * make sure everything was done.
@@ -1128,11 +1063,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1128 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1063 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1129 } 1064 }
1130 1065
1131 /* Maintain LRU order of "inactive" objects */ 1066 drm_gem_object_unreference(&obj->base);
1132 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1133 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1134
1135 drm_gem_object_unreference(obj);
1136unlock: 1067unlock:
1137 mutex_unlock(&dev->struct_mutex); 1068 mutex_unlock(&dev->struct_mutex);
1138 return ret; 1069 return ret;
@@ -1143,10 +1074,10 @@ unlock:
1143 */ 1074 */
1144int 1075int
1145i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1146 struct drm_file *file_priv) 1077 struct drm_file *file)
1147{ 1078{
1148 struct drm_i915_gem_sw_finish *args = data; 1079 struct drm_i915_gem_sw_finish *args = data;
1149 struct drm_gem_object *obj; 1080 struct drm_i915_gem_object *obj;
1150 int ret = 0; 1081 int ret = 0;
1151 1082
1152 if (!(dev->driver->driver_features & DRIVER_GEM)) 1083 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1156,17 +1087,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1156 if (ret) 1087 if (ret)
1157 return ret; 1088 return ret;
1158 1089
1159 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1090 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1160 if (obj == NULL) { 1091 if (obj == NULL) {
1161 ret = -ENOENT; 1092 ret = -ENOENT;
1162 goto unlock; 1093 goto unlock;
1163 } 1094 }
1164 1095
1165 /* Pinned buffers may be scanout, so flush the cache */ 1096 /* Pinned buffers may be scanout, so flush the cache */
1166 if (to_intel_bo(obj)->pin_count) 1097 if (obj->pin_count)
1167 i915_gem_object_flush_cpu_write_domain(obj); 1098 i915_gem_object_flush_cpu_write_domain(obj);
1168 1099
1169 drm_gem_object_unreference(obj); 1100 drm_gem_object_unreference(&obj->base);
1170unlock: 1101unlock:
1171 mutex_unlock(&dev->struct_mutex); 1102 mutex_unlock(&dev->struct_mutex);
1172 return ret; 1103 return ret;
@@ -1181,8 +1112,9 @@ unlock:
1181 */ 1112 */
1182int 1113int
1183i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1114i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1184 struct drm_file *file_priv) 1115 struct drm_file *file)
1185{ 1116{
1117 struct drm_i915_private *dev_priv = dev->dev_private;
1186 struct drm_i915_gem_mmap *args = data; 1118 struct drm_i915_gem_mmap *args = data;
1187 struct drm_gem_object *obj; 1119 struct drm_gem_object *obj;
1188 loff_t offset; 1120 loff_t offset;
@@ -1191,10 +1123,15 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1191 if (!(dev->driver->driver_features & DRIVER_GEM)) 1123 if (!(dev->driver->driver_features & DRIVER_GEM))
1192 return -ENODEV; 1124 return -ENODEV;
1193 1125
1194 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1126 obj = drm_gem_object_lookup(dev, file, args->handle);
1195 if (obj == NULL) 1127 if (obj == NULL)
1196 return -ENOENT; 1128 return -ENOENT;
1197 1129
1130 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1131 drm_gem_object_unreference_unlocked(obj);
1132 return -E2BIG;
1133 }
1134
1198 offset = args->offset; 1135 offset = args->offset;
1199 1136
1200 down_write(&current->mm->mmap_sem); 1137 down_write(&current->mm->mmap_sem);
@@ -1229,10 +1166,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1229 */ 1166 */
1230int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 1167int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1231{ 1168{
1232 struct drm_gem_object *obj = vma->vm_private_data; 1169 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1233 struct drm_device *dev = obj->dev; 1170 struct drm_device *dev = obj->base.dev;
1234 drm_i915_private_t *dev_priv = dev->dev_private; 1171 drm_i915_private_t *dev_priv = dev->dev_private;
1235 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1236 pgoff_t page_offset; 1172 pgoff_t page_offset;
1237 unsigned long pfn; 1173 unsigned long pfn;
1238 int ret = 0; 1174 int ret = 0;
@@ -1244,27 +1180,35 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1244 1180
1245 /* Now bind it into the GTT if needed */ 1181 /* Now bind it into the GTT if needed */
1246 mutex_lock(&dev->struct_mutex); 1182 mutex_lock(&dev->struct_mutex);
1247 if (!obj_priv->gtt_space) {
1248 ret = i915_gem_object_bind_to_gtt(obj, 0);
1249 if (ret)
1250 goto unlock;
1251 1183
1252 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1184 if (!obj->map_and_fenceable) {
1185 ret = i915_gem_object_unbind(obj);
1253 if (ret) 1186 if (ret)
1254 goto unlock; 1187 goto unlock;
1255 } 1188 }
1256 1189 if (!obj->gtt_space) {
1257 /* Need a new fence register? */ 1190 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1258 if (obj_priv->tiling_mode != I915_TILING_NONE) {
1259 ret = i915_gem_object_get_fence_reg(obj, true);
1260 if (ret) 1191 if (ret)
1261 goto unlock; 1192 goto unlock;
1262 } 1193 }
1263 1194
1264 if (i915_gem_object_is_inactive(obj_priv)) 1195 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1265 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 1196 if (ret)
1197 goto unlock;
1198
1199 if (obj->tiling_mode == I915_TILING_NONE)
1200 ret = i915_gem_object_put_fence(obj);
1201 else
1202 ret = i915_gem_object_get_fence(obj, NULL, true);
1203 if (ret)
1204 goto unlock;
1205
1206 if (i915_gem_object_is_inactive(obj))
1207 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1208
1209 obj->fault_mappable = true;
1266 1210
1267 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1211 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1268 page_offset; 1212 page_offset;
1269 1213
1270 /* Finally, remap it using the new GTT offset */ 1214 /* Finally, remap it using the new GTT offset */
@@ -1273,11 +1217,12 @@ unlock:
1273 mutex_unlock(&dev->struct_mutex); 1217 mutex_unlock(&dev->struct_mutex);
1274 1218
1275 switch (ret) { 1219 switch (ret) {
1220 case -EAGAIN:
1221 set_need_resched();
1276 case 0: 1222 case 0:
1277 case -ERESTARTSYS: 1223 case -ERESTARTSYS:
1278 return VM_FAULT_NOPAGE; 1224 return VM_FAULT_NOPAGE;
1279 case -ENOMEM: 1225 case -ENOMEM:
1280 case -EAGAIN:
1281 return VM_FAULT_OOM; 1226 return VM_FAULT_OOM;
1282 default: 1227 default:
1283 return VM_FAULT_SIGBUS; 1228 return VM_FAULT_SIGBUS;
@@ -1296,37 +1241,39 @@ unlock:
1296 * This routine allocates and attaches a fake offset for @obj. 1241 * This routine allocates and attaches a fake offset for @obj.
1297 */ 1242 */
1298static int 1243static int
1299i915_gem_create_mmap_offset(struct drm_gem_object *obj) 1244i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1300{ 1245{
1301 struct drm_device *dev = obj->dev; 1246 struct drm_device *dev = obj->base.dev;
1302 struct drm_gem_mm *mm = dev->mm_private; 1247 struct drm_gem_mm *mm = dev->mm_private;
1303 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1304 struct drm_map_list *list; 1248 struct drm_map_list *list;
1305 struct drm_local_map *map; 1249 struct drm_local_map *map;
1306 int ret = 0; 1250 int ret = 0;
1307 1251
1308 /* Set the object up for mmap'ing */ 1252 /* Set the object up for mmap'ing */
1309 list = &obj->map_list; 1253 list = &obj->base.map_list;
1310 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); 1254 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1311 if (!list->map) 1255 if (!list->map)
1312 return -ENOMEM; 1256 return -ENOMEM;
1313 1257
1314 map = list->map; 1258 map = list->map;
1315 map->type = _DRM_GEM; 1259 map->type = _DRM_GEM;
1316 map->size = obj->size; 1260 map->size = obj->base.size;
1317 map->handle = obj; 1261 map->handle = obj;
1318 1262
1319 /* Get a DRM GEM mmap offset allocated... */ 1263 /* Get a DRM GEM mmap offset allocated... */
1320 list->file_offset_node = drm_mm_search_free(&mm->offset_manager, 1264 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1321 obj->size / PAGE_SIZE, 0, 0); 1265 obj->base.size / PAGE_SIZE,
1266 0, 0);
1322 if (!list->file_offset_node) { 1267 if (!list->file_offset_node) {
1323 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); 1268 DRM_ERROR("failed to allocate offset for bo %d\n",
1269 obj->base.name);
1324 ret = -ENOSPC; 1270 ret = -ENOSPC;
1325 goto out_free_list; 1271 goto out_free_list;
1326 } 1272 }
1327 1273
1328 list->file_offset_node = drm_mm_get_block(list->file_offset_node, 1274 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1329 obj->size / PAGE_SIZE, 0); 1275 obj->base.size / PAGE_SIZE,
1276 0);
1330 if (!list->file_offset_node) { 1277 if (!list->file_offset_node) {
1331 ret = -ENOMEM; 1278 ret = -ENOMEM;
1332 goto out_free_list; 1279 goto out_free_list;
@@ -1339,16 +1286,13 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1339 goto out_free_mm; 1286 goto out_free_mm;
1340 } 1287 }
1341 1288
1342 /* By now we should be all set, any drm_mmap request on the offset
1343 * below will get to our mmap & fault handler */
1344 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1345
1346 return 0; 1289 return 0;
1347 1290
1348out_free_mm: 1291out_free_mm:
1349 drm_mm_put_block(list->file_offset_node); 1292 drm_mm_put_block(list->file_offset_node);
1350out_free_list: 1293out_free_list:
1351 kfree(list->map); 1294 kfree(list->map);
1295 list->map = NULL;
1352 1296
1353 return ret; 1297 return ret;
1354} 1298}
@@ -1368,38 +1312,51 @@ out_free_list:
1368 * fixup by i915_gem_fault(). 1312 * fixup by i915_gem_fault().
1369 */ 1313 */
1370void 1314void
1371i915_gem_release_mmap(struct drm_gem_object *obj) 1315i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1372{ 1316{
1373 struct drm_device *dev = obj->dev; 1317 if (!obj->fault_mappable)
1374 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1318 return;
1319
1320 unmap_mapping_range(obj->base.dev->dev_mapping,
1321 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1322 obj->base.size, 1);
1375 1323
1376 if (dev->dev_mapping) 1324 obj->fault_mappable = false;
1377 unmap_mapping_range(dev->dev_mapping,
1378 obj_priv->mmap_offset, obj->size, 1);
1379} 1325}
1380 1326
1381static void 1327static void
1382i915_gem_free_mmap_offset(struct drm_gem_object *obj) 1328i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1383{ 1329{
1384 struct drm_device *dev = obj->dev; 1330 struct drm_device *dev = obj->base.dev;
1385 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1386 struct drm_gem_mm *mm = dev->mm_private; 1331 struct drm_gem_mm *mm = dev->mm_private;
1387 struct drm_map_list *list; 1332 struct drm_map_list *list = &obj->base.map_list;
1388 1333
1389 list = &obj->map_list;
1390 drm_ht_remove_item(&mm->offset_hash, &list->hash); 1334 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1335 drm_mm_put_block(list->file_offset_node);
1336 kfree(list->map);
1337 list->map = NULL;
1338}
1391 1339
1392 if (list->file_offset_node) { 1340static uint32_t
1393 drm_mm_put_block(list->file_offset_node); 1341i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1394 list->file_offset_node = NULL; 1342{
1395 } 1343 struct drm_device *dev = obj->base.dev;
1344 uint32_t size;
1396 1345
1397 if (list->map) { 1346 if (INTEL_INFO(dev)->gen >= 4 ||
1398 kfree(list->map); 1347 obj->tiling_mode == I915_TILING_NONE)
1399 list->map = NULL; 1348 return obj->base.size;
1400 }
1401 1349
1402 obj_priv->mmap_offset = 0; 1350 /* Previous chips need a power-of-two fence region when tiling */
1351 if (INTEL_INFO(dev)->gen == 3)
1352 size = 1024*1024;
1353 else
1354 size = 512*1024;
1355
1356 while (size < obj->base.size)
1357 size <<= 1;
1358
1359 return size;
1403} 1360}
1404 1361
1405/** 1362/**
@@ -1407,42 +1364,68 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1407 * @obj: object to check 1364 * @obj: object to check
1408 * 1365 *
1409 * Return the required GTT alignment for an object, taking into account 1366 * Return the required GTT alignment for an object, taking into account
1410 * potential fence register mapping if needed. 1367 * potential fence register mapping.
1411 */ 1368 */
1412static uint32_t 1369static uint32_t
1413i915_gem_get_gtt_alignment(struct drm_gem_object *obj) 1370i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
1414{ 1371{
1415 struct drm_device *dev = obj->dev; 1372 struct drm_device *dev = obj->base.dev;
1416 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1417 int start, i;
1418 1373
1419 /* 1374 /*
1420 * Minimum alignment is 4k (GTT page size), but might be greater 1375 * Minimum alignment is 4k (GTT page size), but might be greater
1421 * if a fence register is needed for the object. 1376 * if a fence register is needed for the object.
1422 */ 1377 */
1423 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE) 1378 if (INTEL_INFO(dev)->gen >= 4 ||
1379 obj->tiling_mode == I915_TILING_NONE)
1424 return 4096; 1380 return 4096;
1425 1381
1426 /* 1382 /*
1427 * Previous chips need to be aligned to the size of the smallest 1383 * Previous chips need to be aligned to the size of the smallest
1428 * fence register that can contain the object. 1384 * fence register that can contain the object.
1429 */ 1385 */
1430 if (INTEL_INFO(dev)->gen == 3) 1386 return i915_gem_get_gtt_size(obj);
1431 start = 1024*1024; 1387}
1432 else
1433 start = 512*1024;
1434 1388
1435 for (i = start; i < obj->size; i <<= 1) 1389/**
1436 ; 1390 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1391 * unfenced object
1392 * @obj: object to check
1393 *
1394 * Return the required GTT alignment for an object, only taking into account
1395 * unfenced tiled surface requirements.
1396 */
1397static uint32_t
1398i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
1399{
1400 struct drm_device *dev = obj->base.dev;
1401 int tile_height;
1402
1403 /*
1404 * Minimum alignment is 4k (GTT page size) for sane hw.
1405 */
1406 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1407 obj->tiling_mode == I915_TILING_NONE)
1408 return 4096;
1409
1410 /*
1411 * Older chips need unfenced tiled buffers to be aligned to the left
1412 * edge of an even tile row (where tile rows are counted as if the bo is
1413 * placed in a fenced gtt region).
1414 */
1415 if (IS_GEN2(dev) ||
1416 (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
1417 tile_height = 32;
1418 else
1419 tile_height = 8;
1437 1420
1438 return i; 1421 return tile_height * obj->stride * 2;
1439} 1422}
1440 1423
1441/** 1424/**
1442 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 1425 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1443 * @dev: DRM device 1426 * @dev: DRM device
1444 * @data: GTT mapping ioctl data 1427 * @data: GTT mapping ioctl data
1445 * @file_priv: GEM object info 1428 * @file: GEM object info
1446 * 1429 *
1447 * Simply returns the fake offset to userspace so it can mmap it. 1430 * Simply returns the fake offset to userspace so it can mmap it.
1448 * The mmap call will end up in drm_gem_mmap(), which will set things 1431 * The mmap call will end up in drm_gem_mmap(), which will set things
@@ -1455,11 +1438,11 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1455 */ 1438 */
1456int 1439int
1457i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1440i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1458 struct drm_file *file_priv) 1441 struct drm_file *file)
1459{ 1442{
1443 struct drm_i915_private *dev_priv = dev->dev_private;
1460 struct drm_i915_gem_mmap_gtt *args = data; 1444 struct drm_i915_gem_mmap_gtt *args = data;
1461 struct drm_gem_object *obj; 1445 struct drm_i915_gem_object *obj;
1462 struct drm_i915_gem_object *obj_priv;
1463 int ret; 1446 int ret;
1464 1447
1465 if (!(dev->driver->driver_features & DRIVER_GEM)) 1448 if (!(dev->driver->driver_features & DRIVER_GEM))
@@ -1469,130 +1452,196 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1469 if (ret) 1452 if (ret)
1470 return ret; 1453 return ret;
1471 1454
1472 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 1455 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1473 if (obj == NULL) { 1456 if (obj == NULL) {
1474 ret = -ENOENT; 1457 ret = -ENOENT;
1475 goto unlock; 1458 goto unlock;
1476 } 1459 }
1477 obj_priv = to_intel_bo(obj);
1478 1460
1479 if (obj_priv->madv != I915_MADV_WILLNEED) { 1461 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1462 ret = -E2BIG;
1463 goto unlock;
1464 }
1465
1466 if (obj->madv != I915_MADV_WILLNEED) {
1480 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1467 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1481 ret = -EINVAL; 1468 ret = -EINVAL;
1482 goto out; 1469 goto out;
1483 } 1470 }
1484 1471
1485 if (!obj_priv->mmap_offset) { 1472 if (!obj->base.map_list.map) {
1486 ret = i915_gem_create_mmap_offset(obj); 1473 ret = i915_gem_create_mmap_offset(obj);
1487 if (ret) 1474 if (ret)
1488 goto out; 1475 goto out;
1489 } 1476 }
1490 1477
1491 args->offset = obj_priv->mmap_offset; 1478 args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1492
1493 /*
1494 * Pull it into the GTT so that we have a page list (makes the
1495 * initial fault faster and any subsequent flushing possible).
1496 */
1497 if (!obj_priv->agp_mem) {
1498 ret = i915_gem_object_bind_to_gtt(obj, 0);
1499 if (ret)
1500 goto out;
1501 }
1502 1479
1503out: 1480out:
1504 drm_gem_object_unreference(obj); 1481 drm_gem_object_unreference(&obj->base);
1505unlock: 1482unlock:
1506 mutex_unlock(&dev->struct_mutex); 1483 mutex_unlock(&dev->struct_mutex);
1507 return ret; 1484 return ret;
1508} 1485}
1509 1486
1487static int
1488i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1489 gfp_t gfpmask)
1490{
1491 int page_count, i;
1492 struct address_space *mapping;
1493 struct inode *inode;
1494 struct page *page;
1495
1496 /* Get the list of pages out of our struct file. They'll be pinned
1497 * at this point until we release them.
1498 */
1499 page_count = obj->base.size / PAGE_SIZE;
1500 BUG_ON(obj->pages != NULL);
1501 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1502 if (obj->pages == NULL)
1503 return -ENOMEM;
1504
1505 inode = obj->base.filp->f_path.dentry->d_inode;
1506 mapping = inode->i_mapping;
1507 for (i = 0; i < page_count; i++) {
1508 page = read_cache_page_gfp(mapping, i,
1509 GFP_HIGHUSER |
1510 __GFP_COLD |
1511 __GFP_RECLAIMABLE |
1512 gfpmask);
1513 if (IS_ERR(page))
1514 goto err_pages;
1515
1516 obj->pages[i] = page;
1517 }
1518
1519 if (obj->tiling_mode != I915_TILING_NONE)
1520 i915_gem_object_do_bit_17_swizzle(obj);
1521
1522 return 0;
1523
1524err_pages:
1525 while (i--)
1526 page_cache_release(obj->pages[i]);
1527
1528 drm_free_large(obj->pages);
1529 obj->pages = NULL;
1530 return PTR_ERR(page);
1531}
1532
1510static void 1533static void
1511i915_gem_object_put_pages(struct drm_gem_object *obj) 1534i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1512{ 1535{
1513 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1536 int page_count = obj->base.size / PAGE_SIZE;
1514 int page_count = obj->size / PAGE_SIZE;
1515 int i; 1537 int i;
1516 1538
1517 BUG_ON(obj_priv->pages_refcount == 0); 1539 BUG_ON(obj->madv == __I915_MADV_PURGED);
1518 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1519 1540
1520 if (--obj_priv->pages_refcount != 0) 1541 if (obj->tiling_mode != I915_TILING_NONE)
1521 return;
1522
1523 if (obj_priv->tiling_mode != I915_TILING_NONE)
1524 i915_gem_object_save_bit_17_swizzle(obj); 1542 i915_gem_object_save_bit_17_swizzle(obj);
1525 1543
1526 if (obj_priv->madv == I915_MADV_DONTNEED) 1544 if (obj->madv == I915_MADV_DONTNEED)
1527 obj_priv->dirty = 0; 1545 obj->dirty = 0;
1528 1546
1529 for (i = 0; i < page_count; i++) { 1547 for (i = 0; i < page_count; i++) {
1530 if (obj_priv->dirty) 1548 if (obj->dirty)
1531 set_page_dirty(obj_priv->pages[i]); 1549 set_page_dirty(obj->pages[i]);
1532 1550
1533 if (obj_priv->madv == I915_MADV_WILLNEED) 1551 if (obj->madv == I915_MADV_WILLNEED)
1534 mark_page_accessed(obj_priv->pages[i]); 1552 mark_page_accessed(obj->pages[i]);
1535 1553
1536 page_cache_release(obj_priv->pages[i]); 1554 page_cache_release(obj->pages[i]);
1537 } 1555 }
1538 obj_priv->dirty = 0; 1556 obj->dirty = 0;
1539
1540 drm_free_large(obj_priv->pages);
1541 obj_priv->pages = NULL;
1542}
1543
1544static uint32_t
1545i915_gem_next_request_seqno(struct drm_device *dev,
1546 struct intel_ring_buffer *ring)
1547{
1548 drm_i915_private_t *dev_priv = dev->dev_private;
1549 1557
1550 ring->outstanding_lazy_request = true; 1558 drm_free_large(obj->pages);
1551 return dev_priv->next_seqno; 1559 obj->pages = NULL;
1552} 1560}
1553 1561
1554static void 1562void
1555i915_gem_object_move_to_active(struct drm_gem_object *obj, 1563i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1556 struct intel_ring_buffer *ring) 1564 struct intel_ring_buffer *ring,
1565 u32 seqno)
1557{ 1566{
1558 struct drm_device *dev = obj->dev; 1567 struct drm_device *dev = obj->base.dev;
1559 struct drm_i915_private *dev_priv = dev->dev_private; 1568 struct drm_i915_private *dev_priv = dev->dev_private;
1560 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1561 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1562 1569
1563 BUG_ON(ring == NULL); 1570 BUG_ON(ring == NULL);
1564 obj_priv->ring = ring; 1571 obj->ring = ring;
1565 1572
1566 /* Add a reference if we're newly entering the active list. */ 1573 /* Add a reference if we're newly entering the active list. */
1567 if (!obj_priv->active) { 1574 if (!obj->active) {
1568 drm_gem_object_reference(obj); 1575 drm_gem_object_reference(&obj->base);
1569 obj_priv->active = 1; 1576 obj->active = 1;
1570 } 1577 }
1571 1578
1572 /* Move from whatever list we were on to the tail of execution. */ 1579 /* Move from whatever list we were on to the tail of execution. */
1573 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list); 1580 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1574 list_move_tail(&obj_priv->ring_list, &ring->active_list); 1581 list_move_tail(&obj->ring_list, &ring->active_list);
1575 obj_priv->last_rendering_seqno = seqno; 1582
1583 obj->last_rendering_seqno = seqno;
1584 if (obj->fenced_gpu_access) {
1585 struct drm_i915_fence_reg *reg;
1586
1587 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1588
1589 obj->last_fenced_seqno = seqno;
1590 obj->last_fenced_ring = ring;
1591
1592 reg = &dev_priv->fence_regs[obj->fence_reg];
1593 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
1594 }
1595}
1596
1597static void
1598i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1599{
1600 list_del_init(&obj->ring_list);
1601 obj->last_rendering_seqno = 0;
1576} 1602}
1577 1603
1578static void 1604static void
1579i915_gem_object_move_to_flushing(struct drm_gem_object *obj) 1605i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1580{ 1606{
1581 struct drm_device *dev = obj->dev; 1607 struct drm_device *dev = obj->base.dev;
1582 drm_i915_private_t *dev_priv = dev->dev_private; 1608 drm_i915_private_t *dev_priv = dev->dev_private;
1583 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1584 1609
1585 BUG_ON(!obj_priv->active); 1610 BUG_ON(!obj->active);
1586 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list); 1611 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1587 list_del_init(&obj_priv->ring_list); 1612
1588 obj_priv->last_rendering_seqno = 0; 1613 i915_gem_object_move_off_active(obj);
1614}
1615
1616static void
1617i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1618{
1619 struct drm_device *dev = obj->base.dev;
1620 struct drm_i915_private *dev_priv = dev->dev_private;
1621
1622 if (obj->pin_count != 0)
1623 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1624 else
1625 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1626
1627 BUG_ON(!list_empty(&obj->gpu_write_list));
1628 BUG_ON(!obj->active);
1629 obj->ring = NULL;
1630
1631 i915_gem_object_move_off_active(obj);
1632 obj->fenced_gpu_access = false;
1633
1634 obj->active = 0;
1635 obj->pending_gpu_write = false;
1636 drm_gem_object_unreference(&obj->base);
1637
1638 WARN_ON(i915_verify_lists(dev));
1589} 1639}
1590 1640
1591/* Immediately discard the backing storage */ 1641/* Immediately discard the backing storage */
1592static void 1642static void
1593i915_gem_object_truncate(struct drm_gem_object *obj) 1643i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1594{ 1644{
1595 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1596 struct inode *inode; 1645 struct inode *inode;
1597 1646
1598 /* Our goal here is to return as much of the memory as 1647 /* Our goal here is to return as much of the memory as
@@ -1601,42 +1650,18 @@ i915_gem_object_truncate(struct drm_gem_object *obj)
1601 * backing pages, *now*. Here we mirror the actions taken 1650 * backing pages, *now*. Here we mirror the actions taken
1602 * when by shmem_delete_inode() to release the backing store. 1651 * when by shmem_delete_inode() to release the backing store.
1603 */ 1652 */
1604 inode = obj->filp->f_path.dentry->d_inode; 1653 inode = obj->base.filp->f_path.dentry->d_inode;
1605 truncate_inode_pages(inode->i_mapping, 0); 1654 truncate_inode_pages(inode->i_mapping, 0);
1606 if (inode->i_op->truncate_range) 1655 if (inode->i_op->truncate_range)
1607 inode->i_op->truncate_range(inode, 0, (loff_t)-1); 1656 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1608 1657
1609 obj_priv->madv = __I915_MADV_PURGED; 1658 obj->madv = __I915_MADV_PURGED;
1610} 1659}
1611 1660
1612static inline int 1661static inline int
1613i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) 1662i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1614{
1615 return obj_priv->madv == I915_MADV_DONTNEED;
1616}
1617
1618static void
1619i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1620{ 1663{
1621 struct drm_device *dev = obj->dev; 1664 return obj->madv == I915_MADV_DONTNEED;
1622 drm_i915_private_t *dev_priv = dev->dev_private;
1623 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1624
1625 if (obj_priv->pin_count != 0)
1626 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1627 else
1628 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1629 list_del_init(&obj_priv->ring_list);
1630
1631 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1632
1633 obj_priv->last_rendering_seqno = 0;
1634 obj_priv->ring = NULL;
1635 if (obj_priv->active) {
1636 obj_priv->active = 0;
1637 drm_gem_object_unreference(obj);
1638 }
1639 WARN_ON(i915_verify_lists(dev));
1640} 1665}
1641 1666
1642static void 1667static void
@@ -1644,37 +1669,27 @@ i915_gem_process_flushing_list(struct drm_device *dev,
1644 uint32_t flush_domains, 1669 uint32_t flush_domains,
1645 struct intel_ring_buffer *ring) 1670 struct intel_ring_buffer *ring)
1646{ 1671{
1647 drm_i915_private_t *dev_priv = dev->dev_private; 1672 struct drm_i915_gem_object *obj, *next;
1648 struct drm_i915_gem_object *obj_priv, *next;
1649 1673
1650 list_for_each_entry_safe(obj_priv, next, 1674 list_for_each_entry_safe(obj, next,
1651 &ring->gpu_write_list, 1675 &ring->gpu_write_list,
1652 gpu_write_list) { 1676 gpu_write_list) {
1653 struct drm_gem_object *obj = &obj_priv->base; 1677 if (obj->base.write_domain & flush_domains) {
1654 1678 uint32_t old_write_domain = obj->base.write_domain;
1655 if (obj->write_domain & flush_domains) {
1656 uint32_t old_write_domain = obj->write_domain;
1657 1679
1658 obj->write_domain = 0; 1680 obj->base.write_domain = 0;
1659 list_del_init(&obj_priv->gpu_write_list); 1681 list_del_init(&obj->gpu_write_list);
1660 i915_gem_object_move_to_active(obj, ring); 1682 i915_gem_object_move_to_active(obj, ring,
1661 1683 i915_gem_next_request_seqno(dev, ring));
1662 /* update the fence lru list */
1663 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1664 struct drm_i915_fence_reg *reg =
1665 &dev_priv->fence_regs[obj_priv->fence_reg];
1666 list_move_tail(&reg->lru_list,
1667 &dev_priv->mm.fence_list);
1668 }
1669 1684
1670 trace_i915_gem_object_change_domain(obj, 1685 trace_i915_gem_object_change_domain(obj,
1671 obj->read_domains, 1686 obj->base.read_domains,
1672 old_write_domain); 1687 old_write_domain);
1673 } 1688 }
1674 } 1689 }
1675} 1690}
1676 1691
1677uint32_t 1692int
1678i915_add_request(struct drm_device *dev, 1693i915_add_request(struct drm_device *dev,
1679 struct drm_file *file, 1694 struct drm_file *file,
1680 struct drm_i915_gem_request *request, 1695 struct drm_i915_gem_request *request,
@@ -1684,17 +1699,17 @@ i915_add_request(struct drm_device *dev,
1684 struct drm_i915_file_private *file_priv = NULL; 1699 struct drm_i915_file_private *file_priv = NULL;
1685 uint32_t seqno; 1700 uint32_t seqno;
1686 int was_empty; 1701 int was_empty;
1702 int ret;
1703
1704 BUG_ON(request == NULL);
1687 1705
1688 if (file != NULL) 1706 if (file != NULL)
1689 file_priv = file->driver_priv; 1707 file_priv = file->driver_priv;
1690 1708
1691 if (request == NULL) { 1709 ret = ring->add_request(ring, &seqno);
1692 request = kzalloc(sizeof(*request), GFP_KERNEL); 1710 if (ret)
1693 if (request == NULL) 1711 return ret;
1694 return 0;
1695 }
1696 1712
1697 seqno = ring->add_request(dev, ring, 0);
1698 ring->outstanding_lazy_request = false; 1713 ring->outstanding_lazy_request = false;
1699 1714
1700 request->seqno = seqno; 1715 request->seqno = seqno;
@@ -1718,26 +1733,7 @@ i915_add_request(struct drm_device *dev,
1718 queue_delayed_work(dev_priv->wq, 1733 queue_delayed_work(dev_priv->wq,
1719 &dev_priv->mm.retire_work, HZ); 1734 &dev_priv->mm.retire_work, HZ);
1720 } 1735 }
1721 return seqno; 1736 return 0;
1722}
1723
1724/**
1725 * Command execution barrier
1726 *
1727 * Ensures that all commands in the ring are finished
1728 * before signalling the CPU
1729 */
1730static void
1731i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1732{
1733 uint32_t flush_domains = 0;
1734
1735 /* The sampler always gets flushed on i965 (sigh) */
1736 if (INTEL_INFO(dev)->gen >= 4)
1737 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1738
1739 ring->flush(dev, ring,
1740 I915_GEM_DOMAIN_COMMAND, flush_domains);
1741} 1737}
1742 1738
1743static inline void 1739static inline void
@@ -1770,62 +1766,76 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1770 } 1766 }
1771 1767
1772 while (!list_empty(&ring->active_list)) { 1768 while (!list_empty(&ring->active_list)) {
1773 struct drm_i915_gem_object *obj_priv; 1769 struct drm_i915_gem_object *obj;
1774 1770
1775 obj_priv = list_first_entry(&ring->active_list, 1771 obj = list_first_entry(&ring->active_list,
1776 struct drm_i915_gem_object, 1772 struct drm_i915_gem_object,
1777 ring_list); 1773 ring_list);
1778 1774
1779 obj_priv->base.write_domain = 0; 1775 obj->base.write_domain = 0;
1780 list_del_init(&obj_priv->gpu_write_list); 1776 list_del_init(&obj->gpu_write_list);
1781 i915_gem_object_move_to_inactive(&obj_priv->base); 1777 i915_gem_object_move_to_inactive(obj);
1778 }
1779}
1780
1781static void i915_gem_reset_fences(struct drm_device *dev)
1782{
1783 struct drm_i915_private *dev_priv = dev->dev_private;
1784 int i;
1785
1786 for (i = 0; i < 16; i++) {
1787 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1788 struct drm_i915_gem_object *obj = reg->obj;
1789
1790 if (!obj)
1791 continue;
1792
1793 if (obj->tiling_mode)
1794 i915_gem_release_mmap(obj);
1795
1796 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1797 reg->obj->fenced_gpu_access = false;
1798 reg->obj->last_fenced_seqno = 0;
1799 reg->obj->last_fenced_ring = NULL;
1800 i915_gem_clear_fence_reg(dev, reg);
1782 } 1801 }
1783} 1802}
1784 1803
1785void i915_gem_reset(struct drm_device *dev) 1804void i915_gem_reset(struct drm_device *dev)
1786{ 1805{
1787 struct drm_i915_private *dev_priv = dev->dev_private; 1806 struct drm_i915_private *dev_priv = dev->dev_private;
1788 struct drm_i915_gem_object *obj_priv; 1807 struct drm_i915_gem_object *obj;
1789 int i; 1808 int i;
1790 1809
1791 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); 1810 for (i = 0; i < I915_NUM_RINGS; i++)
1792 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); 1811 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1793 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1794 1812
1795 /* Remove anything from the flushing lists. The GPU cache is likely 1813 /* Remove anything from the flushing lists. The GPU cache is likely
1796 * to be lost on reset along with the data, so simply move the 1814 * to be lost on reset along with the data, so simply move the
1797 * lost bo to the inactive list. 1815 * lost bo to the inactive list.
1798 */ 1816 */
1799 while (!list_empty(&dev_priv->mm.flushing_list)) { 1817 while (!list_empty(&dev_priv->mm.flushing_list)) {
1800 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 1818 obj= list_first_entry(&dev_priv->mm.flushing_list,
1801 struct drm_i915_gem_object, 1819 struct drm_i915_gem_object,
1802 mm_list); 1820 mm_list);
1803 1821
1804 obj_priv->base.write_domain = 0; 1822 obj->base.write_domain = 0;
1805 list_del_init(&obj_priv->gpu_write_list); 1823 list_del_init(&obj->gpu_write_list);
1806 i915_gem_object_move_to_inactive(&obj_priv->base); 1824 i915_gem_object_move_to_inactive(obj);
1807 } 1825 }
1808 1826
1809 /* Move everything out of the GPU domains to ensure we do any 1827 /* Move everything out of the GPU domains to ensure we do any
1810 * necessary invalidation upon reuse. 1828 * necessary invalidation upon reuse.
1811 */ 1829 */
1812 list_for_each_entry(obj_priv, 1830 list_for_each_entry(obj,
1813 &dev_priv->mm.inactive_list, 1831 &dev_priv->mm.inactive_list,
1814 mm_list) 1832 mm_list)
1815 { 1833 {
1816 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1834 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1817 } 1835 }
1818 1836
1819 /* The fence registers are invalidated so clear them out */ 1837 /* The fence registers are invalidated so clear them out */
1820 for (i = 0; i < 16; i++) { 1838 i915_gem_reset_fences(dev);
1821 struct drm_i915_fence_reg *reg;
1822
1823 reg = &dev_priv->fence_regs[i];
1824 if (!reg->obj)
1825 continue;
1826
1827 i915_gem_clear_fence_reg(reg->obj);
1828 }
1829} 1839}
1830 1840
1831/** 1841/**
@@ -1837,6 +1847,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1837{ 1847{
1838 drm_i915_private_t *dev_priv = dev->dev_private; 1848 drm_i915_private_t *dev_priv = dev->dev_private;
1839 uint32_t seqno; 1849 uint32_t seqno;
1850 int i;
1840 1851
1841 if (!ring->status_page.page_addr || 1852 if (!ring->status_page.page_addr ||
1842 list_empty(&ring->request_list)) 1853 list_empty(&ring->request_list))
@@ -1844,7 +1855,12 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1844 1855
1845 WARN_ON(i915_verify_lists(dev)); 1856 WARN_ON(i915_verify_lists(dev));
1846 1857
1847 seqno = ring->get_seqno(dev, ring); 1858 seqno = ring->get_seqno(ring);
1859
1860 for (i = 0; i < I915_NUM_RINGS; i++)
1861 if (seqno >= ring->sync_seqno[i])
1862 ring->sync_seqno[i] = 0;
1863
1848 while (!list_empty(&ring->request_list)) { 1864 while (!list_empty(&ring->request_list)) {
1849 struct drm_i915_gem_request *request; 1865 struct drm_i915_gem_request *request;
1850 1866
@@ -1866,18 +1882,16 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1866 * by the ringbuffer to the flushing/inactive lists as appropriate. 1882 * by the ringbuffer to the flushing/inactive lists as appropriate.
1867 */ 1883 */
1868 while (!list_empty(&ring->active_list)) { 1884 while (!list_empty(&ring->active_list)) {
1869 struct drm_gem_object *obj; 1885 struct drm_i915_gem_object *obj;
1870 struct drm_i915_gem_object *obj_priv;
1871 1886
1872 obj_priv = list_first_entry(&ring->active_list, 1887 obj= list_first_entry(&ring->active_list,
1873 struct drm_i915_gem_object, 1888 struct drm_i915_gem_object,
1874 ring_list); 1889 ring_list);
1875 1890
1876 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) 1891 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1877 break; 1892 break;
1878 1893
1879 obj = &obj_priv->base; 1894 if (obj->base.write_domain != 0)
1880 if (obj->write_domain != 0)
1881 i915_gem_object_move_to_flushing(obj); 1895 i915_gem_object_move_to_flushing(obj);
1882 else 1896 else
1883 i915_gem_object_move_to_inactive(obj); 1897 i915_gem_object_move_to_inactive(obj);
@@ -1885,7 +1899,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1885 1899
1886 if (unlikely (dev_priv->trace_irq_seqno && 1900 if (unlikely (dev_priv->trace_irq_seqno &&
1887 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { 1901 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1888 ring->user_irq_put(dev, ring); 1902 ring->irq_put(ring);
1889 dev_priv->trace_irq_seqno = 0; 1903 dev_priv->trace_irq_seqno = 0;
1890 } 1904 }
1891 1905
@@ -1896,24 +1910,24 @@ void
1896i915_gem_retire_requests(struct drm_device *dev) 1910i915_gem_retire_requests(struct drm_device *dev)
1897{ 1911{
1898 drm_i915_private_t *dev_priv = dev->dev_private; 1912 drm_i915_private_t *dev_priv = dev->dev_private;
1913 int i;
1899 1914
1900 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 1915 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1901 struct drm_i915_gem_object *obj_priv, *tmp; 1916 struct drm_i915_gem_object *obj, *next;
1902 1917
1903 /* We must be careful that during unbind() we do not 1918 /* We must be careful that during unbind() we do not
1904 * accidentally infinitely recurse into retire requests. 1919 * accidentally infinitely recurse into retire requests.
1905 * Currently: 1920 * Currently:
1906 * retire -> free -> unbind -> wait -> retire_ring 1921 * retire -> free -> unbind -> wait -> retire_ring
1907 */ 1922 */
1908 list_for_each_entry_safe(obj_priv, tmp, 1923 list_for_each_entry_safe(obj, next,
1909 &dev_priv->mm.deferred_free_list, 1924 &dev_priv->mm.deferred_free_list,
1910 mm_list) 1925 mm_list)
1911 i915_gem_free_object_tail(&obj_priv->base); 1926 i915_gem_free_object_tail(obj);
1912 } 1927 }
1913 1928
1914 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); 1929 for (i = 0; i < I915_NUM_RINGS; i++)
1915 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); 1930 i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]);
1916 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1917} 1931}
1918 1932
1919static void 1933static void
@@ -1935,9 +1949,9 @@ i915_gem_retire_work_handler(struct work_struct *work)
1935 i915_gem_retire_requests(dev); 1949 i915_gem_retire_requests(dev);
1936 1950
1937 if (!dev_priv->mm.suspended && 1951 if (!dev_priv->mm.suspended &&
1938 (!list_empty(&dev_priv->render_ring.request_list) || 1952 (!list_empty(&dev_priv->ring[RCS].request_list) ||
1939 !list_empty(&dev_priv->bsd_ring.request_list) || 1953 !list_empty(&dev_priv->ring[VCS].request_list) ||
1940 !list_empty(&dev_priv->blt_ring.request_list))) 1954 !list_empty(&dev_priv->ring[BCS].request_list)))
1941 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 1955 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1942 mutex_unlock(&dev->struct_mutex); 1956 mutex_unlock(&dev->struct_mutex);
1943} 1957}
@@ -1955,14 +1969,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1955 if (atomic_read(&dev_priv->mm.wedged)) 1969 if (atomic_read(&dev_priv->mm.wedged))
1956 return -EAGAIN; 1970 return -EAGAIN;
1957 1971
1958 if (ring->outstanding_lazy_request) { 1972 if (seqno == ring->outstanding_lazy_request) {
1959 seqno = i915_add_request(dev, NULL, NULL, ring); 1973 struct drm_i915_gem_request *request;
1960 if (seqno == 0) 1974
1975 request = kzalloc(sizeof(*request), GFP_KERNEL);
1976 if (request == NULL)
1961 return -ENOMEM; 1977 return -ENOMEM;
1978
1979 ret = i915_add_request(dev, NULL, request, ring);
1980 if (ret) {
1981 kfree(request);
1982 return ret;
1983 }
1984
1985 seqno = request->seqno;
1962 } 1986 }
1963 BUG_ON(seqno == dev_priv->next_seqno);
1964 1987
1965 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 1988 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
1966 if (HAS_PCH_SPLIT(dev)) 1989 if (HAS_PCH_SPLIT(dev))
1967 ier = I915_READ(DEIER) | I915_READ(GTIER); 1990 ier = I915_READ(DEIER) | I915_READ(GTIER);
1968 else 1991 else
@@ -1976,21 +1999,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1976 1999
1977 trace_i915_gem_request_wait_begin(dev, seqno); 2000 trace_i915_gem_request_wait_begin(dev, seqno);
1978 2001
1979 ring->waiting_gem_seqno = seqno; 2002 ring->waiting_seqno = seqno;
1980 ring->user_irq_get(dev, ring); 2003 if (ring->irq_get(ring)) {
1981 if (interruptible) 2004 if (interruptible)
1982 ret = wait_event_interruptible(ring->irq_queue, 2005 ret = wait_event_interruptible(ring->irq_queue,
1983 i915_seqno_passed( 2006 i915_seqno_passed(ring->get_seqno(ring), seqno)
1984 ring->get_seqno(dev, ring), seqno) 2007 || atomic_read(&dev_priv->mm.wedged));
1985 || atomic_read(&dev_priv->mm.wedged)); 2008 else
1986 else 2009 wait_event(ring->irq_queue,
1987 wait_event(ring->irq_queue, 2010 i915_seqno_passed(ring->get_seqno(ring), seqno)
1988 i915_seqno_passed( 2011 || atomic_read(&dev_priv->mm.wedged));
1989 ring->get_seqno(dev, ring), seqno)
1990 || atomic_read(&dev_priv->mm.wedged));
1991 2012
1992 ring->user_irq_put(dev, ring); 2013 ring->irq_put(ring);
1993 ring->waiting_gem_seqno = 0; 2014 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2015 seqno) ||
2016 atomic_read(&dev_priv->mm.wedged), 3000))
2017 ret = -EBUSY;
2018 ring->waiting_seqno = 0;
1994 2019
1995 trace_i915_gem_request_wait_end(dev, seqno); 2020 trace_i915_gem_request_wait_end(dev, seqno);
1996 } 2021 }
@@ -1999,7 +2024,7 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1999 2024
2000 if (ret && ret != -ERESTARTSYS) 2025 if (ret && ret != -ERESTARTSYS)
2001 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", 2026 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2002 __func__, ret, seqno, ring->get_seqno(dev, ring), 2027 __func__, ret, seqno, ring->get_seqno(ring),
2003 dev_priv->next_seqno); 2028 dev_priv->next_seqno);
2004 2029
2005 /* Directly dispatch request retiring. While we have the work queue 2030 /* Directly dispatch request retiring. While we have the work queue
@@ -2024,70 +2049,30 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno,
2024 return i915_do_wait_request(dev, seqno, 1, ring); 2049 return i915_do_wait_request(dev, seqno, 1, ring);
2025} 2050}
2026 2051
2027static void
2028i915_gem_flush_ring(struct drm_device *dev,
2029 struct drm_file *file_priv,
2030 struct intel_ring_buffer *ring,
2031 uint32_t invalidate_domains,
2032 uint32_t flush_domains)
2033{
2034 ring->flush(dev, ring, invalidate_domains, flush_domains);
2035 i915_gem_process_flushing_list(dev, flush_domains, ring);
2036}
2037
2038static void
2039i915_gem_flush(struct drm_device *dev,
2040 struct drm_file *file_priv,
2041 uint32_t invalidate_domains,
2042 uint32_t flush_domains,
2043 uint32_t flush_rings)
2044{
2045 drm_i915_private_t *dev_priv = dev->dev_private;
2046
2047 if (flush_domains & I915_GEM_DOMAIN_CPU)
2048 drm_agp_chipset_flush(dev);
2049
2050 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2051 if (flush_rings & RING_RENDER)
2052 i915_gem_flush_ring(dev, file_priv,
2053 &dev_priv->render_ring,
2054 invalidate_domains, flush_domains);
2055 if (flush_rings & RING_BSD)
2056 i915_gem_flush_ring(dev, file_priv,
2057 &dev_priv->bsd_ring,
2058 invalidate_domains, flush_domains);
2059 if (flush_rings & RING_BLT)
2060 i915_gem_flush_ring(dev, file_priv,
2061 &dev_priv->blt_ring,
2062 invalidate_domains, flush_domains);
2063 }
2064}
2065
2066/** 2052/**
2067 * Ensures that all rendering to the object has completed and the object is 2053 * Ensures that all rendering to the object has completed and the object is
2068 * safe to unbind from the GTT or access from the CPU. 2054 * safe to unbind from the GTT or access from the CPU.
2069 */ 2055 */
2070static int 2056int
2071i915_gem_object_wait_rendering(struct drm_gem_object *obj, 2057i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2072 bool interruptible) 2058 bool interruptible)
2073{ 2059{
2074 struct drm_device *dev = obj->dev; 2060 struct drm_device *dev = obj->base.dev;
2075 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2076 int ret; 2061 int ret;
2077 2062
2078 /* This function only exists to support waiting for existing rendering, 2063 /* This function only exists to support waiting for existing rendering,
2079 * not for emitting required flushes. 2064 * not for emitting required flushes.
2080 */ 2065 */
2081 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); 2066 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2082 2067
2083 /* If there is rendering queued on the buffer being evicted, wait for 2068 /* If there is rendering queued on the buffer being evicted, wait for
2084 * it. 2069 * it.
2085 */ 2070 */
2086 if (obj_priv->active) { 2071 if (obj->active) {
2087 ret = i915_do_wait_request(dev, 2072 ret = i915_do_wait_request(dev,
2088 obj_priv->last_rendering_seqno, 2073 obj->last_rendering_seqno,
2089 interruptible, 2074 interruptible,
2090 obj_priv->ring); 2075 obj->ring);
2091 if (ret) 2076 if (ret)
2092 return ret; 2077 return ret;
2093 } 2078 }
@@ -2099,17 +2084,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2099 * Unbinds an object from the GTT aperture. 2084 * Unbinds an object from the GTT aperture.
2100 */ 2085 */
2101int 2086int
2102i915_gem_object_unbind(struct drm_gem_object *obj) 2087i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2103{ 2088{
2104 struct drm_device *dev = obj->dev;
2105 struct drm_i915_private *dev_priv = dev->dev_private;
2106 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2107 int ret = 0; 2089 int ret = 0;
2108 2090
2109 if (obj_priv->gtt_space == NULL) 2091 if (obj->gtt_space == NULL)
2110 return 0; 2092 return 0;
2111 2093
2112 if (obj_priv->pin_count != 0) { 2094 if (obj->pin_count != 0) {
2113 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2095 DRM_ERROR("Attempting to unbind pinned buffer\n");
2114 return -EINVAL; 2096 return -EINVAL;
2115 } 2097 }
@@ -2132,27 +2114,27 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2132 */ 2114 */
2133 if (ret) { 2115 if (ret) {
2134 i915_gem_clflush_object(obj); 2116 i915_gem_clflush_object(obj);
2135 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; 2117 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2136 } 2118 }
2137 2119
2138 /* release the fence reg _after_ flushing */ 2120 /* release the fence reg _after_ flushing */
2139 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) 2121 ret = i915_gem_object_put_fence(obj);
2140 i915_gem_clear_fence_reg(obj); 2122 if (ret == -ERESTARTSYS)
2141 2123 return ret;
2142 drm_unbind_agp(obj_priv->agp_mem);
2143 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2144 2124
2145 i915_gem_object_put_pages(obj); 2125 i915_gem_gtt_unbind_object(obj);
2146 BUG_ON(obj_priv->pages_refcount); 2126 i915_gem_object_put_pages_gtt(obj);
2147 2127
2148 i915_gem_info_remove_gtt(dev_priv, obj->size); 2128 list_del_init(&obj->gtt_list);
2149 list_del_init(&obj_priv->mm_list); 2129 list_del_init(&obj->mm_list);
2130 /* Avoid an unnecessary call to unbind on rebind. */
2131 obj->map_and_fenceable = true;
2150 2132
2151 drm_mm_put_block(obj_priv->gtt_space); 2133 drm_mm_put_block(obj->gtt_space);
2152 obj_priv->gtt_space = NULL; 2134 obj->gtt_space = NULL;
2153 obj_priv->gtt_offset = 0; 2135 obj->gtt_offset = 0;
2154 2136
2155 if (i915_gem_object_is_purgeable(obj_priv)) 2137 if (i915_gem_object_is_purgeable(obj))
2156 i915_gem_object_truncate(obj); 2138 i915_gem_object_truncate(obj);
2157 2139
2158 trace_i915_gem_object_unbind(obj); 2140 trace_i915_gem_object_unbind(obj);
@@ -2160,14 +2142,25 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2160 return ret; 2142 return ret;
2161} 2143}
2162 2144
2145void
2146i915_gem_flush_ring(struct drm_device *dev,
2147 struct intel_ring_buffer *ring,
2148 uint32_t invalidate_domains,
2149 uint32_t flush_domains)
2150{
2151 ring->flush(ring, invalidate_domains, flush_domains);
2152 i915_gem_process_flushing_list(dev, flush_domains, ring);
2153}
2154
2163static int i915_ring_idle(struct drm_device *dev, 2155static int i915_ring_idle(struct drm_device *dev,
2164 struct intel_ring_buffer *ring) 2156 struct intel_ring_buffer *ring)
2165{ 2157{
2166 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2158 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2167 return 0; 2159 return 0;
2168 2160
2169 i915_gem_flush_ring(dev, NULL, ring, 2161 if (!list_empty(&ring->gpu_write_list))
2170 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 2162 i915_gem_flush_ring(dev, ring,
2163 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2171 return i915_wait_request(dev, 2164 return i915_wait_request(dev,
2172 i915_gem_next_request_seqno(dev, ring), 2165 i915_gem_next_request_seqno(dev, ring),
2173 ring); 2166 ring);
@@ -2178,7 +2171,7 @@ i915_gpu_idle(struct drm_device *dev)
2178{ 2171{
2179 drm_i915_private_t *dev_priv = dev->dev_private; 2172 drm_i915_private_t *dev_priv = dev->dev_private;
2180 bool lists_empty; 2173 bool lists_empty;
2181 int ret; 2174 int ret, i;
2182 2175
2183 lists_empty = (list_empty(&dev_priv->mm.flushing_list) && 2176 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2184 list_empty(&dev_priv->mm.active_list)); 2177 list_empty(&dev_priv->mm.active_list));
@@ -2186,258 +2179,296 @@ i915_gpu_idle(struct drm_device *dev)
2186 return 0; 2179 return 0;
2187 2180
2188 /* Flush everything onto the inactive list. */ 2181 /* Flush everything onto the inactive list. */
2189 ret = i915_ring_idle(dev, &dev_priv->render_ring); 2182 for (i = 0; i < I915_NUM_RINGS; i++) {
2190 if (ret) 2183 ret = i915_ring_idle(dev, &dev_priv->ring[i]);
2191 return ret; 2184 if (ret)
2192 2185 return ret;
2193 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2194 if (ret)
2195 return ret;
2196
2197 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2198 if (ret)
2199 return ret;
2200
2201 return 0;
2202}
2203
2204static int
2205i915_gem_object_get_pages(struct drm_gem_object *obj,
2206 gfp_t gfpmask)
2207{
2208 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2209 int page_count, i;
2210 struct address_space *mapping;
2211 struct inode *inode;
2212 struct page *page;
2213
2214 BUG_ON(obj_priv->pages_refcount
2215 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2216
2217 if (obj_priv->pages_refcount++ != 0)
2218 return 0;
2219
2220 /* Get the list of pages out of our struct file. They'll be pinned
2221 * at this point until we release them.
2222 */
2223 page_count = obj->size / PAGE_SIZE;
2224 BUG_ON(obj_priv->pages != NULL);
2225 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2226 if (obj_priv->pages == NULL) {
2227 obj_priv->pages_refcount--;
2228 return -ENOMEM;
2229 }
2230
2231 inode = obj->filp->f_path.dentry->d_inode;
2232 mapping = inode->i_mapping;
2233 for (i = 0; i < page_count; i++) {
2234 page = read_cache_page_gfp(mapping, i,
2235 GFP_HIGHUSER |
2236 __GFP_COLD |
2237 __GFP_RECLAIMABLE |
2238 gfpmask);
2239 if (IS_ERR(page))
2240 goto err_pages;
2241
2242 obj_priv->pages[i] = page;
2243 } 2186 }
2244 2187
2245 if (obj_priv->tiling_mode != I915_TILING_NONE)
2246 i915_gem_object_do_bit_17_swizzle(obj);
2247
2248 return 0; 2188 return 0;
2249
2250err_pages:
2251 while (i--)
2252 page_cache_release(obj_priv->pages[i]);
2253
2254 drm_free_large(obj_priv->pages);
2255 obj_priv->pages = NULL;
2256 obj_priv->pages_refcount--;
2257 return PTR_ERR(page);
2258} 2189}
2259 2190
2260static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) 2191static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2192 struct intel_ring_buffer *pipelined)
2261{ 2193{
2262 struct drm_gem_object *obj = reg->obj; 2194 struct drm_device *dev = obj->base.dev;
2263 struct drm_device *dev = obj->dev;
2264 drm_i915_private_t *dev_priv = dev->dev_private; 2195 drm_i915_private_t *dev_priv = dev->dev_private;
2265 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2196 u32 size = obj->gtt_space->size;
2266 int regnum = obj_priv->fence_reg; 2197 int regnum = obj->fence_reg;
2267 uint64_t val; 2198 uint64_t val;
2268 2199
2269 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2200 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2270 0xfffff000) << 32; 2201 0xfffff000) << 32;
2271 val |= obj_priv->gtt_offset & 0xfffff000; 2202 val |= obj->gtt_offset & 0xfffff000;
2272 val |= (uint64_t)((obj_priv->stride / 128) - 1) << 2203 val |= (uint64_t)((obj->stride / 128) - 1) <<
2273 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2204 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2274 2205
2275 if (obj_priv->tiling_mode == I915_TILING_Y) 2206 if (obj->tiling_mode == I915_TILING_Y)
2276 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2207 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2277 val |= I965_FENCE_REG_VALID; 2208 val |= I965_FENCE_REG_VALID;
2278 2209
2279 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val); 2210 if (pipelined) {
2211 int ret = intel_ring_begin(pipelined, 6);
2212 if (ret)
2213 return ret;
2214
2215 intel_ring_emit(pipelined, MI_NOOP);
2216 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2217 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2218 intel_ring_emit(pipelined, (u32)val);
2219 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2220 intel_ring_emit(pipelined, (u32)(val >> 32));
2221 intel_ring_advance(pipelined);
2222 } else
2223 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2224
2225 return 0;
2280} 2226}
2281 2227
2282static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) 2228static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2229 struct intel_ring_buffer *pipelined)
2283{ 2230{
2284 struct drm_gem_object *obj = reg->obj; 2231 struct drm_device *dev = obj->base.dev;
2285 struct drm_device *dev = obj->dev;
2286 drm_i915_private_t *dev_priv = dev->dev_private; 2232 drm_i915_private_t *dev_priv = dev->dev_private;
2287 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2233 u32 size = obj->gtt_space->size;
2288 int regnum = obj_priv->fence_reg; 2234 int regnum = obj->fence_reg;
2289 uint64_t val; 2235 uint64_t val;
2290 2236
2291 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & 2237 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2292 0xfffff000) << 32; 2238 0xfffff000) << 32;
2293 val |= obj_priv->gtt_offset & 0xfffff000; 2239 val |= obj->gtt_offset & 0xfffff000;
2294 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2240 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2295 if (obj_priv->tiling_mode == I915_TILING_Y) 2241 if (obj->tiling_mode == I915_TILING_Y)
2296 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2242 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2297 val |= I965_FENCE_REG_VALID; 2243 val |= I965_FENCE_REG_VALID;
2298 2244
2299 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); 2245 if (pipelined) {
2246 int ret = intel_ring_begin(pipelined, 6);
2247 if (ret)
2248 return ret;
2249
2250 intel_ring_emit(pipelined, MI_NOOP);
2251 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2252 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2253 intel_ring_emit(pipelined, (u32)val);
2254 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2255 intel_ring_emit(pipelined, (u32)(val >> 32));
2256 intel_ring_advance(pipelined);
2257 } else
2258 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2259
2260 return 0;
2300} 2261}
2301 2262
2302static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) 2263static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2264 struct intel_ring_buffer *pipelined)
2303{ 2265{
2304 struct drm_gem_object *obj = reg->obj; 2266 struct drm_device *dev = obj->base.dev;
2305 struct drm_device *dev = obj->dev;
2306 drm_i915_private_t *dev_priv = dev->dev_private; 2267 drm_i915_private_t *dev_priv = dev->dev_private;
2307 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2268 u32 size = obj->gtt_space->size;
2308 int regnum = obj_priv->fence_reg; 2269 u32 fence_reg, val, pitch_val;
2309 int tile_width; 2270 int tile_width;
2310 uint32_t fence_reg, val;
2311 uint32_t pitch_val;
2312 2271
2313 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || 2272 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2314 (obj_priv->gtt_offset & (obj->size - 1))) { 2273 (size & -size) != size ||
2315 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", 2274 (obj->gtt_offset & (size - 1)),
2316 __func__, obj_priv->gtt_offset, obj->size); 2275 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2317 return; 2276 obj->gtt_offset, obj->map_and_fenceable, size))
2318 } 2277 return -EINVAL;
2319 2278
2320 if (obj_priv->tiling_mode == I915_TILING_Y && 2279 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2321 HAS_128_BYTE_Y_TILING(dev))
2322 tile_width = 128; 2280 tile_width = 128;
2323 else 2281 else
2324 tile_width = 512; 2282 tile_width = 512;
2325 2283
2326 /* Note: pitch better be a power of two tile widths */ 2284 /* Note: pitch better be a power of two tile widths */
2327 pitch_val = obj_priv->stride / tile_width; 2285 pitch_val = obj->stride / tile_width;
2328 pitch_val = ffs(pitch_val) - 1; 2286 pitch_val = ffs(pitch_val) - 1;
2329 2287
2330 if (obj_priv->tiling_mode == I915_TILING_Y && 2288 val = obj->gtt_offset;
2331 HAS_128_BYTE_Y_TILING(dev)) 2289 if (obj->tiling_mode == I915_TILING_Y)
2332 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2333 else
2334 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2335
2336 val = obj_priv->gtt_offset;
2337 if (obj_priv->tiling_mode == I915_TILING_Y)
2338 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2290 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2339 val |= I915_FENCE_SIZE_BITS(obj->size); 2291 val |= I915_FENCE_SIZE_BITS(size);
2340 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2292 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2341 val |= I830_FENCE_REG_VALID; 2293 val |= I830_FENCE_REG_VALID;
2342 2294
2343 if (regnum < 8) 2295 fence_reg = obj->fence_reg;
2344 fence_reg = FENCE_REG_830_0 + (regnum * 4); 2296 if (fence_reg < 8)
2297 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2345 else 2298 else
2346 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); 2299 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2347 I915_WRITE(fence_reg, val); 2300
2301 if (pipelined) {
2302 int ret = intel_ring_begin(pipelined, 4);
2303 if (ret)
2304 return ret;
2305
2306 intel_ring_emit(pipelined, MI_NOOP);
2307 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2308 intel_ring_emit(pipelined, fence_reg);
2309 intel_ring_emit(pipelined, val);
2310 intel_ring_advance(pipelined);
2311 } else
2312 I915_WRITE(fence_reg, val);
2313
2314 return 0;
2348} 2315}
2349 2316
2350static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) 2317static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2318 struct intel_ring_buffer *pipelined)
2351{ 2319{
2352 struct drm_gem_object *obj = reg->obj; 2320 struct drm_device *dev = obj->base.dev;
2353 struct drm_device *dev = obj->dev;
2354 drm_i915_private_t *dev_priv = dev->dev_private; 2321 drm_i915_private_t *dev_priv = dev->dev_private;
2355 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2322 u32 size = obj->gtt_space->size;
2356 int regnum = obj_priv->fence_reg; 2323 int regnum = obj->fence_reg;
2357 uint32_t val; 2324 uint32_t val;
2358 uint32_t pitch_val; 2325 uint32_t pitch_val;
2359 uint32_t fence_size_bits;
2360 2326
2361 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || 2327 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2362 (obj_priv->gtt_offset & (obj->size - 1))) { 2328 (size & -size) != size ||
2363 WARN(1, "%s: object 0x%08x not 512K or size aligned\n", 2329 (obj->gtt_offset & (size - 1)),
2364 __func__, obj_priv->gtt_offset); 2330 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2365 return; 2331 obj->gtt_offset, size))
2366 } 2332 return -EINVAL;
2367 2333
2368 pitch_val = obj_priv->stride / 128; 2334 pitch_val = obj->stride / 128;
2369 pitch_val = ffs(pitch_val) - 1; 2335 pitch_val = ffs(pitch_val) - 1;
2370 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2371 2336
2372 val = obj_priv->gtt_offset; 2337 val = obj->gtt_offset;
2373 if (obj_priv->tiling_mode == I915_TILING_Y) 2338 if (obj->tiling_mode == I915_TILING_Y)
2374 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2339 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2375 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); 2340 val |= I830_FENCE_SIZE_BITS(size);
2376 WARN_ON(fence_size_bits & ~0x00000f00);
2377 val |= fence_size_bits;
2378 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2341 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2379 val |= I830_FENCE_REG_VALID; 2342 val |= I830_FENCE_REG_VALID;
2380 2343
2381 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); 2344 if (pipelined) {
2345 int ret = intel_ring_begin(pipelined, 4);
2346 if (ret)
2347 return ret;
2348
2349 intel_ring_emit(pipelined, MI_NOOP);
2350 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2351 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2352 intel_ring_emit(pipelined, val);
2353 intel_ring_advance(pipelined);
2354 } else
2355 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2356
2357 return 0;
2382} 2358}
2383 2359
2384static int i915_find_fence_reg(struct drm_device *dev, 2360static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2385 bool interruptible) 2361{
2362 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2363}
2364
2365static int
2366i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2367 struct intel_ring_buffer *pipelined,
2368 bool interruptible)
2369{
2370 int ret;
2371
2372 if (obj->fenced_gpu_access) {
2373 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2374 i915_gem_flush_ring(obj->base.dev,
2375 obj->last_fenced_ring,
2376 0, obj->base.write_domain);
2377
2378 obj->fenced_gpu_access = false;
2379 }
2380
2381 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2382 if (!ring_passed_seqno(obj->last_fenced_ring,
2383 obj->last_fenced_seqno)) {
2384 ret = i915_do_wait_request(obj->base.dev,
2385 obj->last_fenced_seqno,
2386 interruptible,
2387 obj->last_fenced_ring);
2388 if (ret)
2389 return ret;
2390 }
2391
2392 obj->last_fenced_seqno = 0;
2393 obj->last_fenced_ring = NULL;
2394 }
2395
2396 return 0;
2397}
2398
2399int
2400i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2401{
2402 int ret;
2403
2404 if (obj->tiling_mode)
2405 i915_gem_release_mmap(obj);
2406
2407 ret = i915_gem_object_flush_fence(obj, NULL, true);
2408 if (ret)
2409 return ret;
2410
2411 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2412 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2413 i915_gem_clear_fence_reg(obj->base.dev,
2414 &dev_priv->fence_regs[obj->fence_reg]);
2415
2416 obj->fence_reg = I915_FENCE_REG_NONE;
2417 }
2418
2419 return 0;
2420}
2421
2422static struct drm_i915_fence_reg *
2423i915_find_fence_reg(struct drm_device *dev,
2424 struct intel_ring_buffer *pipelined)
2386{ 2425{
2387 struct drm_i915_fence_reg *reg = NULL;
2388 struct drm_i915_gem_object *obj_priv = NULL;
2389 struct drm_i915_private *dev_priv = dev->dev_private; 2426 struct drm_i915_private *dev_priv = dev->dev_private;
2390 struct drm_gem_object *obj = NULL; 2427 struct drm_i915_fence_reg *reg, *first, *avail;
2391 int i, avail, ret; 2428 int i;
2392 2429
2393 /* First try to find a free reg */ 2430 /* First try to find a free reg */
2394 avail = 0; 2431 avail = NULL;
2395 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 2432 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2396 reg = &dev_priv->fence_regs[i]; 2433 reg = &dev_priv->fence_regs[i];
2397 if (!reg->obj) 2434 if (!reg->obj)
2398 return i; 2435 return reg;
2399 2436
2400 obj_priv = to_intel_bo(reg->obj); 2437 if (!reg->obj->pin_count)
2401 if (!obj_priv->pin_count) 2438 avail = reg;
2402 avail++;
2403 } 2439 }
2404 2440
2405 if (avail == 0) 2441 if (avail == NULL)
2406 return -ENOSPC; 2442 return NULL;
2407 2443
2408 /* None available, try to steal one or wait for a user to finish */ 2444 /* None available, try to steal one or wait for a user to finish */
2409 i = I915_FENCE_REG_NONE; 2445 avail = first = NULL;
2410 list_for_each_entry(reg, &dev_priv->mm.fence_list, 2446 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2411 lru_list) { 2447 if (reg->obj->pin_count)
2412 obj = reg->obj;
2413 obj_priv = to_intel_bo(obj);
2414
2415 if (obj_priv->pin_count)
2416 continue; 2448 continue;
2417 2449
2418 /* found one! */ 2450 if (first == NULL)
2419 i = obj_priv->fence_reg; 2451 first = reg;
2420 break; 2452
2453 if (!pipelined ||
2454 !reg->obj->last_fenced_ring ||
2455 reg->obj->last_fenced_ring == pipelined) {
2456 avail = reg;
2457 break;
2458 }
2421 } 2459 }
2422 2460
2423 BUG_ON(i == I915_FENCE_REG_NONE); 2461 if (avail == NULL)
2462 avail = first;
2424 2463
2425 /* We only have a reference on obj from the active list. put_fence_reg 2464 return avail;
2426 * might drop that one, causing a use-after-free in it. So hold a
2427 * private reference to obj like the other callers of put_fence_reg
2428 * (set_tiling ioctl) do. */
2429 drm_gem_object_reference(obj);
2430 ret = i915_gem_object_put_fence_reg(obj, interruptible);
2431 drm_gem_object_unreference(obj);
2432 if (ret != 0)
2433 return ret;
2434
2435 return i;
2436} 2465}
2437 2466
2438/** 2467/**
2439 * i915_gem_object_get_fence_reg - set up a fence reg for an object 2468 * i915_gem_object_get_fence - set up a fence reg for an object
2440 * @obj: object to map through a fence reg 2469 * @obj: object to map through a fence reg
2470 * @pipelined: ring on which to queue the change, or NULL for CPU access
2471 * @interruptible: must we wait uninterruptibly for the register to retire?
2441 * 2472 *
2442 * When mapping objects through the GTT, userspace wants to be able to write 2473 * When mapping objects through the GTT, userspace wants to be able to write
2443 * to them without having to worry about swizzling if the object is tiled. 2474 * to them without having to worry about swizzling if the object is tiled.
@@ -2449,72 +2480,138 @@ static int i915_find_fence_reg(struct drm_device *dev,
2449 * and tiling format. 2480 * and tiling format.
2450 */ 2481 */
2451int 2482int
2452i915_gem_object_get_fence_reg(struct drm_gem_object *obj, 2483i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2453 bool interruptible) 2484 struct intel_ring_buffer *pipelined,
2485 bool interruptible)
2454{ 2486{
2455 struct drm_device *dev = obj->dev; 2487 struct drm_device *dev = obj->base.dev;
2456 struct drm_i915_private *dev_priv = dev->dev_private; 2488 struct drm_i915_private *dev_priv = dev->dev_private;
2457 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2489 struct drm_i915_fence_reg *reg;
2458 struct drm_i915_fence_reg *reg = NULL;
2459 int ret; 2490 int ret;
2460 2491
2461 /* Just update our place in the LRU if our fence is getting used. */ 2492 /* XXX disable pipelining. There are bugs. Shocking. */
2462 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { 2493 pipelined = NULL;
2463 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2494
2495 /* Just update our place in the LRU if our fence is getting reused. */
2496 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2497 reg = &dev_priv->fence_regs[obj->fence_reg];
2464 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2498 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2499
2500 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2501 pipelined = NULL;
2502
2503 if (!pipelined) {
2504 if (reg->setup_seqno) {
2505 if (!ring_passed_seqno(obj->last_fenced_ring,
2506 reg->setup_seqno)) {
2507 ret = i915_do_wait_request(obj->base.dev,
2508 reg->setup_seqno,
2509 interruptible,
2510 obj->last_fenced_ring);
2511 if (ret)
2512 return ret;
2513 }
2514
2515 reg->setup_seqno = 0;
2516 }
2517 } else if (obj->last_fenced_ring &&
2518 obj->last_fenced_ring != pipelined) {
2519 ret = i915_gem_object_flush_fence(obj,
2520 pipelined,
2521 interruptible);
2522 if (ret)
2523 return ret;
2524 } else if (obj->tiling_changed) {
2525 if (obj->fenced_gpu_access) {
2526 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2527 i915_gem_flush_ring(obj->base.dev, obj->ring,
2528 0, obj->base.write_domain);
2529
2530 obj->fenced_gpu_access = false;
2531 }
2532 }
2533
2534 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2535 pipelined = NULL;
2536 BUG_ON(!pipelined && reg->setup_seqno);
2537
2538 if (obj->tiling_changed) {
2539 if (pipelined) {
2540 reg->setup_seqno =
2541 i915_gem_next_request_seqno(dev, pipelined);
2542 obj->last_fenced_seqno = reg->setup_seqno;
2543 obj->last_fenced_ring = pipelined;
2544 }
2545 goto update;
2546 }
2547
2465 return 0; 2548 return 0;
2466 } 2549 }
2467 2550
2468 switch (obj_priv->tiling_mode) { 2551 reg = i915_find_fence_reg(dev, pipelined);
2469 case I915_TILING_NONE: 2552 if (reg == NULL)
2470 WARN(1, "allocating a fence for non-tiled object?\n"); 2553 return -ENOSPC;
2471 break;
2472 case I915_TILING_X:
2473 if (!obj_priv->stride)
2474 return -EINVAL;
2475 WARN((obj_priv->stride & (512 - 1)),
2476 "object 0x%08x is X tiled but has non-512B pitch\n",
2477 obj_priv->gtt_offset);
2478 break;
2479 case I915_TILING_Y:
2480 if (!obj_priv->stride)
2481 return -EINVAL;
2482 WARN((obj_priv->stride & (128 - 1)),
2483 "object 0x%08x is Y tiled but has non-128B pitch\n",
2484 obj_priv->gtt_offset);
2485 break;
2486 }
2487 2554
2488 ret = i915_find_fence_reg(dev, interruptible); 2555 ret = i915_gem_object_flush_fence(obj, pipelined, interruptible);
2489 if (ret < 0) 2556 if (ret)
2490 return ret; 2557 return ret;
2491 2558
2492 obj_priv->fence_reg = ret; 2559 if (reg->obj) {
2493 reg = &dev_priv->fence_regs[obj_priv->fence_reg]; 2560 struct drm_i915_gem_object *old = reg->obj;
2494 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list); 2561
2562 drm_gem_object_reference(&old->base);
2563
2564 if (old->tiling_mode)
2565 i915_gem_release_mmap(old);
2566
2567 ret = i915_gem_object_flush_fence(old,
2568 pipelined,
2569 interruptible);
2570 if (ret) {
2571 drm_gem_object_unreference(&old->base);
2572 return ret;
2573 }
2574
2575 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2576 pipelined = NULL;
2577
2578 old->fence_reg = I915_FENCE_REG_NONE;
2579 old->last_fenced_ring = pipelined;
2580 old->last_fenced_seqno =
2581 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2582
2583 drm_gem_object_unreference(&old->base);
2584 } else if (obj->last_fenced_seqno == 0)
2585 pipelined = NULL;
2495 2586
2496 reg->obj = obj; 2587 reg->obj = obj;
2588 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2589 obj->fence_reg = reg - dev_priv->fence_regs;
2590 obj->last_fenced_ring = pipelined;
2591
2592 reg->setup_seqno =
2593 pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0;
2594 obj->last_fenced_seqno = reg->setup_seqno;
2497 2595
2596update:
2597 obj->tiling_changed = false;
2498 switch (INTEL_INFO(dev)->gen) { 2598 switch (INTEL_INFO(dev)->gen) {
2499 case 6: 2599 case 6:
2500 sandybridge_write_fence_reg(reg); 2600 ret = sandybridge_write_fence_reg(obj, pipelined);
2501 break; 2601 break;
2502 case 5: 2602 case 5:
2503 case 4: 2603 case 4:
2504 i965_write_fence_reg(reg); 2604 ret = i965_write_fence_reg(obj, pipelined);
2505 break; 2605 break;
2506 case 3: 2606 case 3:
2507 i915_write_fence_reg(reg); 2607 ret = i915_write_fence_reg(obj, pipelined);
2508 break; 2608 break;
2509 case 2: 2609 case 2:
2510 i830_write_fence_reg(reg); 2610 ret = i830_write_fence_reg(obj, pipelined);
2511 break; 2611 break;
2512 } 2612 }
2513 2613
2514 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg, 2614 return ret;
2515 obj_priv->tiling_mode);
2516
2517 return 0;
2518} 2615}
2519 2616
2520/** 2617/**
@@ -2522,154 +2619,127 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2522 * @obj: object to clear 2619 * @obj: object to clear
2523 * 2620 *
2524 * Zeroes out the fence register itself and clears out the associated 2621 * Zeroes out the fence register itself and clears out the associated
2525 * data structures in dev_priv and obj_priv. 2622 * data structures in dev_priv and obj.
2526 */ 2623 */
2527static void 2624static void
2528i915_gem_clear_fence_reg(struct drm_gem_object *obj) 2625i915_gem_clear_fence_reg(struct drm_device *dev,
2626 struct drm_i915_fence_reg *reg)
2529{ 2627{
2530 struct drm_device *dev = obj->dev;
2531 drm_i915_private_t *dev_priv = dev->dev_private; 2628 drm_i915_private_t *dev_priv = dev->dev_private;
2532 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 2629 uint32_t fence_reg = reg - dev_priv->fence_regs;
2533 struct drm_i915_fence_reg *reg =
2534 &dev_priv->fence_regs[obj_priv->fence_reg];
2535 uint32_t fence_reg;
2536 2630
2537 switch (INTEL_INFO(dev)->gen) { 2631 switch (INTEL_INFO(dev)->gen) {
2538 case 6: 2632 case 6:
2539 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + 2633 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2540 (obj_priv->fence_reg * 8), 0);
2541 break; 2634 break;
2542 case 5: 2635 case 5:
2543 case 4: 2636 case 4:
2544 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); 2637 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2545 break; 2638 break;
2546 case 3: 2639 case 3:
2547 if (obj_priv->fence_reg >= 8) 2640 if (fence_reg >= 8)
2548 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; 2641 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2549 else 2642 else
2550 case 2: 2643 case 2:
2551 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; 2644 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2552 2645
2553 I915_WRITE(fence_reg, 0); 2646 I915_WRITE(fence_reg, 0);
2554 break; 2647 break;
2555 } 2648 }
2556 2649
2557 reg->obj = NULL;
2558 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2559 list_del_init(&reg->lru_list); 2650 list_del_init(&reg->lru_list);
2560} 2651 reg->obj = NULL;
2561 2652 reg->setup_seqno = 0;
2562/**
2563 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2564 * to the buffer to finish, and then resets the fence register.
2565 * @obj: tiled object holding a fence register.
2566 * @bool: whether the wait upon the fence is interruptible
2567 *
2568 * Zeroes out the fence register itself and clears out the associated
2569 * data structures in dev_priv and obj_priv.
2570 */
2571int
2572i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2573 bool interruptible)
2574{
2575 struct drm_device *dev = obj->dev;
2576 struct drm_i915_private *dev_priv = dev->dev_private;
2577 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2578 struct drm_i915_fence_reg *reg;
2579
2580 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2581 return 0;
2582
2583 /* If we've changed tiling, GTT-mappings of the object
2584 * need to re-fault to ensure that the correct fence register
2585 * setup is in place.
2586 */
2587 i915_gem_release_mmap(obj);
2588
2589 /* On the i915, GPU access to tiled buffers is via a fence,
2590 * therefore we must wait for any outstanding access to complete
2591 * before clearing the fence.
2592 */
2593 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2594 if (reg->gpu) {
2595 int ret;
2596
2597 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2598 if (ret)
2599 return ret;
2600
2601 ret = i915_gem_object_wait_rendering(obj, interruptible);
2602 if (ret)
2603 return ret;
2604
2605 reg->gpu = false;
2606 }
2607
2608 i915_gem_object_flush_gtt_write_domain(obj);
2609 i915_gem_clear_fence_reg(obj);
2610
2611 return 0;
2612} 2653}
2613 2654
2614/** 2655/**
2615 * Finds free space in the GTT aperture and binds the object there. 2656 * Finds free space in the GTT aperture and binds the object there.
2616 */ 2657 */
2617static int 2658static int
2618i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) 2659i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2660 unsigned alignment,
2661 bool map_and_fenceable)
2619{ 2662{
2620 struct drm_device *dev = obj->dev; 2663 struct drm_device *dev = obj->base.dev;
2621 drm_i915_private_t *dev_priv = dev->dev_private; 2664 drm_i915_private_t *dev_priv = dev->dev_private;
2622 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2623 struct drm_mm_node *free_space; 2665 struct drm_mm_node *free_space;
2624 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; 2666 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2667 u32 size, fence_size, fence_alignment, unfenced_alignment;
2668 bool mappable, fenceable;
2625 int ret; 2669 int ret;
2626 2670
2627 if (obj_priv->madv != I915_MADV_WILLNEED) { 2671 if (obj->madv != I915_MADV_WILLNEED) {
2628 DRM_ERROR("Attempting to bind a purgeable object\n"); 2672 DRM_ERROR("Attempting to bind a purgeable object\n");
2629 return -EINVAL; 2673 return -EINVAL;
2630 } 2674 }
2631 2675
2676 fence_size = i915_gem_get_gtt_size(obj);
2677 fence_alignment = i915_gem_get_gtt_alignment(obj);
2678 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
2679
2632 if (alignment == 0) 2680 if (alignment == 0)
2633 alignment = i915_gem_get_gtt_alignment(obj); 2681 alignment = map_and_fenceable ? fence_alignment :
2634 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { 2682 unfenced_alignment;
2683 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2635 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2684 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2636 return -EINVAL; 2685 return -EINVAL;
2637 } 2686 }
2638 2687
2688 size = map_and_fenceable ? fence_size : obj->base.size;
2689
2639 /* If the object is bigger than the entire aperture, reject it early 2690 /* If the object is bigger than the entire aperture, reject it early
2640 * before evicting everything in a vain attempt to find space. 2691 * before evicting everything in a vain attempt to find space.
2641 */ 2692 */
2642 if (obj->size > dev_priv->mm.gtt_total) { 2693 if (obj->base.size >
2694 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2643 DRM_ERROR("Attempting to bind an object larger than the aperture\n"); 2695 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2644 return -E2BIG; 2696 return -E2BIG;
2645 } 2697 }
2646 2698
2647 search_free: 2699 search_free:
2648 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2700 if (map_and_fenceable)
2649 obj->size, alignment, 0); 2701 free_space =
2650 if (free_space != NULL) 2702 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2651 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2703 size, alignment, 0,
2652 alignment); 2704 dev_priv->mm.gtt_mappable_end,
2653 if (obj_priv->gtt_space == NULL) { 2705 0);
2706 else
2707 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2708 size, alignment, 0);
2709
2710 if (free_space != NULL) {
2711 if (map_and_fenceable)
2712 obj->gtt_space =
2713 drm_mm_get_block_range_generic(free_space,
2714 size, alignment, 0,
2715 dev_priv->mm.gtt_mappable_end,
2716 0);
2717 else
2718 obj->gtt_space =
2719 drm_mm_get_block(free_space, size, alignment);
2720 }
2721 if (obj->gtt_space == NULL) {
2654 /* If the gtt is empty and we're still having trouble 2722 /* If the gtt is empty and we're still having trouble
2655 * fitting our object in, we're out of memory. 2723 * fitting our object in, we're out of memory.
2656 */ 2724 */
2657 ret = i915_gem_evict_something(dev, obj->size, alignment); 2725 ret = i915_gem_evict_something(dev, size, alignment,
2726 map_and_fenceable);
2658 if (ret) 2727 if (ret)
2659 return ret; 2728 return ret;
2660 2729
2661 goto search_free; 2730 goto search_free;
2662 } 2731 }
2663 2732
2664 ret = i915_gem_object_get_pages(obj, gfpmask); 2733 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2665 if (ret) { 2734 if (ret) {
2666 drm_mm_put_block(obj_priv->gtt_space); 2735 drm_mm_put_block(obj->gtt_space);
2667 obj_priv->gtt_space = NULL; 2736 obj->gtt_space = NULL;
2668 2737
2669 if (ret == -ENOMEM) { 2738 if (ret == -ENOMEM) {
2670 /* first try to clear up some space from the GTT */ 2739 /* first try to clear up some space from the GTT */
2671 ret = i915_gem_evict_something(dev, obj->size, 2740 ret = i915_gem_evict_something(dev, size,
2672 alignment); 2741 alignment,
2742 map_and_fenceable);
2673 if (ret) { 2743 if (ret) {
2674 /* now try to shrink everyone else */ 2744 /* now try to shrink everyone else */
2675 if (gfpmask) { 2745 if (gfpmask) {
@@ -2686,126 +2756,113 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2686 return ret; 2756 return ret;
2687 } 2757 }
2688 2758
2689 /* Create an AGP memory structure pointing at our pages, and bind it 2759 ret = i915_gem_gtt_bind_object(obj);
2690 * into the GTT. 2760 if (ret) {
2691 */ 2761 i915_gem_object_put_pages_gtt(obj);
2692 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2762 drm_mm_put_block(obj->gtt_space);
2693 obj_priv->pages, 2763 obj->gtt_space = NULL;
2694 obj->size >> PAGE_SHIFT, 2764
2695 obj_priv->gtt_space->start, 2765 ret = i915_gem_evict_something(dev, size,
2696 obj_priv->agp_type); 2766 alignment, map_and_fenceable);
2697 if (obj_priv->agp_mem == NULL) {
2698 i915_gem_object_put_pages(obj);
2699 drm_mm_put_block(obj_priv->gtt_space);
2700 obj_priv->gtt_space = NULL;
2701
2702 ret = i915_gem_evict_something(dev, obj->size, alignment);
2703 if (ret) 2767 if (ret)
2704 return ret; 2768 return ret;
2705 2769
2706 goto search_free; 2770 goto search_free;
2707 } 2771 }
2708 2772
2709 /* keep track of bounds object by adding it to the inactive list */ 2773 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2710 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); 2774 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2711 i915_gem_info_add_gtt(dev_priv, obj->size);
2712 2775
2713 /* Assert that the object is not currently in any GPU domain. As it 2776 /* Assert that the object is not currently in any GPU domain. As it
2714 * wasn't in the GTT, there shouldn't be any way it could have been in 2777 * wasn't in the GTT, there shouldn't be any way it could have been in
2715 * a GPU cache 2778 * a GPU cache
2716 */ 2779 */
2717 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2780 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2718 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2781 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2719 2782
2720 obj_priv->gtt_offset = obj_priv->gtt_space->start; 2783 obj->gtt_offset = obj->gtt_space->start;
2721 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2722 2784
2785 fenceable =
2786 obj->gtt_space->size == fence_size &&
2787 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2788
2789 mappable =
2790 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2791
2792 obj->map_and_fenceable = mappable && fenceable;
2793
2794 trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
2723 return 0; 2795 return 0;
2724} 2796}
2725 2797
2726void 2798void
2727i915_gem_clflush_object(struct drm_gem_object *obj) 2799i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2728{ 2800{
2729 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2730
2731 /* If we don't have a page list set up, then we're not pinned 2801 /* If we don't have a page list set up, then we're not pinned
2732 * to GPU, and we can ignore the cache flush because it'll happen 2802 * to GPU, and we can ignore the cache flush because it'll happen
2733 * again at bind time. 2803 * again at bind time.
2734 */ 2804 */
2735 if (obj_priv->pages == NULL) 2805 if (obj->pages == NULL)
2736 return; 2806 return;
2737 2807
2738 trace_i915_gem_object_clflush(obj); 2808 trace_i915_gem_object_clflush(obj);
2739 2809
2740 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); 2810 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2741} 2811}
2742 2812
2743/** Flushes any GPU write domain for the object if it's dirty. */ 2813/** Flushes any GPU write domain for the object if it's dirty. */
2744static int 2814static void
2745i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, 2815i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2746 bool pipelined)
2747{ 2816{
2748 struct drm_device *dev = obj->dev; 2817 struct drm_device *dev = obj->base.dev;
2749 uint32_t old_write_domain;
2750 2818
2751 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) 2819 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2752 return 0; 2820 return;
2753 2821
2754 /* Queue the GPU write cache flushing we need. */ 2822 /* Queue the GPU write cache flushing we need. */
2755 old_write_domain = obj->write_domain; 2823 i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
2756 i915_gem_flush_ring(dev, NULL, 2824 BUG_ON(obj->base.write_domain);
2757 to_intel_bo(obj)->ring,
2758 0, obj->write_domain);
2759 BUG_ON(obj->write_domain);
2760
2761 trace_i915_gem_object_change_domain(obj,
2762 obj->read_domains,
2763 old_write_domain);
2764
2765 if (pipelined)
2766 return 0;
2767
2768 return i915_gem_object_wait_rendering(obj, true);
2769} 2825}
2770 2826
2771/** Flushes the GTT write domain for the object if it's dirty. */ 2827/** Flushes the GTT write domain for the object if it's dirty. */
2772static void 2828static void
2773i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) 2829i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2774{ 2830{
2775 uint32_t old_write_domain; 2831 uint32_t old_write_domain;
2776 2832
2777 if (obj->write_domain != I915_GEM_DOMAIN_GTT) 2833 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2778 return; 2834 return;
2779 2835
2780 /* No actual flushing is required for the GTT write domain. Writes 2836 /* No actual flushing is required for the GTT write domain. Writes
2781 * to it immediately go to main memory as far as we know, so there's 2837 * to it immediately go to main memory as far as we know, so there's
2782 * no chipset flush. It also doesn't land in render cache. 2838 * no chipset flush. It also doesn't land in render cache.
2783 */ 2839 */
2784 old_write_domain = obj->write_domain; 2840 i915_gem_release_mmap(obj);
2785 obj->write_domain = 0; 2841
2842 old_write_domain = obj->base.write_domain;
2843 obj->base.write_domain = 0;
2786 2844
2787 trace_i915_gem_object_change_domain(obj, 2845 trace_i915_gem_object_change_domain(obj,
2788 obj->read_domains, 2846 obj->base.read_domains,
2789 old_write_domain); 2847 old_write_domain);
2790} 2848}
2791 2849
2792/** Flushes the CPU write domain for the object if it's dirty. */ 2850/** Flushes the CPU write domain for the object if it's dirty. */
2793static void 2851static void
2794i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) 2852i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2795{ 2853{
2796 struct drm_device *dev = obj->dev;
2797 uint32_t old_write_domain; 2854 uint32_t old_write_domain;
2798 2855
2799 if (obj->write_domain != I915_GEM_DOMAIN_CPU) 2856 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2800 return; 2857 return;
2801 2858
2802 i915_gem_clflush_object(obj); 2859 i915_gem_clflush_object(obj);
2803 drm_agp_chipset_flush(dev); 2860 intel_gtt_chipset_flush();
2804 old_write_domain = obj->write_domain; 2861 old_write_domain = obj->base.write_domain;
2805 obj->write_domain = 0; 2862 obj->base.write_domain = 0;
2806 2863
2807 trace_i915_gem_object_change_domain(obj, 2864 trace_i915_gem_object_change_domain(obj,
2808 obj->read_domains, 2865 obj->base.read_domains,
2809 old_write_domain); 2866 old_write_domain);
2810} 2867}
2811 2868
@@ -2816,40 +2873,36 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2816 * flushes to occur. 2873 * flushes to occur.
2817 */ 2874 */
2818int 2875int
2819i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) 2876i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2820{ 2877{
2821 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2822 uint32_t old_write_domain, old_read_domains; 2878 uint32_t old_write_domain, old_read_domains;
2823 int ret; 2879 int ret;
2824 2880
2825 /* Not valid to be called on unbound objects. */ 2881 /* Not valid to be called on unbound objects. */
2826 if (obj_priv->gtt_space == NULL) 2882 if (obj->gtt_space == NULL)
2827 return -EINVAL; 2883 return -EINVAL;
2828 2884
2829 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 2885 i915_gem_object_flush_gpu_write_domain(obj);
2830 if (ret != 0) 2886 if (obj->pending_gpu_write || write) {
2831 return ret;
2832
2833 i915_gem_object_flush_cpu_write_domain(obj);
2834
2835 if (write) {
2836 ret = i915_gem_object_wait_rendering(obj, true); 2887 ret = i915_gem_object_wait_rendering(obj, true);
2837 if (ret) 2888 if (ret)
2838 return ret; 2889 return ret;
2839 } 2890 }
2840 2891
2841 old_write_domain = obj->write_domain; 2892 i915_gem_object_flush_cpu_write_domain(obj);
2842 old_read_domains = obj->read_domains; 2893
2894 old_write_domain = obj->base.write_domain;
2895 old_read_domains = obj->base.read_domains;
2843 2896
2844 /* It should now be out of any other write domains, and we can update 2897 /* It should now be out of any other write domains, and we can update
2845 * the domain values for our changes. 2898 * the domain values for our changes.
2846 */ 2899 */
2847 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 2900 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2848 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2901 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2849 if (write) { 2902 if (write) {
2850 obj->read_domains = I915_GEM_DOMAIN_GTT; 2903 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2851 obj->write_domain = I915_GEM_DOMAIN_GTT; 2904 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2852 obj_priv->dirty = 1; 2905 obj->dirty = 1;
2853 } 2906 }
2854 2907
2855 trace_i915_gem_object_change_domain(obj, 2908 trace_i915_gem_object_change_domain(obj,
@@ -2864,23 +2917,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2864 * wait, as in modesetting process we're not supposed to be interrupted. 2917 * wait, as in modesetting process we're not supposed to be interrupted.
2865 */ 2918 */
2866int 2919int
2867i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, 2920i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
2868 bool pipelined) 2921 struct intel_ring_buffer *pipelined)
2869{ 2922{
2870 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2871 uint32_t old_read_domains; 2923 uint32_t old_read_domains;
2872 int ret; 2924 int ret;
2873 2925
2874 /* Not valid to be called on unbound objects. */ 2926 /* Not valid to be called on unbound objects. */
2875 if (obj_priv->gtt_space == NULL) 2927 if (obj->gtt_space == NULL)
2876 return -EINVAL; 2928 return -EINVAL;
2877 2929
2878 ret = i915_gem_object_flush_gpu_write_domain(obj, true); 2930 i915_gem_object_flush_gpu_write_domain(obj);
2879 if (ret)
2880 return ret;
2881 2931
2882 /* Currently, we are always called from an non-interruptible context. */ 2932 /* Currently, we are always called from an non-interruptible context. */
2883 if (!pipelined) { 2933 if (pipelined != obj->ring) {
2884 ret = i915_gem_object_wait_rendering(obj, false); 2934 ret = i915_gem_object_wait_rendering(obj, false);
2885 if (ret) 2935 if (ret)
2886 return ret; 2936 return ret;
@@ -2888,12 +2938,12 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2888 2938
2889 i915_gem_object_flush_cpu_write_domain(obj); 2939 i915_gem_object_flush_cpu_write_domain(obj);
2890 2940
2891 old_read_domains = obj->read_domains; 2941 old_read_domains = obj->base.read_domains;
2892 obj->read_domains |= I915_GEM_DOMAIN_GTT; 2942 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2893 2943
2894 trace_i915_gem_object_change_domain(obj, 2944 trace_i915_gem_object_change_domain(obj,
2895 old_read_domains, 2945 old_read_domains,
2896 obj->write_domain); 2946 obj->base.write_domain);
2897 2947
2898 return 0; 2948 return 0;
2899} 2949}
@@ -2906,10 +2956,10 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2906 return 0; 2956 return 0;
2907 2957
2908 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) 2958 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2909 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring, 2959 i915_gem_flush_ring(obj->base.dev, obj->ring,
2910 0, obj->base.write_domain); 2960 0, obj->base.write_domain);
2911 2961
2912 return i915_gem_object_wait_rendering(&obj->base, interruptible); 2962 return i915_gem_object_wait_rendering(obj, interruptible);
2913} 2963}
2914 2964
2915/** 2965/**
@@ -2919,13 +2969,14 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2919 * flushes to occur. 2969 * flushes to occur.
2920 */ 2970 */
2921static int 2971static int
2922i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) 2972i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2923{ 2973{
2924 uint32_t old_write_domain, old_read_domains; 2974 uint32_t old_write_domain, old_read_domains;
2925 int ret; 2975 int ret;
2926 2976
2927 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 2977 i915_gem_object_flush_gpu_write_domain(obj);
2928 if (ret != 0) 2978 ret = i915_gem_object_wait_rendering(obj, true);
2979 if (ret)
2929 return ret; 2980 return ret;
2930 2981
2931 i915_gem_object_flush_gtt_write_domain(obj); 2982 i915_gem_object_flush_gtt_write_domain(obj);
@@ -2935,33 +2986,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2935 */ 2986 */
2936 i915_gem_object_set_to_full_cpu_read_domain(obj); 2987 i915_gem_object_set_to_full_cpu_read_domain(obj);
2937 2988
2938 if (write) { 2989 old_write_domain = obj->base.write_domain;
2939 ret = i915_gem_object_wait_rendering(obj, true); 2990 old_read_domains = obj->base.read_domains;
2940 if (ret)
2941 return ret;
2942 }
2943
2944 old_write_domain = obj->write_domain;
2945 old_read_domains = obj->read_domains;
2946 2991
2947 /* Flush the CPU cache if it's still invalid. */ 2992 /* Flush the CPU cache if it's still invalid. */
2948 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { 2993 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2949 i915_gem_clflush_object(obj); 2994 i915_gem_clflush_object(obj);
2950 2995
2951 obj->read_domains |= I915_GEM_DOMAIN_CPU; 2996 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2952 } 2997 }
2953 2998
2954 /* It should now be out of any other write domains, and we can update 2999 /* It should now be out of any other write domains, and we can update
2955 * the domain values for our changes. 3000 * the domain values for our changes.
2956 */ 3001 */
2957 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3002 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2958 3003
2959 /* If we're writing through the CPU, then the GPU read domains will 3004 /* If we're writing through the CPU, then the GPU read domains will
2960 * need to be invalidated at next use. 3005 * need to be invalidated at next use.
2961 */ 3006 */
2962 if (write) { 3007 if (write) {
2963 obj->read_domains = I915_GEM_DOMAIN_CPU; 3008 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2964 obj->write_domain = I915_GEM_DOMAIN_CPU; 3009 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2965 } 3010 }
2966 3011
2967 trace_i915_gem_object_change_domain(obj, 3012 trace_i915_gem_object_change_domain(obj,
@@ -2971,184 +3016,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2971 return 0; 3016 return 0;
2972} 3017}
2973 3018
2974/*
2975 * Set the next domain for the specified object. This
2976 * may not actually perform the necessary flushing/invaliding though,
2977 * as that may want to be batched with other set_domain operations
2978 *
2979 * This is (we hope) the only really tricky part of gem. The goal
2980 * is fairly simple -- track which caches hold bits of the object
2981 * and make sure they remain coherent. A few concrete examples may
2982 * help to explain how it works. For shorthand, we use the notation
2983 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2984 * a pair of read and write domain masks.
2985 *
2986 * Case 1: the batch buffer
2987 *
2988 * 1. Allocated
2989 * 2. Written by CPU
2990 * 3. Mapped to GTT
2991 * 4. Read by GPU
2992 * 5. Unmapped from GTT
2993 * 6. Freed
2994 *
2995 * Let's take these a step at a time
2996 *
2997 * 1. Allocated
2998 * Pages allocated from the kernel may still have
2999 * cache contents, so we set them to (CPU, CPU) always.
3000 * 2. Written by CPU (using pwrite)
3001 * The pwrite function calls set_domain (CPU, CPU) and
3002 * this function does nothing (as nothing changes)
3003 * 3. Mapped by GTT
3004 * This function asserts that the object is not
3005 * currently in any GPU-based read or write domains
3006 * 4. Read by GPU
3007 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3008 * As write_domain is zero, this function adds in the
3009 * current read domains (CPU+COMMAND, 0).
3010 * flush_domains is set to CPU.
3011 * invalidate_domains is set to COMMAND
3012 * clflush is run to get data out of the CPU caches
3013 * then i915_dev_set_domain calls i915_gem_flush to
3014 * emit an MI_FLUSH and drm_agp_chipset_flush
3015 * 5. Unmapped from GTT
3016 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3017 * flush_domains and invalidate_domains end up both zero
3018 * so no flushing/invalidating happens
3019 * 6. Freed
3020 * yay, done
3021 *
3022 * Case 2: The shared render buffer
3023 *
3024 * 1. Allocated
3025 * 2. Mapped to GTT
3026 * 3. Read/written by GPU
3027 * 4. set_domain to (CPU,CPU)
3028 * 5. Read/written by CPU
3029 * 6. Read/written by GPU
3030 *
3031 * 1. Allocated
3032 * Same as last example, (CPU, CPU)
3033 * 2. Mapped to GTT
3034 * Nothing changes (assertions find that it is not in the GPU)
3035 * 3. Read/written by GPU
3036 * execbuffer calls set_domain (RENDER, RENDER)
3037 * flush_domains gets CPU
3038 * invalidate_domains gets GPU
3039 * clflush (obj)
3040 * MI_FLUSH and drm_agp_chipset_flush
3041 * 4. set_domain (CPU, CPU)
3042 * flush_domains gets GPU
3043 * invalidate_domains gets CPU
3044 * wait_rendering (obj) to make sure all drawing is complete.
3045 * This will include an MI_FLUSH to get the data from GPU
3046 * to memory
3047 * clflush (obj) to invalidate the CPU cache
3048 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3049 * 5. Read/written by CPU
3050 * cache lines are loaded and dirtied
3051 * 6. Read written by GPU
3052 * Same as last GPU access
3053 *
3054 * Case 3: The constant buffer
3055 *
3056 * 1. Allocated
3057 * 2. Written by CPU
3058 * 3. Read by GPU
3059 * 4. Updated (written) by CPU again
3060 * 5. Read by GPU
3061 *
3062 * 1. Allocated
3063 * (CPU, CPU)
3064 * 2. Written by CPU
3065 * (CPU, CPU)
3066 * 3. Read by GPU
3067 * (CPU+RENDER, 0)
3068 * flush_domains = CPU
3069 * invalidate_domains = RENDER
3070 * clflush (obj)
3071 * MI_FLUSH
3072 * drm_agp_chipset_flush
3073 * 4. Updated (written) by CPU again
3074 * (CPU, CPU)
3075 * flush_domains = 0 (no previous write domain)
3076 * invalidate_domains = 0 (no new read domains)
3077 * 5. Read by GPU
3078 * (CPU+RENDER, 0)
3079 * flush_domains = CPU
3080 * invalidate_domains = RENDER
3081 * clflush (obj)
3082 * MI_FLUSH
3083 * drm_agp_chipset_flush
3084 */
3085static void
3086i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3087 struct intel_ring_buffer *ring)
3088{
3089 struct drm_device *dev = obj->dev;
3090 struct drm_i915_private *dev_priv = dev->dev_private;
3091 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3092 uint32_t invalidate_domains = 0;
3093 uint32_t flush_domains = 0;
3094 uint32_t old_read_domains;
3095
3096 intel_mark_busy(dev, obj);
3097
3098 /*
3099 * If the object isn't moving to a new write domain,
3100 * let the object stay in multiple read domains
3101 */
3102 if (obj->pending_write_domain == 0)
3103 obj->pending_read_domains |= obj->read_domains;
3104 else
3105 obj_priv->dirty = 1;
3106
3107 /*
3108 * Flush the current write domain if
3109 * the new read domains don't match. Invalidate
3110 * any read domains which differ from the old
3111 * write domain
3112 */
3113 if (obj->write_domain &&
3114 (obj->write_domain != obj->pending_read_domains ||
3115 obj_priv->ring != ring)) {
3116 flush_domains |= obj->write_domain;
3117 invalidate_domains |=
3118 obj->pending_read_domains & ~obj->write_domain;
3119 }
3120 /*
3121 * Invalidate any read caches which may have
3122 * stale data. That is, any new read domains.
3123 */
3124 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3125 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3126 i915_gem_clflush_object(obj);
3127
3128 old_read_domains = obj->read_domains;
3129
3130 /* The actual obj->write_domain will be updated with
3131 * pending_write_domain after we emit the accumulated flush for all
3132 * of our domain changes in execbuffers (which clears objects'
3133 * write_domains). So if we have a current write domain that we
3134 * aren't changing, set pending_write_domain to that.
3135 */
3136 if (flush_domains == 0 && obj->pending_write_domain == 0)
3137 obj->pending_write_domain = obj->write_domain;
3138 obj->read_domains = obj->pending_read_domains;
3139
3140 dev->invalidate_domains |= invalidate_domains;
3141 dev->flush_domains |= flush_domains;
3142 if (flush_domains & I915_GEM_GPU_DOMAINS)
3143 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3144 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3145 dev_priv->mm.flush_rings |= ring->id;
3146
3147 trace_i915_gem_object_change_domain(obj,
3148 old_read_domains,
3149 obj->write_domain);
3150}
3151
3152/** 3019/**
3153 * Moves the object from a partially CPU read to a full one. 3020 * Moves the object from a partially CPU read to a full one.
3154 * 3021 *
@@ -3156,30 +3023,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3156 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). 3023 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3157 */ 3024 */
3158static void 3025static void
3159i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) 3026i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3160{ 3027{
3161 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3028 if (!obj->page_cpu_valid)
3162
3163 if (!obj_priv->page_cpu_valid)
3164 return; 3029 return;
3165 3030
3166 /* If we're partially in the CPU read domain, finish moving it in. 3031 /* If we're partially in the CPU read domain, finish moving it in.
3167 */ 3032 */
3168 if (obj->read_domains & I915_GEM_DOMAIN_CPU) { 3033 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3169 int i; 3034 int i;
3170 3035
3171 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { 3036 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3172 if (obj_priv->page_cpu_valid[i]) 3037 if (obj->page_cpu_valid[i])
3173 continue; 3038 continue;
3174 drm_clflush_pages(obj_priv->pages + i, 1); 3039 drm_clflush_pages(obj->pages + i, 1);
3175 } 3040 }
3176 } 3041 }
3177 3042
3178 /* Free the page_cpu_valid mappings which are now stale, whether 3043 /* Free the page_cpu_valid mappings which are now stale, whether
3179 * or not we've got I915_GEM_DOMAIN_CPU. 3044 * or not we've got I915_GEM_DOMAIN_CPU.
3180 */ 3045 */
3181 kfree(obj_priv->page_cpu_valid); 3046 kfree(obj->page_cpu_valid);
3182 obj_priv->page_cpu_valid = NULL; 3047 obj->page_cpu_valid = NULL;
3183} 3048}
3184 3049
3185/** 3050/**
@@ -3195,354 +3060,62 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3195 * flushes to occur. 3060 * flushes to occur.
3196 */ 3061 */
3197static int 3062static int
3198i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, 3063i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3199 uint64_t offset, uint64_t size) 3064 uint64_t offset, uint64_t size)
3200{ 3065{
3201 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3202 uint32_t old_read_domains; 3066 uint32_t old_read_domains;
3203 int i, ret; 3067 int i, ret;
3204 3068
3205 if (offset == 0 && size == obj->size) 3069 if (offset == 0 && size == obj->base.size)
3206 return i915_gem_object_set_to_cpu_domain(obj, 0); 3070 return i915_gem_object_set_to_cpu_domain(obj, 0);
3207 3071
3208 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 3072 i915_gem_object_flush_gpu_write_domain(obj);
3209 if (ret != 0) 3073 ret = i915_gem_object_wait_rendering(obj, true);
3074 if (ret)
3210 return ret; 3075 return ret;
3076
3211 i915_gem_object_flush_gtt_write_domain(obj); 3077 i915_gem_object_flush_gtt_write_domain(obj);
3212 3078
3213 /* If we're already fully in the CPU read domain, we're done. */ 3079 /* If we're already fully in the CPU read domain, we're done. */
3214 if (obj_priv->page_cpu_valid == NULL && 3080 if (obj->page_cpu_valid == NULL &&
3215 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) 3081 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3216 return 0; 3082 return 0;
3217 3083
3218 /* Otherwise, create/clear the per-page CPU read domain flag if we're 3084 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3219 * newly adding I915_GEM_DOMAIN_CPU 3085 * newly adding I915_GEM_DOMAIN_CPU
3220 */ 3086 */
3221 if (obj_priv->page_cpu_valid == NULL) { 3087 if (obj->page_cpu_valid == NULL) {
3222 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE, 3088 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3223 GFP_KERNEL); 3089 GFP_KERNEL);
3224 if (obj_priv->page_cpu_valid == NULL) 3090 if (obj->page_cpu_valid == NULL)
3225 return -ENOMEM; 3091 return -ENOMEM;
3226 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) 3092 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3227 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); 3093 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3228 3094
3229 /* Flush the cache on any pages that are still invalid from the CPU's 3095 /* Flush the cache on any pages that are still invalid from the CPU's
3230 * perspective. 3096 * perspective.
3231 */ 3097 */
3232 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 3098 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3233 i++) { 3099 i++) {
3234 if (obj_priv->page_cpu_valid[i]) 3100 if (obj->page_cpu_valid[i])
3235 continue; 3101 continue;
3236 3102
3237 drm_clflush_pages(obj_priv->pages + i, 1); 3103 drm_clflush_pages(obj->pages + i, 1);
3238 3104
3239 obj_priv->page_cpu_valid[i] = 1; 3105 obj->page_cpu_valid[i] = 1;
3240 } 3106 }
3241 3107
3242 /* It should now be out of any other write domains, and we can update 3108 /* It should now be out of any other write domains, and we can update
3243 * the domain values for our changes. 3109 * the domain values for our changes.
3244 */ 3110 */
3245 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 3111 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3246 3112
3247 old_read_domains = obj->read_domains; 3113 old_read_domains = obj->base.read_domains;
3248 obj->read_domains |= I915_GEM_DOMAIN_CPU; 3114 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3249 3115
3250 trace_i915_gem_object_change_domain(obj, 3116 trace_i915_gem_object_change_domain(obj,
3251 old_read_domains, 3117 old_read_domains,
3252 obj->write_domain); 3118 obj->base.write_domain);
3253
3254 return 0;
3255}
3256
3257/**
3258 * Pin an object to the GTT and evaluate the relocations landing in it.
3259 */
3260static int
3261i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3262 struct drm_file *file_priv,
3263 struct drm_i915_gem_exec_object2 *entry)
3264{
3265 struct drm_device *dev = obj->base.dev;
3266 drm_i915_private_t *dev_priv = dev->dev_private;
3267 struct drm_i915_gem_relocation_entry __user *user_relocs;
3268 struct drm_gem_object *target_obj = NULL;
3269 uint32_t target_handle = 0;
3270 int i, ret = 0;
3271
3272 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3273 for (i = 0; i < entry->relocation_count; i++) {
3274 struct drm_i915_gem_relocation_entry reloc;
3275 uint32_t target_offset;
3276
3277 if (__copy_from_user_inatomic(&reloc,
3278 user_relocs+i,
3279 sizeof(reloc))) {
3280 ret = -EFAULT;
3281 break;
3282 }
3283
3284 if (reloc.target_handle != target_handle) {
3285 drm_gem_object_unreference(target_obj);
3286
3287 target_obj = drm_gem_object_lookup(dev, file_priv,
3288 reloc.target_handle);
3289 if (target_obj == NULL) {
3290 ret = -ENOENT;
3291 break;
3292 }
3293
3294 target_handle = reloc.target_handle;
3295 }
3296 target_offset = to_intel_bo(target_obj)->gtt_offset;
3297
3298#if WATCH_RELOC
3299 DRM_INFO("%s: obj %p offset %08x target %d "
3300 "read %08x write %08x gtt %08x "
3301 "presumed %08x delta %08x\n",
3302 __func__,
3303 obj,
3304 (int) reloc.offset,
3305 (int) reloc.target_handle,
3306 (int) reloc.read_domains,
3307 (int) reloc.write_domain,
3308 (int) target_offset,
3309 (int) reloc.presumed_offset,
3310 reloc.delta);
3311#endif
3312
3313 /* The target buffer should have appeared before us in the
3314 * exec_object list, so it should have a GTT space bound by now.
3315 */
3316 if (target_offset == 0) {
3317 DRM_ERROR("No GTT space found for object %d\n",
3318 reloc.target_handle);
3319 ret = -EINVAL;
3320 break;
3321 }
3322
3323 /* Validate that the target is in a valid r/w GPU domain */
3324 if (reloc.write_domain & (reloc.write_domain - 1)) {
3325 DRM_ERROR("reloc with multiple write domains: "
3326 "obj %p target %d offset %d "
3327 "read %08x write %08x",
3328 obj, reloc.target_handle,
3329 (int) reloc.offset,
3330 reloc.read_domains,
3331 reloc.write_domain);
3332 ret = -EINVAL;
3333 break;
3334 }
3335 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3336 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3337 DRM_ERROR("reloc with read/write CPU domains: "
3338 "obj %p target %d offset %d "
3339 "read %08x write %08x",
3340 obj, reloc.target_handle,
3341 (int) reloc.offset,
3342 reloc.read_domains,
3343 reloc.write_domain);
3344 ret = -EINVAL;
3345 break;
3346 }
3347 if (reloc.write_domain && target_obj->pending_write_domain &&
3348 reloc.write_domain != target_obj->pending_write_domain) {
3349 DRM_ERROR("Write domain conflict: "
3350 "obj %p target %d offset %d "
3351 "new %08x old %08x\n",
3352 obj, reloc.target_handle,
3353 (int) reloc.offset,
3354 reloc.write_domain,
3355 target_obj->pending_write_domain);
3356 ret = -EINVAL;
3357 break;
3358 }
3359
3360 target_obj->pending_read_domains |= reloc.read_domains;
3361 target_obj->pending_write_domain |= reloc.write_domain;
3362
3363 /* If the relocation already has the right value in it, no
3364 * more work needs to be done.
3365 */
3366 if (target_offset == reloc.presumed_offset)
3367 continue;
3368
3369 /* Check that the relocation address is valid... */
3370 if (reloc.offset > obj->base.size - 4) {
3371 DRM_ERROR("Relocation beyond object bounds: "
3372 "obj %p target %d offset %d size %d.\n",
3373 obj, reloc.target_handle,
3374 (int) reloc.offset, (int) obj->base.size);
3375 ret = -EINVAL;
3376 break;
3377 }
3378 if (reloc.offset & 3) {
3379 DRM_ERROR("Relocation not 4-byte aligned: "
3380 "obj %p target %d offset %d.\n",
3381 obj, reloc.target_handle,
3382 (int) reloc.offset);
3383 ret = -EINVAL;
3384 break;
3385 }
3386
3387 /* and points to somewhere within the target object. */
3388 if (reloc.delta >= target_obj->size) {
3389 DRM_ERROR("Relocation beyond target object bounds: "
3390 "obj %p target %d delta %d size %d.\n",
3391 obj, reloc.target_handle,
3392 (int) reloc.delta, (int) target_obj->size);
3393 ret = -EINVAL;
3394 break;
3395 }
3396
3397 reloc.delta += target_offset;
3398 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3399 uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3400 char *vaddr;
3401
3402 vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3403 *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3404 kunmap_atomic(vaddr);
3405 } else {
3406 uint32_t __iomem *reloc_entry;
3407 void __iomem *reloc_page;
3408
3409 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3410 if (ret)
3411 break;
3412
3413 /* Map the page containing the relocation we're going to perform. */
3414 reloc.offset += obj->gtt_offset;
3415 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3416 reloc.offset & PAGE_MASK);
3417 reloc_entry = (uint32_t __iomem *)
3418 (reloc_page + (reloc.offset & ~PAGE_MASK));
3419 iowrite32(reloc.delta, reloc_entry);
3420 io_mapping_unmap_atomic(reloc_page);
3421 }
3422
3423 /* and update the user's relocation entry */
3424 reloc.presumed_offset = target_offset;
3425 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3426 &reloc.presumed_offset,
3427 sizeof(reloc.presumed_offset))) {
3428 ret = -EFAULT;
3429 break;
3430 }
3431 }
3432
3433 drm_gem_object_unreference(target_obj);
3434 return ret;
3435}
3436
3437static int
3438i915_gem_execbuffer_pin(struct drm_device *dev,
3439 struct drm_file *file,
3440 struct drm_gem_object **object_list,
3441 struct drm_i915_gem_exec_object2 *exec_list,
3442 int count)
3443{
3444 struct drm_i915_private *dev_priv = dev->dev_private;
3445 int ret, i, retry;
3446
3447 /* attempt to pin all of the buffers into the GTT */
3448 for (retry = 0; retry < 2; retry++) {
3449 ret = 0;
3450 for (i = 0; i < count; i++) {
3451 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3452 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3453 bool need_fence =
3454 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3455 obj->tiling_mode != I915_TILING_NONE;
3456
3457 /* Check fence reg constraints and rebind if necessary */
3458 if (need_fence &&
3459 !i915_gem_object_fence_offset_ok(&obj->base,
3460 obj->tiling_mode)) {
3461 ret = i915_gem_object_unbind(&obj->base);
3462 if (ret)
3463 break;
3464 }
3465
3466 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3467 if (ret)
3468 break;
3469
3470 /*
3471 * Pre-965 chips need a fence register set up in order
3472 * to properly handle blits to/from tiled surfaces.
3473 */
3474 if (need_fence) {
3475 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3476 if (ret) {
3477 i915_gem_object_unpin(&obj->base);
3478 break;
3479 }
3480
3481 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3482 }
3483
3484 entry->offset = obj->gtt_offset;
3485 }
3486
3487 while (i--)
3488 i915_gem_object_unpin(object_list[i]);
3489
3490 if (ret == 0)
3491 break;
3492
3493 if (ret != -ENOSPC || retry)
3494 return ret;
3495
3496 ret = i915_gem_evict_everything(dev);
3497 if (ret)
3498 return ret;
3499 }
3500
3501 return 0;
3502}
3503
3504static int
3505i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3506 struct drm_file *file,
3507 struct intel_ring_buffer *ring,
3508 struct drm_gem_object **objects,
3509 int count)
3510{
3511 struct drm_i915_private *dev_priv = dev->dev_private;
3512 int ret, i;
3513
3514 /* Zero the global flush/invalidate flags. These
3515 * will be modified as new domains are computed
3516 * for each object
3517 */
3518 dev->invalidate_domains = 0;
3519 dev->flush_domains = 0;
3520 dev_priv->mm.flush_rings = 0;
3521 for (i = 0; i < count; i++)
3522 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3523
3524 if (dev->invalidate_domains | dev->flush_domains) {
3525#if WATCH_EXEC
3526 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3527 __func__,
3528 dev->invalidate_domains,
3529 dev->flush_domains);
3530#endif
3531 i915_gem_flush(dev, file,
3532 dev->invalidate_domains,
3533 dev->flush_domains,
3534 dev_priv->mm.flush_rings);
3535 }
3536
3537 for (i = 0; i < count; i++) {
3538 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3539 /* XXX replace with semaphores */
3540 if (obj->ring && ring != obj->ring) {
3541 ret = i915_gem_object_wait_rendering(&obj->base, true);
3542 if (ret)
3543 return ret;
3544 }
3545 }
3546 3119
3547 return 0; 3120 return 0;
3548} 3121}
@@ -3582,586 +3155,129 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3582 return 0; 3155 return 0;
3583 3156
3584 ret = 0; 3157 ret = 0;
3585 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { 3158 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3586 /* And wait for the seqno passing without holding any locks and 3159 /* And wait for the seqno passing without holding any locks and
3587 * causing extra latency for others. This is safe as the irq 3160 * causing extra latency for others. This is safe as the irq
3588 * generation is designed to be run atomically and so is 3161 * generation is designed to be run atomically and so is
3589 * lockless. 3162 * lockless.
3590 */ 3163 */
3591 ring->user_irq_get(dev, ring); 3164 if (ring->irq_get(ring)) {
3592 ret = wait_event_interruptible(ring->irq_queue, 3165 ret = wait_event_interruptible(ring->irq_queue,
3593 i915_seqno_passed(ring->get_seqno(dev, ring), seqno) 3166 i915_seqno_passed(ring->get_seqno(ring), seqno)
3594 || atomic_read(&dev_priv->mm.wedged)); 3167 || atomic_read(&dev_priv->mm.wedged));
3595 ring->user_irq_put(dev, ring); 3168 ring->irq_put(ring);
3596
3597 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3598 ret = -EIO;
3599 }
3600
3601 if (ret == 0)
3602 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3603
3604 return ret;
3605}
3606
3607static int
3608i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3609 uint64_t exec_offset)
3610{
3611 uint32_t exec_start, exec_len;
3612
3613 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3614 exec_len = (uint32_t) exec->batch_len;
3615
3616 if ((exec_start | exec_len) & 0x7)
3617 return -EINVAL;
3618
3619 if (!exec_start)
3620 return -EINVAL;
3621
3622 return 0;
3623}
3624
3625static int
3626validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3627 int count)
3628{
3629 int i;
3630
3631 for (i = 0; i < count; i++) {
3632 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3633 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3634
3635 if (!access_ok(VERIFY_READ, ptr, length))
3636 return -EFAULT;
3637
3638 /* we may also need to update the presumed offsets */
3639 if (!access_ok(VERIFY_WRITE, ptr, length))
3640 return -EFAULT;
3641
3642 if (fault_in_pages_readable(ptr, length))
3643 return -EFAULT;
3644 }
3645
3646 return 0;
3647}
3648
3649static int
3650i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3651 struct drm_file *file,
3652 struct drm_i915_gem_execbuffer2 *args,
3653 struct drm_i915_gem_exec_object2 *exec_list)
3654{
3655 drm_i915_private_t *dev_priv = dev->dev_private;
3656 struct drm_gem_object **object_list = NULL;
3657 struct drm_gem_object *batch_obj;
3658 struct drm_i915_gem_object *obj_priv;
3659 struct drm_clip_rect *cliprects = NULL;
3660 struct drm_i915_gem_request *request = NULL;
3661 int ret, i, flips;
3662 uint64_t exec_offset;
3663
3664 struct intel_ring_buffer *ring = NULL;
3665
3666 ret = i915_gem_check_is_wedged(dev);
3667 if (ret)
3668 return ret;
3669
3670 ret = validate_exec_list(exec_list, args->buffer_count);
3671 if (ret)
3672 return ret;
3673
3674#if WATCH_EXEC
3675 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3676 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3677#endif
3678 switch (args->flags & I915_EXEC_RING_MASK) {
3679 case I915_EXEC_DEFAULT:
3680 case I915_EXEC_RENDER:
3681 ring = &dev_priv->render_ring;
3682 break;
3683 case I915_EXEC_BSD:
3684 if (!HAS_BSD(dev)) {
3685 DRM_ERROR("execbuf with invalid ring (BSD)\n");
3686 return -EINVAL;
3687 }
3688 ring = &dev_priv->bsd_ring;
3689 break;
3690 case I915_EXEC_BLT:
3691 if (!HAS_BLT(dev)) {
3692 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3693 return -EINVAL;
3694 }
3695 ring = &dev_priv->blt_ring;
3696 break;
3697 default:
3698 DRM_ERROR("execbuf with unknown ring: %d\n",
3699 (int)(args->flags & I915_EXEC_RING_MASK));
3700 return -EINVAL;
3701 }
3702
3703 if (args->buffer_count < 1) {
3704 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3705 return -EINVAL;
3706 }
3707 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3708 if (object_list == NULL) {
3709 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3710 args->buffer_count);
3711 ret = -ENOMEM;
3712 goto pre_mutex_err;
3713 }
3714
3715 if (args->num_cliprects != 0) {
3716 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3717 GFP_KERNEL);
3718 if (cliprects == NULL) {
3719 ret = -ENOMEM;
3720 goto pre_mutex_err;
3721 }
3722
3723 ret = copy_from_user(cliprects,
3724 (struct drm_clip_rect __user *)
3725 (uintptr_t) args->cliprects_ptr,
3726 sizeof(*cliprects) * args->num_cliprects);
3727 if (ret != 0) {
3728 DRM_ERROR("copy %d cliprects failed: %d\n",
3729 args->num_cliprects, ret);
3730 ret = -EFAULT;
3731 goto pre_mutex_err;
3732 }
3733 }
3734
3735 request = kzalloc(sizeof(*request), GFP_KERNEL);
3736 if (request == NULL) {
3737 ret = -ENOMEM;
3738 goto pre_mutex_err;
3739 }
3740
3741 ret = i915_mutex_lock_interruptible(dev);
3742 if (ret)
3743 goto pre_mutex_err;
3744
3745 if (dev_priv->mm.suspended) {
3746 mutex_unlock(&dev->struct_mutex);
3747 ret = -EBUSY;
3748 goto pre_mutex_err;
3749 }
3750
3751 /* Look up object handles */
3752 for (i = 0; i < args->buffer_count; i++) {
3753 object_list[i] = drm_gem_object_lookup(dev, file,
3754 exec_list[i].handle);
3755 if (object_list[i] == NULL) {
3756 DRM_ERROR("Invalid object handle %d at index %d\n",
3757 exec_list[i].handle, i);
3758 /* prevent error path from reading uninitialized data */
3759 args->buffer_count = i + 1;
3760 ret = -ENOENT;
3761 goto err;
3762 }
3763
3764 obj_priv = to_intel_bo(object_list[i]);
3765 if (obj_priv->in_execbuffer) {
3766 DRM_ERROR("Object %p appears more than once in object list\n",
3767 object_list[i]);
3768 /* prevent error path from reading uninitialized data */
3769 args->buffer_count = i + 1;
3770 ret = -EINVAL;
3771 goto err;
3772 }
3773 obj_priv->in_execbuffer = true;
3774 }
3775
3776 /* Move the objects en-masse into the GTT, evicting if necessary. */
3777 ret = i915_gem_execbuffer_pin(dev, file,
3778 object_list, exec_list,
3779 args->buffer_count);
3780 if (ret)
3781 goto err;
3782
3783 /* The objects are in their final locations, apply the relocations. */
3784 for (i = 0; i < args->buffer_count; i++) {
3785 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3786 obj->base.pending_read_domains = 0;
3787 obj->base.pending_write_domain = 0;
3788 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3789 if (ret)
3790 goto err;
3791 }
3792
3793 /* Set the pending read domains for the batch buffer to COMMAND */
3794 batch_obj = object_list[args->buffer_count-1];
3795 if (batch_obj->pending_write_domain) {
3796 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3797 ret = -EINVAL;
3798 goto err;
3799 }
3800 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3801
3802 /* Sanity check the batch buffer */
3803 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3804 ret = i915_gem_check_execbuffer(args, exec_offset);
3805 if (ret != 0) {
3806 DRM_ERROR("execbuf with invalid offset/length\n");
3807 goto err;
3808 }
3809
3810 ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3811 object_list, args->buffer_count);
3812 if (ret)
3813 goto err;
3814
3815 for (i = 0; i < args->buffer_count; i++) {
3816 struct drm_gem_object *obj = object_list[i];
3817 uint32_t old_write_domain = obj->write_domain;
3818 obj->write_domain = obj->pending_write_domain;
3819 trace_i915_gem_object_change_domain(obj,
3820 obj->read_domains,
3821 old_write_domain);
3822 }
3823
3824#if WATCH_COHERENCY
3825 for (i = 0; i < args->buffer_count; i++) {
3826 i915_gem_object_check_coherency(object_list[i],
3827 exec_list[i].handle);
3828 }
3829#endif
3830
3831#if WATCH_EXEC
3832 i915_gem_dump_object(batch_obj,
3833 args->batch_len,
3834 __func__,
3835 ~0);
3836#endif
3837
3838 /* Check for any pending flips. As we only maintain a flip queue depth
3839 * of 1, we can simply insert a WAIT for the next display flip prior
3840 * to executing the batch and avoid stalling the CPU.
3841 */
3842 flips = 0;
3843 for (i = 0; i < args->buffer_count; i++) {
3844 if (object_list[i]->write_domain)
3845 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3846 }
3847 if (flips) {
3848 int plane, flip_mask;
3849
3850 for (plane = 0; flips >> plane; plane++) {
3851 if (((flips >> plane) & 1) == 0)
3852 continue;
3853
3854 if (plane)
3855 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3856 else
3857 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3858
3859 intel_ring_begin(dev, ring, 2);
3860 intel_ring_emit(dev, ring,
3861 MI_WAIT_FOR_EVENT | flip_mask);
3862 intel_ring_emit(dev, ring, MI_NOOP);
3863 intel_ring_advance(dev, ring);
3864 }
3865 }
3866
3867 /* Exec the batchbuffer */
3868 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3869 cliprects, exec_offset);
3870 if (ret) {
3871 DRM_ERROR("dispatch failed %d\n", ret);
3872 goto err;
3873 }
3874
3875 /*
3876 * Ensure that the commands in the batch buffer are
3877 * finished before the interrupt fires
3878 */
3879 i915_retire_commands(dev, ring);
3880
3881 for (i = 0; i < args->buffer_count; i++) {
3882 struct drm_gem_object *obj = object_list[i];
3883
3884 i915_gem_object_move_to_active(obj, ring);
3885 if (obj->write_domain)
3886 list_move_tail(&to_intel_bo(obj)->gpu_write_list,
3887 &ring->gpu_write_list);
3888 }
3889
3890 i915_add_request(dev, file, request, ring);
3891 request = NULL;
3892
3893err:
3894 for (i = 0; i < args->buffer_count; i++) {
3895 if (object_list[i]) {
3896 obj_priv = to_intel_bo(object_list[i]);
3897 obj_priv->in_execbuffer = false;
3898 }
3899 drm_gem_object_unreference(object_list[i]);
3900 }
3901
3902 mutex_unlock(&dev->struct_mutex);
3903
3904pre_mutex_err:
3905 drm_free_large(object_list);
3906 kfree(cliprects);
3907 kfree(request);
3908
3909 return ret;
3910}
3911
3912/*
3913 * Legacy execbuffer just creates an exec2 list from the original exec object
3914 * list array and passes it to the real function.
3915 */
3916int
3917i915_gem_execbuffer(struct drm_device *dev, void *data,
3918 struct drm_file *file_priv)
3919{
3920 struct drm_i915_gem_execbuffer *args = data;
3921 struct drm_i915_gem_execbuffer2 exec2;
3922 struct drm_i915_gem_exec_object *exec_list = NULL;
3923 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3924 int ret, i;
3925
3926#if WATCH_EXEC
3927 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3928 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3929#endif
3930
3931 if (args->buffer_count < 1) {
3932 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3933 return -EINVAL;
3934 }
3935
3936 /* Copy in the exec list from userland */
3937 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3938 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3939 if (exec_list == NULL || exec2_list == NULL) {
3940 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3941 args->buffer_count);
3942 drm_free_large(exec_list);
3943 drm_free_large(exec2_list);
3944 return -ENOMEM;
3945 }
3946 ret = copy_from_user(exec_list,
3947 (struct drm_i915_relocation_entry __user *)
3948 (uintptr_t) args->buffers_ptr,
3949 sizeof(*exec_list) * args->buffer_count);
3950 if (ret != 0) {
3951 DRM_ERROR("copy %d exec entries failed %d\n",
3952 args->buffer_count, ret);
3953 drm_free_large(exec_list);
3954 drm_free_large(exec2_list);
3955 return -EFAULT;
3956 }
3957
3958 for (i = 0; i < args->buffer_count; i++) {
3959 exec2_list[i].handle = exec_list[i].handle;
3960 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3961 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3962 exec2_list[i].alignment = exec_list[i].alignment;
3963 exec2_list[i].offset = exec_list[i].offset;
3964 if (INTEL_INFO(dev)->gen < 4)
3965 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3966 else
3967 exec2_list[i].flags = 0;
3968 }
3969 3169
3970 exec2.buffers_ptr = args->buffers_ptr; 3170 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3971 exec2.buffer_count = args->buffer_count; 3171 ret = -EIO;
3972 exec2.batch_start_offset = args->batch_start_offset;
3973 exec2.batch_len = args->batch_len;
3974 exec2.DR1 = args->DR1;
3975 exec2.DR4 = args->DR4;
3976 exec2.num_cliprects = args->num_cliprects;
3977 exec2.cliprects_ptr = args->cliprects_ptr;
3978 exec2.flags = I915_EXEC_RENDER;
3979
3980 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3981 if (!ret) {
3982 /* Copy the new buffer offsets back to the user's exec list. */
3983 for (i = 0; i < args->buffer_count; i++)
3984 exec_list[i].offset = exec2_list[i].offset;
3985 /* ... and back out to userspace */
3986 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3987 (uintptr_t) args->buffers_ptr,
3988 exec_list,
3989 sizeof(*exec_list) * args->buffer_count);
3990 if (ret) {
3991 ret = -EFAULT;
3992 DRM_ERROR("failed to copy %d exec entries "
3993 "back to user (%d)\n",
3994 args->buffer_count, ret);
3995 } 3172 }
3996 } 3173 }
3997 3174
3998 drm_free_large(exec_list); 3175 if (ret == 0)
3999 drm_free_large(exec2_list); 3176 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4000 return ret;
4001}
4002
4003int
4004i915_gem_execbuffer2(struct drm_device *dev, void *data,
4005 struct drm_file *file_priv)
4006{
4007 struct drm_i915_gem_execbuffer2 *args = data;
4008 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4009 int ret;
4010
4011#if WATCH_EXEC
4012 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4013 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4014#endif
4015
4016 if (args->buffer_count < 1) {
4017 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4018 return -EINVAL;
4019 }
4020
4021 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4022 if (exec2_list == NULL) {
4023 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4024 args->buffer_count);
4025 return -ENOMEM;
4026 }
4027 ret = copy_from_user(exec2_list,
4028 (struct drm_i915_relocation_entry __user *)
4029 (uintptr_t) args->buffers_ptr,
4030 sizeof(*exec2_list) * args->buffer_count);
4031 if (ret != 0) {
4032 DRM_ERROR("copy %d exec entries failed %d\n",
4033 args->buffer_count, ret);
4034 drm_free_large(exec2_list);
4035 return -EFAULT;
4036 }
4037
4038 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4039 if (!ret) {
4040 /* Copy the new buffer offsets back to the user's exec list. */
4041 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4042 (uintptr_t) args->buffers_ptr,
4043 exec2_list,
4044 sizeof(*exec2_list) * args->buffer_count);
4045 if (ret) {
4046 ret = -EFAULT;
4047 DRM_ERROR("failed to copy %d exec entries "
4048 "back to user (%d)\n",
4049 args->buffer_count, ret);
4050 }
4051 }
4052 3177
4053 drm_free_large(exec2_list);
4054 return ret; 3178 return ret;
4055} 3179}
4056 3180
4057int 3181int
4058i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) 3182i915_gem_object_pin(struct drm_i915_gem_object *obj,
3183 uint32_t alignment,
3184 bool map_and_fenceable)
4059{ 3185{
4060 struct drm_device *dev = obj->dev; 3186 struct drm_device *dev = obj->base.dev;
4061 struct drm_i915_private *dev_priv = dev->dev_private; 3187 struct drm_i915_private *dev_priv = dev->dev_private;
4062 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4063 int ret; 3188 int ret;
4064 3189
4065 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); 3190 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4066 WARN_ON(i915_verify_lists(dev)); 3191 WARN_ON(i915_verify_lists(dev));
4067 3192
4068 if (obj_priv->gtt_space != NULL) { 3193 if (obj->gtt_space != NULL) {
4069 if (alignment == 0) 3194 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4070 alignment = i915_gem_get_gtt_alignment(obj); 3195 (map_and_fenceable && !obj->map_and_fenceable)) {
4071 if (obj_priv->gtt_offset & (alignment - 1)) { 3196 WARN(obj->pin_count,
4072 WARN(obj_priv->pin_count, 3197 "bo is already pinned with incorrect alignment:"
4073 "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n", 3198 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4074 obj_priv->gtt_offset, alignment); 3199 " obj->map_and_fenceable=%d\n",
3200 obj->gtt_offset, alignment,
3201 map_and_fenceable,
3202 obj->map_and_fenceable);
4075 ret = i915_gem_object_unbind(obj); 3203 ret = i915_gem_object_unbind(obj);
4076 if (ret) 3204 if (ret)
4077 return ret; 3205 return ret;
4078 } 3206 }
4079 } 3207 }
4080 3208
4081 if (obj_priv->gtt_space == NULL) { 3209 if (obj->gtt_space == NULL) {
4082 ret = i915_gem_object_bind_to_gtt(obj, alignment); 3210 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3211 map_and_fenceable);
4083 if (ret) 3212 if (ret)
4084 return ret; 3213 return ret;
4085 } 3214 }
4086 3215
4087 obj_priv->pin_count++; 3216 if (obj->pin_count++ == 0) {
4088 3217 if (!obj->active)
4089 /* If the object is not active and not pending a flush, 3218 list_move_tail(&obj->mm_list,
4090 * remove it from the inactive list
4091 */
4092 if (obj_priv->pin_count == 1) {
4093 i915_gem_info_add_pin(dev_priv, obj->size);
4094 if (!obj_priv->active)
4095 list_move_tail(&obj_priv->mm_list,
4096 &dev_priv->mm.pinned_list); 3219 &dev_priv->mm.pinned_list);
4097 } 3220 }
3221 obj->pin_mappable |= map_and_fenceable;
4098 3222
4099 WARN_ON(i915_verify_lists(dev)); 3223 WARN_ON(i915_verify_lists(dev));
4100 return 0; 3224 return 0;
4101} 3225}
4102 3226
4103void 3227void
4104i915_gem_object_unpin(struct drm_gem_object *obj) 3228i915_gem_object_unpin(struct drm_i915_gem_object *obj)
4105{ 3229{
4106 struct drm_device *dev = obj->dev; 3230 struct drm_device *dev = obj->base.dev;
4107 drm_i915_private_t *dev_priv = dev->dev_private; 3231 drm_i915_private_t *dev_priv = dev->dev_private;
4108 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4109 3232
4110 WARN_ON(i915_verify_lists(dev)); 3233 WARN_ON(i915_verify_lists(dev));
4111 obj_priv->pin_count--; 3234 BUG_ON(obj->pin_count == 0);
4112 BUG_ON(obj_priv->pin_count < 0); 3235 BUG_ON(obj->gtt_space == NULL);
4113 BUG_ON(obj_priv->gtt_space == NULL);
4114 3236
4115 /* If the object is no longer pinned, and is 3237 if (--obj->pin_count == 0) {
4116 * neither active nor being flushed, then stick it on 3238 if (!obj->active)
4117 * the inactive list 3239 list_move_tail(&obj->mm_list,
4118 */
4119 if (obj_priv->pin_count == 0) {
4120 if (!obj_priv->active)
4121 list_move_tail(&obj_priv->mm_list,
4122 &dev_priv->mm.inactive_list); 3240 &dev_priv->mm.inactive_list);
4123 i915_gem_info_remove_pin(dev_priv, obj->size); 3241 obj->pin_mappable = false;
4124 } 3242 }
4125 WARN_ON(i915_verify_lists(dev)); 3243 WARN_ON(i915_verify_lists(dev));
4126} 3244}
4127 3245
4128int 3246int
4129i915_gem_pin_ioctl(struct drm_device *dev, void *data, 3247i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4130 struct drm_file *file_priv) 3248 struct drm_file *file)
4131{ 3249{
4132 struct drm_i915_gem_pin *args = data; 3250 struct drm_i915_gem_pin *args = data;
4133 struct drm_gem_object *obj; 3251 struct drm_i915_gem_object *obj;
4134 struct drm_i915_gem_object *obj_priv;
4135 int ret; 3252 int ret;
4136 3253
4137 ret = i915_mutex_lock_interruptible(dev); 3254 ret = i915_mutex_lock_interruptible(dev);
4138 if (ret) 3255 if (ret)
4139 return ret; 3256 return ret;
4140 3257
4141 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3258 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4142 if (obj == NULL) { 3259 if (obj == NULL) {
4143 ret = -ENOENT; 3260 ret = -ENOENT;
4144 goto unlock; 3261 goto unlock;
4145 } 3262 }
4146 obj_priv = to_intel_bo(obj);
4147 3263
4148 if (obj_priv->madv != I915_MADV_WILLNEED) { 3264 if (obj->madv != I915_MADV_WILLNEED) {
4149 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 3265 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4150 ret = -EINVAL; 3266 ret = -EINVAL;
4151 goto out; 3267 goto out;
4152 } 3268 }
4153 3269
4154 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 3270 if (obj->pin_filp != NULL && obj->pin_filp != file) {
4155 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 3271 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4156 args->handle); 3272 args->handle);
4157 ret = -EINVAL; 3273 ret = -EINVAL;
4158 goto out; 3274 goto out;
4159 } 3275 }
4160 3276
4161 obj_priv->user_pin_count++; 3277 obj->user_pin_count++;
4162 obj_priv->pin_filp = file_priv; 3278 obj->pin_filp = file;
4163 if (obj_priv->user_pin_count == 1) { 3279 if (obj->user_pin_count == 1) {
4164 ret = i915_gem_object_pin(obj, args->alignment); 3280 ret = i915_gem_object_pin(obj, args->alignment, true);
4165 if (ret) 3281 if (ret)
4166 goto out; 3282 goto out;
4167 } 3283 }
@@ -4170,9 +3286,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4170 * as the X server doesn't manage domains yet 3286 * as the X server doesn't manage domains yet
4171 */ 3287 */
4172 i915_gem_object_flush_cpu_write_domain(obj); 3288 i915_gem_object_flush_cpu_write_domain(obj);
4173 args->offset = obj_priv->gtt_offset; 3289 args->offset = obj->gtt_offset;
4174out: 3290out:
4175 drm_gem_object_unreference(obj); 3291 drm_gem_object_unreference(&obj->base);
4176unlock: 3292unlock:
4177 mutex_unlock(&dev->struct_mutex); 3293 mutex_unlock(&dev->struct_mutex);
4178 return ret; 3294 return ret;
@@ -4180,38 +3296,36 @@ unlock:
4180 3296
4181int 3297int
4182i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 3298i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4183 struct drm_file *file_priv) 3299 struct drm_file *file)
4184{ 3300{
4185 struct drm_i915_gem_pin *args = data; 3301 struct drm_i915_gem_pin *args = data;
4186 struct drm_gem_object *obj; 3302 struct drm_i915_gem_object *obj;
4187 struct drm_i915_gem_object *obj_priv;
4188 int ret; 3303 int ret;
4189 3304
4190 ret = i915_mutex_lock_interruptible(dev); 3305 ret = i915_mutex_lock_interruptible(dev);
4191 if (ret) 3306 if (ret)
4192 return ret; 3307 return ret;
4193 3308
4194 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3309 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4195 if (obj == NULL) { 3310 if (obj == NULL) {
4196 ret = -ENOENT; 3311 ret = -ENOENT;
4197 goto unlock; 3312 goto unlock;
4198 } 3313 }
4199 obj_priv = to_intel_bo(obj);
4200 3314
4201 if (obj_priv->pin_filp != file_priv) { 3315 if (obj->pin_filp != file) {
4202 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 3316 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4203 args->handle); 3317 args->handle);
4204 ret = -EINVAL; 3318 ret = -EINVAL;
4205 goto out; 3319 goto out;
4206 } 3320 }
4207 obj_priv->user_pin_count--; 3321 obj->user_pin_count--;
4208 if (obj_priv->user_pin_count == 0) { 3322 if (obj->user_pin_count == 0) {
4209 obj_priv->pin_filp = NULL; 3323 obj->pin_filp = NULL;
4210 i915_gem_object_unpin(obj); 3324 i915_gem_object_unpin(obj);
4211 } 3325 }
4212 3326
4213out: 3327out:
4214 drm_gem_object_unreference(obj); 3328 drm_gem_object_unreference(&obj->base);
4215unlock: 3329unlock:
4216 mutex_unlock(&dev->struct_mutex); 3330 mutex_unlock(&dev->struct_mutex);
4217 return ret; 3331 return ret;
@@ -4219,52 +3333,64 @@ unlock:
4219 3333
4220int 3334int
4221i915_gem_busy_ioctl(struct drm_device *dev, void *data, 3335i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4222 struct drm_file *file_priv) 3336 struct drm_file *file)
4223{ 3337{
4224 struct drm_i915_gem_busy *args = data; 3338 struct drm_i915_gem_busy *args = data;
4225 struct drm_gem_object *obj; 3339 struct drm_i915_gem_object *obj;
4226 struct drm_i915_gem_object *obj_priv;
4227 int ret; 3340 int ret;
4228 3341
4229 ret = i915_mutex_lock_interruptible(dev); 3342 ret = i915_mutex_lock_interruptible(dev);
4230 if (ret) 3343 if (ret)
4231 return ret; 3344 return ret;
4232 3345
4233 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3346 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4234 if (obj == NULL) { 3347 if (obj == NULL) {
4235 ret = -ENOENT; 3348 ret = -ENOENT;
4236 goto unlock; 3349 goto unlock;
4237 } 3350 }
4238 obj_priv = to_intel_bo(obj);
4239 3351
4240 /* Count all active objects as busy, even if they are currently not used 3352 /* Count all active objects as busy, even if they are currently not used
4241 * by the gpu. Users of this interface expect objects to eventually 3353 * by the gpu. Users of this interface expect objects to eventually
4242 * become non-busy without any further actions, therefore emit any 3354 * become non-busy without any further actions, therefore emit any
4243 * necessary flushes here. 3355 * necessary flushes here.
4244 */ 3356 */
4245 args->busy = obj_priv->active; 3357 args->busy = obj->active;
4246 if (args->busy) { 3358 if (args->busy) {
4247 /* Unconditionally flush objects, even when the gpu still uses this 3359 /* Unconditionally flush objects, even when the gpu still uses this
4248 * object. Userspace calling this function indicates that it wants to 3360 * object. Userspace calling this function indicates that it wants to
4249 * use this buffer rather sooner than later, so issuing the required 3361 * use this buffer rather sooner than later, so issuing the required
4250 * flush earlier is beneficial. 3362 * flush earlier is beneficial.
4251 */ 3363 */
4252 if (obj->write_domain & I915_GEM_GPU_DOMAINS) 3364 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
4253 i915_gem_flush_ring(dev, file_priv, 3365 i915_gem_flush_ring(dev, obj->ring,
4254 obj_priv->ring, 3366 0, obj->base.write_domain);
4255 0, obj->write_domain); 3367 } else if (obj->ring->outstanding_lazy_request ==
3368 obj->last_rendering_seqno) {
3369 struct drm_i915_gem_request *request;
3370
3371 /* This ring is not being cleared by active usage,
3372 * so emit a request to do so.
3373 */
3374 request = kzalloc(sizeof(*request), GFP_KERNEL);
3375 if (request)
3376 ret = i915_add_request(dev,
3377 NULL, request,
3378 obj->ring);
3379 else
3380 ret = -ENOMEM;
3381 }
4256 3382
4257 /* Update the active list for the hardware's current position. 3383 /* Update the active list for the hardware's current position.
4258 * Otherwise this only updates on a delayed timer or when irqs 3384 * Otherwise this only updates on a delayed timer or when irqs
4259 * are actually unmasked, and our working set ends up being 3385 * are actually unmasked, and our working set ends up being
4260 * larger than required. 3386 * larger than required.
4261 */ 3387 */
4262 i915_gem_retire_requests_ring(dev, obj_priv->ring); 3388 i915_gem_retire_requests_ring(dev, obj->ring);
4263 3389
4264 args->busy = obj_priv->active; 3390 args->busy = obj->active;
4265 } 3391 }
4266 3392
4267 drm_gem_object_unreference(obj); 3393 drm_gem_object_unreference(&obj->base);
4268unlock: 3394unlock:
4269 mutex_unlock(&dev->struct_mutex); 3395 mutex_unlock(&dev->struct_mutex);
4270 return ret; 3396 return ret;
@@ -4282,8 +3408,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4282 struct drm_file *file_priv) 3408 struct drm_file *file_priv)
4283{ 3409{
4284 struct drm_i915_gem_madvise *args = data; 3410 struct drm_i915_gem_madvise *args = data;
4285 struct drm_gem_object *obj; 3411 struct drm_i915_gem_object *obj;
4286 struct drm_i915_gem_object *obj_priv;
4287 int ret; 3412 int ret;
4288 3413
4289 switch (args->madv) { 3414 switch (args->madv) {
@@ -4298,37 +3423,36 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4298 if (ret) 3423 if (ret)
4299 return ret; 3424 return ret;
4300 3425
4301 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 3426 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4302 if (obj == NULL) { 3427 if (obj == NULL) {
4303 ret = -ENOENT; 3428 ret = -ENOENT;
4304 goto unlock; 3429 goto unlock;
4305 } 3430 }
4306 obj_priv = to_intel_bo(obj);
4307 3431
4308 if (obj_priv->pin_count) { 3432 if (obj->pin_count) {
4309 ret = -EINVAL; 3433 ret = -EINVAL;
4310 goto out; 3434 goto out;
4311 } 3435 }
4312 3436
4313 if (obj_priv->madv != __I915_MADV_PURGED) 3437 if (obj->madv != __I915_MADV_PURGED)
4314 obj_priv->madv = args->madv; 3438 obj->madv = args->madv;
4315 3439
4316 /* if the object is no longer bound, discard its backing storage */ 3440 /* if the object is no longer bound, discard its backing storage */
4317 if (i915_gem_object_is_purgeable(obj_priv) && 3441 if (i915_gem_object_is_purgeable(obj) &&
4318 obj_priv->gtt_space == NULL) 3442 obj->gtt_space == NULL)
4319 i915_gem_object_truncate(obj); 3443 i915_gem_object_truncate(obj);
4320 3444
4321 args->retained = obj_priv->madv != __I915_MADV_PURGED; 3445 args->retained = obj->madv != __I915_MADV_PURGED;
4322 3446
4323out: 3447out:
4324 drm_gem_object_unreference(obj); 3448 drm_gem_object_unreference(&obj->base);
4325unlock: 3449unlock:
4326 mutex_unlock(&dev->struct_mutex); 3450 mutex_unlock(&dev->struct_mutex);
4327 return ret; 3451 return ret;
4328} 3452}
4329 3453
4330struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 3454struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4331 size_t size) 3455 size_t size)
4332{ 3456{
4333 struct drm_i915_private *dev_priv = dev->dev_private; 3457 struct drm_i915_private *dev_priv = dev->dev_private;
4334 struct drm_i915_gem_object *obj; 3458 struct drm_i915_gem_object *obj;
@@ -4351,11 +3475,15 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4351 obj->base.driver_private = NULL; 3475 obj->base.driver_private = NULL;
4352 obj->fence_reg = I915_FENCE_REG_NONE; 3476 obj->fence_reg = I915_FENCE_REG_NONE;
4353 INIT_LIST_HEAD(&obj->mm_list); 3477 INIT_LIST_HEAD(&obj->mm_list);
3478 INIT_LIST_HEAD(&obj->gtt_list);
4354 INIT_LIST_HEAD(&obj->ring_list); 3479 INIT_LIST_HEAD(&obj->ring_list);
3480 INIT_LIST_HEAD(&obj->exec_list);
4355 INIT_LIST_HEAD(&obj->gpu_write_list); 3481 INIT_LIST_HEAD(&obj->gpu_write_list);
4356 obj->madv = I915_MADV_WILLNEED; 3482 obj->madv = I915_MADV_WILLNEED;
3483 /* Avoid an unnecessary call to unbind on the first bind. */
3484 obj->map_and_fenceable = true;
4357 3485
4358 return &obj->base; 3486 return obj;
4359} 3487}
4360 3488
4361int i915_gem_init_object(struct drm_gem_object *obj) 3489int i915_gem_init_object(struct drm_gem_object *obj)
@@ -4365,42 +3493,41 @@ int i915_gem_init_object(struct drm_gem_object *obj)
4365 return 0; 3493 return 0;
4366} 3494}
4367 3495
4368static void i915_gem_free_object_tail(struct drm_gem_object *obj) 3496static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
4369{ 3497{
4370 struct drm_device *dev = obj->dev; 3498 struct drm_device *dev = obj->base.dev;
4371 drm_i915_private_t *dev_priv = dev->dev_private; 3499 drm_i915_private_t *dev_priv = dev->dev_private;
4372 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4373 int ret; 3500 int ret;
4374 3501
4375 ret = i915_gem_object_unbind(obj); 3502 ret = i915_gem_object_unbind(obj);
4376 if (ret == -ERESTARTSYS) { 3503 if (ret == -ERESTARTSYS) {
4377 list_move(&obj_priv->mm_list, 3504 list_move(&obj->mm_list,
4378 &dev_priv->mm.deferred_free_list); 3505 &dev_priv->mm.deferred_free_list);
4379 return; 3506 return;
4380 } 3507 }
4381 3508
4382 if (obj_priv->mmap_offset) 3509 if (obj->base.map_list.map)
4383 i915_gem_free_mmap_offset(obj); 3510 i915_gem_free_mmap_offset(obj);
4384 3511
4385 drm_gem_object_release(obj); 3512 drm_gem_object_release(&obj->base);
4386 i915_gem_info_remove_obj(dev_priv, obj->size); 3513 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4387 3514
4388 kfree(obj_priv->page_cpu_valid); 3515 kfree(obj->page_cpu_valid);
4389 kfree(obj_priv->bit_17); 3516 kfree(obj->bit_17);
4390 kfree(obj_priv); 3517 kfree(obj);
4391} 3518}
4392 3519
4393void i915_gem_free_object(struct drm_gem_object *obj) 3520void i915_gem_free_object(struct drm_gem_object *gem_obj)
4394{ 3521{
4395 struct drm_device *dev = obj->dev; 3522 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4396 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3523 struct drm_device *dev = obj->base.dev;
4397 3524
4398 trace_i915_gem_object_destroy(obj); 3525 trace_i915_gem_object_destroy(obj);
4399 3526
4400 while (obj_priv->pin_count > 0) 3527 while (obj->pin_count > 0)
4401 i915_gem_object_unpin(obj); 3528 i915_gem_object_unpin(obj);
4402 3529
4403 if (obj_priv->phys_obj) 3530 if (obj->phys_obj)
4404 i915_gem_detach_phys_object(dev, obj); 3531 i915_gem_detach_phys_object(dev, obj);
4405 3532
4406 i915_gem_free_object_tail(obj); 3533 i915_gem_free_object_tail(obj);
@@ -4427,13 +3554,15 @@ i915_gem_idle(struct drm_device *dev)
4427 3554
4428 /* Under UMS, be paranoid and evict. */ 3555 /* Under UMS, be paranoid and evict. */
4429 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 3556 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4430 ret = i915_gem_evict_inactive(dev); 3557 ret = i915_gem_evict_inactive(dev, false);
4431 if (ret) { 3558 if (ret) {
4432 mutex_unlock(&dev->struct_mutex); 3559 mutex_unlock(&dev->struct_mutex);
4433 return ret; 3560 return ret;
4434 } 3561 }
4435 } 3562 }
4436 3563
3564 i915_gem_reset_fences(dev);
3565
4437 /* Hack! Don't let anybody do execbuf while we don't control the chip. 3566 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4438 * We need to replace this with a semaphore, or something. 3567 * We need to replace this with a semaphore, or something.
4439 * And not confound mm.suspended! 3568 * And not confound mm.suspended!
@@ -4452,82 +3581,15 @@ i915_gem_idle(struct drm_device *dev)
4452 return 0; 3581 return 0;
4453} 3582}
4454 3583
4455/*
4456 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4457 * over cache flushing.
4458 */
4459static int
4460i915_gem_init_pipe_control(struct drm_device *dev)
4461{
4462 drm_i915_private_t *dev_priv = dev->dev_private;
4463 struct drm_gem_object *obj;
4464 struct drm_i915_gem_object *obj_priv;
4465 int ret;
4466
4467 obj = i915_gem_alloc_object(dev, 4096);
4468 if (obj == NULL) {
4469 DRM_ERROR("Failed to allocate seqno page\n");
4470 ret = -ENOMEM;
4471 goto err;
4472 }
4473 obj_priv = to_intel_bo(obj);
4474 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4475
4476 ret = i915_gem_object_pin(obj, 4096);
4477 if (ret)
4478 goto err_unref;
4479
4480 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4481 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4482 if (dev_priv->seqno_page == NULL)
4483 goto err_unpin;
4484
4485 dev_priv->seqno_obj = obj;
4486 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4487
4488 return 0;
4489
4490err_unpin:
4491 i915_gem_object_unpin(obj);
4492err_unref:
4493 drm_gem_object_unreference(obj);
4494err:
4495 return ret;
4496}
4497
4498
4499static void
4500i915_gem_cleanup_pipe_control(struct drm_device *dev)
4501{
4502 drm_i915_private_t *dev_priv = dev->dev_private;
4503 struct drm_gem_object *obj;
4504 struct drm_i915_gem_object *obj_priv;
4505
4506 obj = dev_priv->seqno_obj;
4507 obj_priv = to_intel_bo(obj);
4508 kunmap(obj_priv->pages[0]);
4509 i915_gem_object_unpin(obj);
4510 drm_gem_object_unreference(obj);
4511 dev_priv->seqno_obj = NULL;
4512
4513 dev_priv->seqno_page = NULL;
4514}
4515
4516int 3584int
4517i915_gem_init_ringbuffer(struct drm_device *dev) 3585i915_gem_init_ringbuffer(struct drm_device *dev)
4518{ 3586{
4519 drm_i915_private_t *dev_priv = dev->dev_private; 3587 drm_i915_private_t *dev_priv = dev->dev_private;
4520 int ret; 3588 int ret;
4521 3589
4522 if (HAS_PIPE_CONTROL(dev)) {
4523 ret = i915_gem_init_pipe_control(dev);
4524 if (ret)
4525 return ret;
4526 }
4527
4528 ret = intel_init_render_ring_buffer(dev); 3590 ret = intel_init_render_ring_buffer(dev);
4529 if (ret) 3591 if (ret)
4530 goto cleanup_pipe_control; 3592 return ret;
4531 3593
4532 if (HAS_BSD(dev)) { 3594 if (HAS_BSD(dev)) {
4533 ret = intel_init_bsd_ring_buffer(dev); 3595 ret = intel_init_bsd_ring_buffer(dev);
@@ -4546,12 +3608,9 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
4546 return 0; 3608 return 0;
4547 3609
4548cleanup_bsd_ring: 3610cleanup_bsd_ring:
4549 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3611 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4550cleanup_render_ring: 3612cleanup_render_ring:
4551 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3613 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4552cleanup_pipe_control:
4553 if (HAS_PIPE_CONTROL(dev))
4554 i915_gem_cleanup_pipe_control(dev);
4555 return ret; 3614 return ret;
4556} 3615}
4557 3616
@@ -4559,12 +3618,10 @@ void
4559i915_gem_cleanup_ringbuffer(struct drm_device *dev) 3618i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4560{ 3619{
4561 drm_i915_private_t *dev_priv = dev->dev_private; 3620 drm_i915_private_t *dev_priv = dev->dev_private;
3621 int i;
4562 3622
4563 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 3623 for (i = 0; i < I915_NUM_RINGS; i++)
4564 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 3624 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
4565 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4566 if (HAS_PIPE_CONTROL(dev))
4567 i915_gem_cleanup_pipe_control(dev);
4568} 3625}
4569 3626
4570int 3627int
@@ -4572,7 +3629,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4572 struct drm_file *file_priv) 3629 struct drm_file *file_priv)
4573{ 3630{
4574 drm_i915_private_t *dev_priv = dev->dev_private; 3631 drm_i915_private_t *dev_priv = dev->dev_private;
4575 int ret; 3632 int ret, i;
4576 3633
4577 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3634 if (drm_core_check_feature(dev, DRIVER_MODESET))
4578 return 0; 3635 return 0;
@@ -4592,14 +3649,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4592 } 3649 }
4593 3650
4594 BUG_ON(!list_empty(&dev_priv->mm.active_list)); 3651 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4595 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4596 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4597 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4598 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 3652 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4599 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 3653 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4600 BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); 3654 for (i = 0; i < I915_NUM_RINGS; i++) {
4601 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list)); 3655 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
4602 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list)); 3656 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3657 }
4603 mutex_unlock(&dev->struct_mutex); 3658 mutex_unlock(&dev->struct_mutex);
4604 3659
4605 ret = drm_irq_install(dev); 3660 ret = drm_irq_install(dev);
@@ -4661,17 +3716,14 @@ i915_gem_load(struct drm_device *dev)
4661 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 3716 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4662 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3717 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4663 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 3718 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4664 init_ring_lists(&dev_priv->render_ring); 3719 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
4665 init_ring_lists(&dev_priv->bsd_ring); 3720 for (i = 0; i < I915_NUM_RINGS; i++)
4666 init_ring_lists(&dev_priv->blt_ring); 3721 init_ring_lists(&dev_priv->ring[i]);
4667 for (i = 0; i < 16; i++) 3722 for (i = 0; i < 16; i++)
4668 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 3723 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4669 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 3724 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4670 i915_gem_retire_work_handler); 3725 i915_gem_retire_work_handler);
4671 init_completion(&dev_priv->error_completion); 3726 init_completion(&dev_priv->error_completion);
4672 spin_lock(&shrink_list_lock);
4673 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4674 spin_unlock(&shrink_list_lock);
4675 3727
4676 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 3728 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4677 if (IS_GEN3(dev)) { 3729 if (IS_GEN3(dev)) {
@@ -4683,6 +3735,8 @@ i915_gem_load(struct drm_device *dev)
4683 } 3735 }
4684 } 3736 }
4685 3737
3738 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3739
4686 /* Old X drivers will take 0-2 for front, back, depth buffers */ 3740 /* Old X drivers will take 0-2 for front, back, depth buffers */
4687 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 3741 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4688 dev_priv->fence_reg_start = 3; 3742 dev_priv->fence_reg_start = 3;
@@ -4714,6 +3768,10 @@ i915_gem_load(struct drm_device *dev)
4714 } 3768 }
4715 i915_gem_detect_bit_6_swizzle(dev); 3769 i915_gem_detect_bit_6_swizzle(dev);
4716 init_waitqueue_head(&dev_priv->pending_flip_queue); 3770 init_waitqueue_head(&dev_priv->pending_flip_queue);
3771
3772 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3773 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3774 register_shrinker(&dev_priv->mm.inactive_shrinker);
4717} 3775}
4718 3776
4719/* 3777/*
@@ -4783,47 +3841,47 @@ void i915_gem_free_all_phys_object(struct drm_device *dev)
4783} 3841}
4784 3842
4785void i915_gem_detach_phys_object(struct drm_device *dev, 3843void i915_gem_detach_phys_object(struct drm_device *dev,
4786 struct drm_gem_object *obj) 3844 struct drm_i915_gem_object *obj)
4787{ 3845{
4788 struct drm_i915_gem_object *obj_priv; 3846 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3847 char *vaddr;
4789 int i; 3848 int i;
4790 int ret;
4791 int page_count; 3849 int page_count;
4792 3850
4793 obj_priv = to_intel_bo(obj); 3851 if (!obj->phys_obj)
4794 if (!obj_priv->phys_obj)
4795 return; 3852 return;
3853 vaddr = obj->phys_obj->handle->vaddr;
4796 3854
4797 ret = i915_gem_object_get_pages(obj, 0); 3855 page_count = obj->base.size / PAGE_SIZE;
4798 if (ret)
4799 goto out;
4800
4801 page_count = obj->size / PAGE_SIZE;
4802
4803 for (i = 0; i < page_count; i++) { 3856 for (i = 0; i < page_count; i++) {
4804 char *dst = kmap_atomic(obj_priv->pages[i]); 3857 struct page *page = read_cache_page_gfp(mapping, i,
4805 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3858 GFP_HIGHUSER | __GFP_RECLAIMABLE);
4806 3859 if (!IS_ERR(page)) {
4807 memcpy(dst, src, PAGE_SIZE); 3860 char *dst = kmap_atomic(page);
4808 kunmap_atomic(dst); 3861 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3862 kunmap_atomic(dst);
3863
3864 drm_clflush_pages(&page, 1);
3865
3866 set_page_dirty(page);
3867 mark_page_accessed(page);
3868 page_cache_release(page);
3869 }
4809 } 3870 }
4810 drm_clflush_pages(obj_priv->pages, page_count); 3871 intel_gtt_chipset_flush();
4811 drm_agp_chipset_flush(dev);
4812 3872
4813 i915_gem_object_put_pages(obj); 3873 obj->phys_obj->cur_obj = NULL;
4814out: 3874 obj->phys_obj = NULL;
4815 obj_priv->phys_obj->cur_obj = NULL;
4816 obj_priv->phys_obj = NULL;
4817} 3875}
4818 3876
4819int 3877int
4820i915_gem_attach_phys_object(struct drm_device *dev, 3878i915_gem_attach_phys_object(struct drm_device *dev,
4821 struct drm_gem_object *obj, 3879 struct drm_i915_gem_object *obj,
4822 int id, 3880 int id,
4823 int align) 3881 int align)
4824{ 3882{
3883 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4825 drm_i915_private_t *dev_priv = dev->dev_private; 3884 drm_i915_private_t *dev_priv = dev->dev_private;
4826 struct drm_i915_gem_object *obj_priv;
4827 int ret = 0; 3885 int ret = 0;
4828 int page_count; 3886 int page_count;
4829 int i; 3887 int i;
@@ -4831,10 +3889,8 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4831 if (id > I915_MAX_PHYS_OBJECT) 3889 if (id > I915_MAX_PHYS_OBJECT)
4832 return -EINVAL; 3890 return -EINVAL;
4833 3891
4834 obj_priv = to_intel_bo(obj); 3892 if (obj->phys_obj) {
4835 3893 if (obj->phys_obj->id == id)
4836 if (obj_priv->phys_obj) {
4837 if (obj_priv->phys_obj->id == id)
4838 return 0; 3894 return 0;
4839 i915_gem_detach_phys_object(dev, obj); 3895 i915_gem_detach_phys_object(dev, obj);
4840 } 3896 }
@@ -4842,51 +3898,50 @@ i915_gem_attach_phys_object(struct drm_device *dev,
4842 /* create a new object */ 3898 /* create a new object */
4843 if (!dev_priv->mm.phys_objs[id - 1]) { 3899 if (!dev_priv->mm.phys_objs[id - 1]) {
4844 ret = i915_gem_init_phys_object(dev, id, 3900 ret = i915_gem_init_phys_object(dev, id,
4845 obj->size, align); 3901 obj->base.size, align);
4846 if (ret) { 3902 if (ret) {
4847 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); 3903 DRM_ERROR("failed to init phys object %d size: %zu\n",
4848 goto out; 3904 id, obj->base.size);
3905 return ret;
4849 } 3906 }
4850 } 3907 }
4851 3908
4852 /* bind to the object */ 3909 /* bind to the object */
4853 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3910 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4854 obj_priv->phys_obj->cur_obj = obj; 3911 obj->phys_obj->cur_obj = obj;
4855
4856 ret = i915_gem_object_get_pages(obj, 0);
4857 if (ret) {
4858 DRM_ERROR("failed to get page list\n");
4859 goto out;
4860 }
4861 3912
4862 page_count = obj->size / PAGE_SIZE; 3913 page_count = obj->base.size / PAGE_SIZE;
4863 3914
4864 for (i = 0; i < page_count; i++) { 3915 for (i = 0; i < page_count; i++) {
4865 char *src = kmap_atomic(obj_priv->pages[i]); 3916 struct page *page;
4866 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); 3917 char *dst, *src;
4867 3918
3919 page = read_cache_page_gfp(mapping, i,
3920 GFP_HIGHUSER | __GFP_RECLAIMABLE);
3921 if (IS_ERR(page))
3922 return PTR_ERR(page);
3923
3924 src = kmap_atomic(page);
3925 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4868 memcpy(dst, src, PAGE_SIZE); 3926 memcpy(dst, src, PAGE_SIZE);
4869 kunmap_atomic(src); 3927 kunmap_atomic(src);
4870 }
4871 3928
4872 i915_gem_object_put_pages(obj); 3929 mark_page_accessed(page);
3930 page_cache_release(page);
3931 }
4873 3932
4874 return 0; 3933 return 0;
4875out:
4876 return ret;
4877} 3934}
4878 3935
4879static int 3936static int
4880i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, 3937i915_gem_phys_pwrite(struct drm_device *dev,
3938 struct drm_i915_gem_object *obj,
4881 struct drm_i915_gem_pwrite *args, 3939 struct drm_i915_gem_pwrite *args,
4882 struct drm_file *file_priv) 3940 struct drm_file *file_priv)
4883{ 3941{
4884 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3942 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4885 void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
4886 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr; 3943 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
4887 3944
4888 DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
4889
4890 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 3945 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4891 unsigned long unwritten; 3946 unsigned long unwritten;
4892 3947
@@ -4901,7 +3956,7 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4901 return -EFAULT; 3956 return -EFAULT;
4902 } 3957 }
4903 3958
4904 drm_agp_chipset_flush(dev); 3959 intel_gtt_chipset_flush();
4905 return 0; 3960 return 0;
4906} 3961}
4907 3962
@@ -4939,144 +3994,68 @@ i915_gpu_is_active(struct drm_device *dev)
4939} 3994}
4940 3995
4941static int 3996static int
4942i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) 3997i915_gem_inactive_shrink(struct shrinker *shrinker,
4943{ 3998 int nr_to_scan,
4944 drm_i915_private_t *dev_priv, *next_dev; 3999 gfp_t gfp_mask)
4945 struct drm_i915_gem_object *obj_priv, *next_obj; 4000{
4946 int cnt = 0; 4001 struct drm_i915_private *dev_priv =
4947 int would_deadlock = 1; 4002 container_of(shrinker,
4003 struct drm_i915_private,
4004 mm.inactive_shrinker);
4005 struct drm_device *dev = dev_priv->dev;
4006 struct drm_i915_gem_object *obj, *next;
4007 int cnt;
4008
4009 if (!mutex_trylock(&dev->struct_mutex))
4010 return 0;
4948 4011
4949 /* "fast-path" to count number of available objects */ 4012 /* "fast-path" to count number of available objects */
4950 if (nr_to_scan == 0) { 4013 if (nr_to_scan == 0) {
4951 spin_lock(&shrink_list_lock); 4014 cnt = 0;
4952 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4015 list_for_each_entry(obj,
4953 struct drm_device *dev = dev_priv->dev; 4016 &dev_priv->mm.inactive_list,
4954 4017 mm_list)
4955 if (mutex_trylock(&dev->struct_mutex)) { 4018 cnt++;
4956 list_for_each_entry(obj_priv, 4019 mutex_unlock(&dev->struct_mutex);
4957 &dev_priv->mm.inactive_list, 4020 return cnt / 100 * sysctl_vfs_cache_pressure;
4958 mm_list)
4959 cnt++;
4960 mutex_unlock(&dev->struct_mutex);
4961 }
4962 }
4963 spin_unlock(&shrink_list_lock);
4964
4965 return (cnt / 100) * sysctl_vfs_cache_pressure;
4966 } 4021 }
4967 4022
4968 spin_lock(&shrink_list_lock);
4969
4970rescan: 4023rescan:
4971 /* first scan for clean buffers */ 4024 /* first scan for clean buffers */
4972 list_for_each_entry_safe(dev_priv, next_dev, 4025 i915_gem_retire_requests(dev);
4973 &shrink_list, mm.shrink_list) {
4974 struct drm_device *dev = dev_priv->dev;
4975
4976 if (! mutex_trylock(&dev->struct_mutex))
4977 continue;
4978
4979 spin_unlock(&shrink_list_lock);
4980 i915_gem_retire_requests(dev);
4981 4026
4982 list_for_each_entry_safe(obj_priv, next_obj, 4027 list_for_each_entry_safe(obj, next,
4983 &dev_priv->mm.inactive_list, 4028 &dev_priv->mm.inactive_list,
4984 mm_list) { 4029 mm_list) {
4985 if (i915_gem_object_is_purgeable(obj_priv)) { 4030 if (i915_gem_object_is_purgeable(obj)) {
4986 i915_gem_object_unbind(&obj_priv->base); 4031 if (i915_gem_object_unbind(obj) == 0 &&
4987 if (--nr_to_scan <= 0) 4032 --nr_to_scan == 0)
4988 break; 4033 break;
4989 }
4990 } 4034 }
4991
4992 spin_lock(&shrink_list_lock);
4993 mutex_unlock(&dev->struct_mutex);
4994
4995 would_deadlock = 0;
4996
4997 if (nr_to_scan <= 0)
4998 break;
4999 } 4035 }
5000 4036
5001 /* second pass, evict/count anything still on the inactive list */ 4037 /* second pass, evict/count anything still on the inactive list */
5002 list_for_each_entry_safe(dev_priv, next_dev, 4038 cnt = 0;
5003 &shrink_list, mm.shrink_list) { 4039 list_for_each_entry_safe(obj, next,
5004 struct drm_device *dev = dev_priv->dev; 4040 &dev_priv->mm.inactive_list,
5005 4041 mm_list) {
5006 if (! mutex_trylock(&dev->struct_mutex)) 4042 if (nr_to_scan &&
5007 continue; 4043 i915_gem_object_unbind(obj) == 0)
5008 4044 nr_to_scan--;
5009 spin_unlock(&shrink_list_lock); 4045 else
5010 4046 cnt++;
5011 list_for_each_entry_safe(obj_priv, next_obj,
5012 &dev_priv->mm.inactive_list,
5013 mm_list) {
5014 if (nr_to_scan > 0) {
5015 i915_gem_object_unbind(&obj_priv->base);
5016 nr_to_scan--;
5017 } else
5018 cnt++;
5019 }
5020
5021 spin_lock(&shrink_list_lock);
5022 mutex_unlock(&dev->struct_mutex);
5023
5024 would_deadlock = 0;
5025 } 4047 }
5026 4048
5027 if (nr_to_scan) { 4049 if (nr_to_scan && i915_gpu_is_active(dev)) {
5028 int active = 0;
5029
5030 /* 4050 /*
5031 * We are desperate for pages, so as a last resort, wait 4051 * We are desperate for pages, so as a last resort, wait
5032 * for the GPU to finish and discard whatever we can. 4052 * for the GPU to finish and discard whatever we can.
5033 * This has a dramatic impact to reduce the number of 4053 * This has a dramatic impact to reduce the number of
5034 * OOM-killer events whilst running the GPU aggressively. 4054 * OOM-killer events whilst running the GPU aggressively.
5035 */ 4055 */
5036 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { 4056 if (i915_gpu_idle(dev) == 0)
5037 struct drm_device *dev = dev_priv->dev;
5038
5039 if (!mutex_trylock(&dev->struct_mutex))
5040 continue;
5041
5042 spin_unlock(&shrink_list_lock);
5043
5044 if (i915_gpu_is_active(dev)) {
5045 i915_gpu_idle(dev);
5046 active++;
5047 }
5048
5049 spin_lock(&shrink_list_lock);
5050 mutex_unlock(&dev->struct_mutex);
5051 }
5052
5053 if (active)
5054 goto rescan; 4057 goto rescan;
5055 } 4058 }
5056 4059 mutex_unlock(&dev->struct_mutex);
5057 spin_unlock(&shrink_list_lock); 4060 return cnt / 100 * sysctl_vfs_cache_pressure;
5058
5059 if (would_deadlock)
5060 return -1;
5061 else if (cnt > 0)
5062 return (cnt / 100) * sysctl_vfs_cache_pressure;
5063 else
5064 return 0;
5065}
5066
5067static struct shrinker shrinker = {
5068 .shrink = i915_gem_shrink,
5069 .seeks = DEFAULT_SEEKS,
5070};
5071
5072__init void
5073i915_gem_shrinker_init(void)
5074{
5075 register_shrinker(&shrinker);
5076}
5077
5078__exit void
5079i915_gem_shrinker_exit(void)
5080{
5081 unregister_shrinker(&shrinker);
5082} 4061}
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-10 20:22:48 -0400