aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c1279
1 files changed, 577 insertions, 702 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 100a7537980..6c2618d884e 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -244,12 +244,17 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
244 return -ENOMEM; 244 return -ENOMEM;
245 245
246 ret = drm_gem_handle_create(file_priv, obj, &handle); 246 ret = drm_gem_handle_create(file_priv, obj, &handle);
247 /* drop reference from allocate - handle holds it now */
248 drm_gem_object_unreference_unlocked(obj);
249 if (ret) { 247 if (ret) {
248 drm_gem_object_release(obj);
249 i915_gem_info_remove_obj(dev->dev_private, obj->size);
250 kfree(obj);
250 return ret; 251 return ret;
251 } 252 }
252 253
254 /* drop reference from allocate - handle holds it now */
255 drm_gem_object_unreference(obj);
256 trace_i915_gem_object_create(obj);
257
253 args->handle = handle; 258 args->handle = handle;
254 return 0; 259 return 0;
255} 260}
@@ -260,19 +265,14 @@ fast_shmem_read(struct page **pages,
260 char __user *data, 265 char __user *data,
261 int length) 266 int length)
262{ 267{
263 char __iomem *vaddr; 268 char *vaddr;
264 int unwritten; 269 int ret;
265 270
266 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0); 271 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
267 if (vaddr == NULL) 272 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
268 return -ENOMEM;
269 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
270 kunmap_atomic(vaddr, KM_USER0); 273 kunmap_atomic(vaddr, KM_USER0);
271 274
272 if (unwritten) 275 return ret;
273 return -EFAULT;
274
275 return 0;
276} 276}
277 277
278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj) 278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
@@ -366,24 +366,10 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
366 loff_t offset, page_base; 366 loff_t offset, page_base;
367 char __user *user_data; 367 char __user *user_data;
368 int page_offset, page_length; 368 int page_offset, page_length;
369 int ret;
370 369
371 user_data = (char __user *) (uintptr_t) args->data_ptr; 370 user_data = (char __user *) (uintptr_t) args->data_ptr;
372 remain = args->size; 371 remain = args->size;
373 372
374 ret = i915_mutex_lock_interruptible(dev);
375 if (ret)
376 return ret;
377
378 ret = i915_gem_object_get_pages(obj, 0);
379 if (ret != 0)
380 goto fail_unlock;
381
382 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
383 args->size);
384 if (ret != 0)
385 goto fail_put_pages;
386
387 obj_priv = to_intel_bo(obj); 373 obj_priv = to_intel_bo(obj);
388 offset = args->offset; 374 offset = args->offset;
389 375
@@ -400,23 +386,17 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
400 if ((page_offset + remain) > PAGE_SIZE) 386 if ((page_offset + remain) > PAGE_SIZE)
401 page_length = PAGE_SIZE - page_offset; 387 page_length = PAGE_SIZE - page_offset;
402 388
403 ret = fast_shmem_read(obj_priv->pages, 389 if (fast_shmem_read(obj_priv->pages,
404 page_base, page_offset, 390 page_base, page_offset,
405 user_data, page_length); 391 user_data, page_length))
406 if (ret) 392 return -EFAULT;
407 goto fail_put_pages;
408 393
409 remain -= page_length; 394 remain -= page_length;
410 user_data += page_length; 395 user_data += page_length;
411 offset += page_length; 396 offset += page_length;
412 } 397 }
413 398
414fail_put_pages: 399 return 0;
415 i915_gem_object_put_pages(obj);
416fail_unlock:
417 mutex_unlock(&dev->struct_mutex);
418
419 return ret;
420} 400}
421 401
422static int 402static int
@@ -477,33 +457,28 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
477 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 457 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
478 num_pages = last_data_page - first_data_page + 1; 458 num_pages = last_data_page - first_data_page + 1;
479 459
480 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 460 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
481 if (user_pages == NULL) 461 if (user_pages == NULL)
482 return -ENOMEM; 462 return -ENOMEM;
483 463
464 mutex_unlock(&dev->struct_mutex);
484 down_read(&mm->mmap_sem); 465 down_read(&mm->mmap_sem);
485 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 466 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
486 num_pages, 1, 0, user_pages, NULL); 467 num_pages, 1, 0, user_pages, NULL);
487 up_read(&mm->mmap_sem); 468 up_read(&mm->mmap_sem);
469 mutex_lock(&dev->struct_mutex);
488 if (pinned_pages < num_pages) { 470 if (pinned_pages < num_pages) {
489 ret = -EFAULT; 471 ret = -EFAULT;
490 goto fail_put_user_pages; 472 goto out;
491 } 473 }
492 474
493 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 475 ret = i915_gem_object_set_cpu_read_domain_range(obj,
494 476 args->offset,
495 ret = i915_mutex_lock_interruptible(dev); 477 args->size);
496 if (ret)
497 goto fail_put_user_pages;
498
499 ret = i915_gem_object_get_pages_or_evict(obj);
500 if (ret) 478 if (ret)
501 goto fail_unlock; 479 goto out;
502 480
503 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, 481 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
504 args->size);
505 if (ret != 0)
506 goto fail_put_pages;
507 482
508 obj_priv = to_intel_bo(obj); 483 obj_priv = to_intel_bo(obj);
509 offset = args->offset; 484 offset = args->offset;
@@ -548,11 +523,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
548 offset += page_length; 523 offset += page_length;
549 } 524 }
550 525
551fail_put_pages: 526out:
552 i915_gem_object_put_pages(obj);
553fail_unlock:
554 mutex_unlock(&dev->struct_mutex);
555fail_put_user_pages:
556 for (i = 0; i < pinned_pages; i++) { 527 for (i = 0; i < pinned_pages; i++) {
557 SetPageDirty(user_pages[i]); 528 SetPageDirty(user_pages[i]);
558 page_cache_release(user_pages[i]); 529 page_cache_release(user_pages[i]);
@@ -576,9 +547,15 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
576 struct drm_i915_gem_object *obj_priv; 547 struct drm_i915_gem_object *obj_priv;
577 int ret = 0; 548 int ret = 0;
578 549
550 ret = i915_mutex_lock_interruptible(dev);
551 if (ret)
552 return ret;
553
579 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 554 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
580 if (obj == NULL) 555 if (obj == NULL) {
581 return -ENOENT; 556 ret = -ENOENT;
557 goto unlock;
558 }
582 obj_priv = to_intel_bo(obj); 559 obj_priv = to_intel_bo(obj);
583 560
584 /* Bounds check source. */ 561 /* Bounds check source. */
@@ -597,17 +574,35 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
597 goto out; 574 goto out;
598 } 575 }
599 576
600 if (i915_gem_object_needs_bit17_swizzle(obj)) { 577 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
601 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); 578 args->size);
602 } else { 579 if (ret) {
603 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); 580 ret = -EFAULT;
604 if (ret != 0) 581 goto out;
605 ret = i915_gem_shmem_pread_slow(dev, obj, args,
606 file_priv);
607 } 582 }
608 583
584 ret = i915_gem_object_get_pages_or_evict(obj);
585 if (ret)
586 goto out;
587
588 ret = i915_gem_object_set_cpu_read_domain_range(obj,
589 args->offset,
590 args->size);
591 if (ret)
592 goto out_put;
593
594 ret = -EFAULT;
595 if (!i915_gem_object_needs_bit17_swizzle(obj))
596 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
597 if (ret == -EFAULT)
598 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
599
600out_put:
601 i915_gem_object_put_pages(obj);
609out: 602out:
610 drm_gem_object_unreference_unlocked(obj); 603 drm_gem_object_unreference(obj);
604unlock:
605 mutex_unlock(&dev->struct_mutex);
611 return ret; 606 return ret;
612} 607}
613 608
@@ -628,9 +623,7 @@ fast_user_write(struct io_mapping *mapping,
628 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset, 623 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
629 user_data, length); 624 user_data, length);
630 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0); 625 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
631 if (unwritten) 626 return unwritten;
632 return -EFAULT;
633 return 0;
634} 627}
635 628
636/* Here's the write path which can sleep for 629/* Here's the write path which can sleep for
@@ -663,18 +656,14 @@ fast_shmem_write(struct page **pages,
663 char __user *data, 656 char __user *data,
664 int length) 657 int length)
665{ 658{
666 char __iomem *vaddr; 659 char *vaddr;
667 unsigned long unwritten; 660 int ret;
668 661
669 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0); 662 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
670 if (vaddr == NULL) 663 ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
671 return -ENOMEM;
672 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
673 kunmap_atomic(vaddr, KM_USER0); 664 kunmap_atomic(vaddr, KM_USER0);
674 665
675 if (unwritten) 666 return ret;
676 return -EFAULT;
677 return 0;
678} 667}
679 668
680/** 669/**
@@ -692,24 +681,10 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
692 loff_t offset, page_base; 681 loff_t offset, page_base;
693 char __user *user_data; 682 char __user *user_data;
694 int page_offset, page_length; 683 int page_offset, page_length;
695 int ret;
696 684
697 user_data = (char __user *) (uintptr_t) args->data_ptr; 685 user_data = (char __user *) (uintptr_t) args->data_ptr;
698 remain = args->size; 686 remain = args->size;
699 687
700 ret = i915_mutex_lock_interruptible(dev);
701 if (ret)
702 return ret;
703
704 ret = i915_gem_object_pin(obj, 0);
705 if (ret) {
706 mutex_unlock(&dev->struct_mutex);
707 return ret;
708 }
709 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
710 if (ret)
711 goto fail;
712
713 obj_priv = to_intel_bo(obj); 688 obj_priv = to_intel_bo(obj);
714 offset = obj_priv->gtt_offset + args->offset; 689 offset = obj_priv->gtt_offset + args->offset;
715 690
@@ -726,26 +701,21 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
726 if ((page_offset + remain) > PAGE_SIZE) 701 if ((page_offset + remain) > PAGE_SIZE)
727 page_length = PAGE_SIZE - page_offset; 702 page_length = PAGE_SIZE - page_offset;
728 703
729 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
730 page_offset, user_data, page_length);
731
732 /* If we get a fault while copying data, then (presumably) our 704 /* If we get a fault while copying data, then (presumably) our
733 * source page isn't available. Return the error and we'll 705 * source page isn't available. Return the error and we'll
734 * retry in the slow path. 706 * retry in the slow path.
735 */ 707 */
736 if (ret) 708 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
737 goto fail; 709 page_offset, user_data, page_length))
710
711 return -EFAULT;
738 712
739 remain -= page_length; 713 remain -= page_length;
740 user_data += page_length; 714 user_data += page_length;
741 offset += page_length; 715 offset += page_length;
742 } 716 }
743 717
744fail: 718 return 0;
745 i915_gem_object_unpin(obj);
746 mutex_unlock(&dev->struct_mutex);
747
748 return ret;
749} 719}
750 720
751/** 721/**
@@ -782,30 +752,24 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
782 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 752 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
783 num_pages = last_data_page - first_data_page + 1; 753 num_pages = last_data_page - first_data_page + 1;
784 754
785 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 755 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
786 if (user_pages == NULL) 756 if (user_pages == NULL)
787 return -ENOMEM; 757 return -ENOMEM;
788 758
759 mutex_unlock(&dev->struct_mutex);
789 down_read(&mm->mmap_sem); 760 down_read(&mm->mmap_sem);
790 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 761 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
791 num_pages, 0, 0, user_pages, NULL); 762 num_pages, 0, 0, user_pages, NULL);
792 up_read(&mm->mmap_sem); 763 up_read(&mm->mmap_sem);
764 mutex_lock(&dev->struct_mutex);
793 if (pinned_pages < num_pages) { 765 if (pinned_pages < num_pages) {
794 ret = -EFAULT; 766 ret = -EFAULT;
795 goto out_unpin_pages; 767 goto out_unpin_pages;
796 } 768 }
797 769
798 ret = i915_mutex_lock_interruptible(dev);
799 if (ret)
800 goto out_unpin_pages;
801
802 ret = i915_gem_object_pin(obj, 0);
803 if (ret)
804 goto out_unlock;
805
806 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 770 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
807 if (ret) 771 if (ret)
808 goto out_unpin_object; 772 goto out_unpin_pages;
809 773
810 obj_priv = to_intel_bo(obj); 774 obj_priv = to_intel_bo(obj);
811 offset = obj_priv->gtt_offset + args->offset; 775 offset = obj_priv->gtt_offset + args->offset;
@@ -841,10 +805,6 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
841 data_ptr += page_length; 805 data_ptr += page_length;
842 } 806 }
843 807
844out_unpin_object:
845 i915_gem_object_unpin(obj);
846out_unlock:
847 mutex_unlock(&dev->struct_mutex);
848out_unpin_pages: 808out_unpin_pages:
849 for (i = 0; i < pinned_pages; i++) 809 for (i = 0; i < pinned_pages; i++)
850 page_cache_release(user_pages[i]); 810 page_cache_release(user_pages[i]);
@@ -867,23 +827,10 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
867 loff_t offset, page_base; 827 loff_t offset, page_base;
868 char __user *user_data; 828 char __user *user_data;
869 int page_offset, page_length; 829 int page_offset, page_length;
870 int ret;
871 830
872 user_data = (char __user *) (uintptr_t) args->data_ptr; 831 user_data = (char __user *) (uintptr_t) args->data_ptr;
873 remain = args->size; 832 remain = args->size;
874 833
875 ret = i915_mutex_lock_interruptible(dev);
876 if (ret)
877 return ret;
878
879 ret = i915_gem_object_get_pages(obj, 0);
880 if (ret != 0)
881 goto fail_unlock;
882
883 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
884 if (ret != 0)
885 goto fail_put_pages;
886
887 obj_priv = to_intel_bo(obj); 834 obj_priv = to_intel_bo(obj);
888 offset = args->offset; 835 offset = args->offset;
889 obj_priv->dirty = 1; 836 obj_priv->dirty = 1;
@@ -901,23 +848,17 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
901 if ((page_offset + remain) > PAGE_SIZE) 848 if ((page_offset + remain) > PAGE_SIZE)
902 page_length = PAGE_SIZE - page_offset; 849 page_length = PAGE_SIZE - page_offset;
903 850
904 ret = fast_shmem_write(obj_priv->pages, 851 if (fast_shmem_write(obj_priv->pages,
905 page_base, page_offset, 852 page_base, page_offset,
906 user_data, page_length); 853 user_data, page_length))
907 if (ret) 854 return -EFAULT;
908 goto fail_put_pages;
909 855
910 remain -= page_length; 856 remain -= page_length;
911 user_data += page_length; 857 user_data += page_length;
912 offset += page_length; 858 offset += page_length;
913 } 859 }
914 860
915fail_put_pages: 861 return 0;
916 i915_gem_object_put_pages(obj);
917fail_unlock:
918 mutex_unlock(&dev->struct_mutex);
919
920 return ret;
921} 862}
922 863
923/** 864/**
@@ -955,32 +896,26 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
955 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; 896 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
956 num_pages = last_data_page - first_data_page + 1; 897 num_pages = last_data_page - first_data_page + 1;
957 898
958 user_pages = drm_calloc_large(num_pages, sizeof(struct page *)); 899 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
959 if (user_pages == NULL) 900 if (user_pages == NULL)
960 return -ENOMEM; 901 return -ENOMEM;
961 902
903 mutex_unlock(&dev->struct_mutex);
962 down_read(&mm->mmap_sem); 904 down_read(&mm->mmap_sem);
963 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, 905 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
964 num_pages, 0, 0, user_pages, NULL); 906 num_pages, 0, 0, user_pages, NULL);
965 up_read(&mm->mmap_sem); 907 up_read(&mm->mmap_sem);
908 mutex_lock(&dev->struct_mutex);
966 if (pinned_pages < num_pages) { 909 if (pinned_pages < num_pages) {
967 ret = -EFAULT; 910 ret = -EFAULT;
968 goto fail_put_user_pages; 911 goto out;
969 } 912 }
970 913
971 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 914 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
972
973 ret = i915_mutex_lock_interruptible(dev);
974 if (ret)
975 goto fail_put_user_pages;
976
977 ret = i915_gem_object_get_pages_or_evict(obj);
978 if (ret) 915 if (ret)
979 goto fail_unlock; 916 goto out;
980 917
981 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 918 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
982 if (ret != 0)
983 goto fail_put_pages;
984 919
985 obj_priv = to_intel_bo(obj); 920 obj_priv = to_intel_bo(obj);
986 offset = args->offset; 921 offset = args->offset;
@@ -1026,11 +961,7 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
1026 offset += page_length; 961 offset += page_length;
1027 } 962 }
1028 963
1029fail_put_pages: 964out:
1030 i915_gem_object_put_pages(obj);
1031fail_unlock:
1032 mutex_unlock(&dev->struct_mutex);
1033fail_put_user_pages:
1034 for (i = 0; i < pinned_pages; i++) 965 for (i = 0; i < pinned_pages; i++)
1035 page_cache_release(user_pages[i]); 966 page_cache_release(user_pages[i]);
1036 drm_free_large(user_pages); 967 drm_free_large(user_pages);
@@ -1045,18 +976,25 @@ fail_put_user_pages:
1045 */ 976 */
1046int 977int
1047i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 978i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1048 struct drm_file *file_priv) 979 struct drm_file *file)
1049{ 980{
1050 struct drm_i915_gem_pwrite *args = data; 981 struct drm_i915_gem_pwrite *args = data;
1051 struct drm_gem_object *obj; 982 struct drm_gem_object *obj;
1052 struct drm_i915_gem_object *obj_priv; 983 struct drm_i915_gem_object *obj_priv;
1053 int ret = 0; 984 int ret = 0;
1054 985
1055 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 986 ret = i915_mutex_lock_interruptible(dev);
1056 if (obj == NULL) 987 if (ret)
1057 return -ENOENT; 988 return ret;
989
990 obj = drm_gem_object_lookup(dev, file, args->handle);
991 if (obj == NULL) {
992 ret = -ENOENT;
993 goto unlock;
994 }
1058 obj_priv = to_intel_bo(obj); 995 obj_priv = to_intel_bo(obj);
1059 996
997
1060 /* Bounds check destination. */ 998 /* Bounds check destination. */
1061 if (args->offset > obj->size || args->size > obj->size - args->offset) { 999 if (args->offset > obj->size || args->size > obj->size - args->offset) {
1062 ret = -EINVAL; 1000 ret = -EINVAL;
@@ -1073,6 +1011,13 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1073 goto out; 1011 goto out;
1074 } 1012 }
1075 1013
1014 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1015 args->size);
1016 if (ret) {
1017 ret = -EFAULT;
1018 goto out;
1019 }
1020
1076 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1021 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1077 * it would end up going through the fenced access, and we'll get 1022 * it would end up going through the fenced access, and we'll get
1078 * different detiling behavior between reading and writing. 1023 * different detiling behavior between reading and writing.
@@ -1080,32 +1025,47 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1080 * perspective, requiring manual detiling by the client. 1025 * perspective, requiring manual detiling by the client.
1081 */ 1026 */
1082 if (obj_priv->phys_obj) 1027 if (obj_priv->phys_obj)
1083 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv); 1028 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1084 else if (obj_priv->tiling_mode == I915_TILING_NONE && 1029 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1085 obj_priv->gtt_space && 1030 obj_priv->gtt_space &&
1086 obj->write_domain != I915_GEM_DOMAIN_CPU) { 1031 obj->write_domain != I915_GEM_DOMAIN_CPU) {
1087 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv); 1032 ret = i915_gem_object_pin(obj, 0);
1088 if (ret == -EFAULT) { 1033 if (ret)
1089 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, 1034 goto out;
1090 file_priv); 1035
1091 } 1036 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1092 } else if (i915_gem_object_needs_bit17_swizzle(obj)) { 1037 if (ret)
1093 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv); 1038 goto out_unpin;
1039
1040 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1041 if (ret == -EFAULT)
1042 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1043
1044out_unpin:
1045 i915_gem_object_unpin(obj);
1094 } else { 1046 } else {
1095 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv); 1047 ret = i915_gem_object_get_pages_or_evict(obj);
1096 if (ret == -EFAULT) { 1048 if (ret)
1097 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, 1049 goto out;
1098 file_priv);
1099 }
1100 }
1101 1050
1102#if WATCH_PWRITE 1051 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1103 if (ret) 1052 if (ret)
1104 DRM_INFO("pwrite failed %d\n", ret); 1053 goto out_put;
1105#endif 1054
1055 ret = -EFAULT;
1056 if (!i915_gem_object_needs_bit17_swizzle(obj))
1057 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1058 if (ret == -EFAULT)
1059 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1060
1061out_put:
1062 i915_gem_object_put_pages(obj);
1063 }
1106 1064
1107out: 1065out:
1108 drm_gem_object_unreference_unlocked(obj); 1066 drm_gem_object_unreference(obj);
1067unlock:
1068 mutex_unlock(&dev->struct_mutex);
1109 return ret; 1069 return ret;
1110} 1070}
1111 1071
@@ -1141,16 +1101,16 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1141 if (write_domain != 0 && read_domains != write_domain) 1101 if (write_domain != 0 && read_domains != write_domain)
1142 return -EINVAL; 1102 return -EINVAL;
1143 1103
1144 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1145 if (obj == NULL)
1146 return -ENOENT;
1147 obj_priv = to_intel_bo(obj);
1148
1149 ret = i915_mutex_lock_interruptible(dev); 1104 ret = i915_mutex_lock_interruptible(dev);
1150 if (ret) { 1105 if (ret)
1151 drm_gem_object_unreference_unlocked(obj);
1152 return ret; 1106 return ret;
1107
1108 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1109 if (obj == NULL) {
1110 ret = -ENOENT;
1111 goto unlock;
1153 } 1112 }
1113 obj_priv = to_intel_bo(obj);
1154 1114
1155 intel_mark_busy(dev, obj); 1115 intel_mark_busy(dev, obj);
1156 1116
@@ -1179,9 +1139,10 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1179 1139
1180 /* Maintain LRU order of "inactive" objects */ 1140 /* Maintain LRU order of "inactive" objects */
1181 if (ret == 0 && i915_gem_object_is_inactive(obj_priv)) 1141 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1182 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1142 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1183 1143
1184 drm_gem_object_unreference(obj); 1144 drm_gem_object_unreference(obj);
1145unlock:
1185 mutex_unlock(&dev->struct_mutex); 1146 mutex_unlock(&dev->struct_mutex);
1186 return ret; 1147 return ret;
1187} 1148}
@@ -1200,14 +1161,14 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1200 if (!(dev->driver->driver_features & DRIVER_GEM)) 1161 if (!(dev->driver->driver_features & DRIVER_GEM))
1201 return -ENODEV; 1162 return -ENODEV;
1202 1163
1203 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1204 if (obj == NULL)
1205 return -ENOENT;
1206
1207 ret = i915_mutex_lock_interruptible(dev); 1164 ret = i915_mutex_lock_interruptible(dev);
1208 if (ret) { 1165 if (ret)
1209 drm_gem_object_unreference_unlocked(obj);
1210 return ret; 1166 return ret;
1167
1168 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1169 if (obj == NULL) {
1170 ret = -ENOENT;
1171 goto unlock;
1211 } 1172 }
1212 1173
1213 /* Pinned buffers may be scanout, so flush the cache */ 1174 /* Pinned buffers may be scanout, so flush the cache */
@@ -1215,6 +1176,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1215 i915_gem_object_flush_cpu_write_domain(obj); 1176 i915_gem_object_flush_cpu_write_domain(obj);
1216 1177
1217 drm_gem_object_unreference(obj); 1178 drm_gem_object_unreference(obj);
1179unlock:
1218 mutex_unlock(&dev->struct_mutex); 1180 mutex_unlock(&dev->struct_mutex);
1219 return ret; 1181 return ret;
1220} 1182}
@@ -1309,7 +1271,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1309 } 1271 }
1310 1272
1311 if (i915_gem_object_is_inactive(obj_priv)) 1273 if (i915_gem_object_is_inactive(obj_priv))
1312 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1274 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1313 1275
1314 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + 1276 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1315 page_offset; 1277 page_offset;
@@ -1512,33 +1474,27 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1512 if (!(dev->driver->driver_features & DRIVER_GEM)) 1474 if (!(dev->driver->driver_features & DRIVER_GEM))
1513 return -ENODEV; 1475 return -ENODEV;
1514 1476
1515 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1516 if (obj == NULL)
1517 return -ENOENT;
1518
1519 ret = i915_mutex_lock_interruptible(dev); 1477 ret = i915_mutex_lock_interruptible(dev);
1520 if (ret) { 1478 if (ret)
1521 drm_gem_object_unreference_unlocked(obj);
1522 return ret; 1479 return ret;
1523 }
1524 1480
1481 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1482 if (obj == NULL) {
1483 ret = -ENOENT;
1484 goto unlock;
1485 }
1525 obj_priv = to_intel_bo(obj); 1486 obj_priv = to_intel_bo(obj);
1526 1487
1527 if (obj_priv->madv != I915_MADV_WILLNEED) { 1488 if (obj_priv->madv != I915_MADV_WILLNEED) {
1528 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1489 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1529 drm_gem_object_unreference(obj); 1490 ret = -EINVAL;
1530 mutex_unlock(&dev->struct_mutex); 1491 goto out;
1531 return -EINVAL;
1532 } 1492 }
1533 1493
1534
1535 if (!obj_priv->mmap_offset) { 1494 if (!obj_priv->mmap_offset) {
1536 ret = i915_gem_create_mmap_offset(obj); 1495 ret = i915_gem_create_mmap_offset(obj);
1537 if (ret) { 1496 if (ret)
1538 drm_gem_object_unreference(obj); 1497 goto out;
1539 mutex_unlock(&dev->struct_mutex);
1540 return ret;
1541 }
1542 } 1498 }
1543 1499
1544 args->offset = obj_priv->mmap_offset; 1500 args->offset = obj_priv->mmap_offset;
@@ -1549,17 +1505,15 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1549 */ 1505 */
1550 if (!obj_priv->agp_mem) { 1506 if (!obj_priv->agp_mem) {
1551 ret = i915_gem_object_bind_to_gtt(obj, 0); 1507 ret = i915_gem_object_bind_to_gtt(obj, 0);
1552 if (ret) { 1508 if (ret)
1553 drm_gem_object_unreference(obj); 1509 goto out;
1554 mutex_unlock(&dev->struct_mutex);
1555 return ret;
1556 }
1557 } 1510 }
1558 1511
1512out:
1559 drm_gem_object_unreference(obj); 1513 drm_gem_object_unreference(obj);
1514unlock:
1560 mutex_unlock(&dev->struct_mutex); 1515 mutex_unlock(&dev->struct_mutex);
1561 1516 return ret;
1562 return 0;
1563} 1517}
1564 1518
1565static void 1519static void
@@ -1611,6 +1565,7 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj,
1611 struct intel_ring_buffer *ring) 1565 struct intel_ring_buffer *ring)
1612{ 1566{
1613 struct drm_device *dev = obj->dev; 1567 struct drm_device *dev = obj->dev;
1568 struct drm_i915_private *dev_priv = dev->dev_private;
1614 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1569 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1615 uint32_t seqno = i915_gem_next_request_seqno(dev, ring); 1570 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1616 1571
@@ -1624,7 +1579,8 @@ i915_gem_object_move_to_active(struct drm_gem_object *obj,
1624 } 1579 }
1625 1580
1626 /* Move from whatever list we were on to the tail of execution. */ 1581 /* Move from whatever list we were on to the tail of execution. */
1627 list_move_tail(&obj_priv->list, &ring->active_list); 1582 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1583 list_move_tail(&obj_priv->ring_list, &ring->active_list);
1628 obj_priv->last_rendering_seqno = seqno; 1584 obj_priv->last_rendering_seqno = seqno;
1629} 1585}
1630 1586
@@ -1636,7 +1592,8 @@ i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1636 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1592 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1637 1593
1638 BUG_ON(!obj_priv->active); 1594 BUG_ON(!obj_priv->active);
1639 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list); 1595 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1596 list_del_init(&obj_priv->ring_list);
1640 obj_priv->last_rendering_seqno = 0; 1597 obj_priv->last_rendering_seqno = 0;
1641} 1598}
1642 1599
@@ -1675,9 +1632,10 @@ i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1675 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 1632 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1676 1633
1677 if (obj_priv->pin_count != 0) 1634 if (obj_priv->pin_count != 0)
1678 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list); 1635 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1679 else 1636 else
1680 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 1637 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1638 list_del_init(&obj_priv->ring_list);
1681 1639
1682 BUG_ON(!list_empty(&obj_priv->gpu_write_list)); 1640 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1683 1641
@@ -1699,12 +1657,11 @@ i915_gem_process_flushing_list(struct drm_device *dev,
1699 struct drm_i915_gem_object *obj_priv, *next; 1657 struct drm_i915_gem_object *obj_priv, *next;
1700 1658
1701 list_for_each_entry_safe(obj_priv, next, 1659 list_for_each_entry_safe(obj_priv, next,
1702 &dev_priv->mm.gpu_write_list, 1660 &ring->gpu_write_list,
1703 gpu_write_list) { 1661 gpu_write_list) {
1704 struct drm_gem_object *obj = &obj_priv->base; 1662 struct drm_gem_object *obj = &obj_priv->base;
1705 1663
1706 if (obj->write_domain & flush_domains && 1664 if (obj->write_domain & flush_domains) {
1707 obj_priv->ring == ring) {
1708 uint32_t old_write_domain = obj->write_domain; 1665 uint32_t old_write_domain = obj->write_domain;
1709 1666
1710 obj->write_domain = 0; 1667 obj->write_domain = 0;
@@ -1826,7 +1783,7 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1826 1783
1827 obj_priv = list_first_entry(&ring->active_list, 1784 obj_priv = list_first_entry(&ring->active_list,
1828 struct drm_i915_gem_object, 1785 struct drm_i915_gem_object,
1829 list); 1786 ring_list);
1830 1787
1831 obj_priv->base.write_domain = 0; 1788 obj_priv->base.write_domain = 0;
1832 list_del_init(&obj_priv->gpu_write_list); 1789 list_del_init(&obj_priv->gpu_write_list);
@@ -1841,8 +1798,8 @@ void i915_gem_reset(struct drm_device *dev)
1841 int i; 1798 int i;
1842 1799
1843 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); 1800 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1844 if (HAS_BSD(dev)) 1801 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1845 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); 1802 i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1846 1803
1847 /* Remove anything from the flushing lists. The GPU cache is likely 1804 /* Remove anything from the flushing lists. The GPU cache is likely
1848 * to be lost on reset along with the data, so simply move the 1805 * to be lost on reset along with the data, so simply move the
@@ -1851,7 +1808,7 @@ void i915_gem_reset(struct drm_device *dev)
1851 while (!list_empty(&dev_priv->mm.flushing_list)) { 1808 while (!list_empty(&dev_priv->mm.flushing_list)) {
1852 obj_priv = list_first_entry(&dev_priv->mm.flushing_list, 1809 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1853 struct drm_i915_gem_object, 1810 struct drm_i915_gem_object,
1854 list); 1811 mm_list);
1855 1812
1856 obj_priv->base.write_domain = 0; 1813 obj_priv->base.write_domain = 0;
1857 list_del_init(&obj_priv->gpu_write_list); 1814 list_del_init(&obj_priv->gpu_write_list);
@@ -1863,7 +1820,7 @@ void i915_gem_reset(struct drm_device *dev)
1863 */ 1820 */
1864 list_for_each_entry(obj_priv, 1821 list_for_each_entry(obj_priv,
1865 &dev_priv->mm.inactive_list, 1822 &dev_priv->mm.inactive_list,
1866 list) 1823 mm_list)
1867 { 1824 {
1868 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1825 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1869 } 1826 }
@@ -1923,7 +1880,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev,
1923 1880
1924 obj_priv = list_first_entry(&ring->active_list, 1881 obj_priv = list_first_entry(&ring->active_list,
1925 struct drm_i915_gem_object, 1882 struct drm_i915_gem_object,
1926 list); 1883 ring_list);
1927 1884
1928 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) 1885 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1929 break; 1886 break;
@@ -1959,13 +1916,13 @@ i915_gem_retire_requests(struct drm_device *dev)
1959 */ 1916 */
1960 list_for_each_entry_safe(obj_priv, tmp, 1917 list_for_each_entry_safe(obj_priv, tmp,
1961 &dev_priv->mm.deferred_free_list, 1918 &dev_priv->mm.deferred_free_list,
1962 list) 1919 mm_list)
1963 i915_gem_free_object_tail(&obj_priv->base); 1920 i915_gem_free_object_tail(&obj_priv->base);
1964 } 1921 }
1965 1922
1966 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); 1923 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1967 if (HAS_BSD(dev)) 1924 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1968 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); 1925 i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1969} 1926}
1970 1927
1971static void 1928static void
@@ -1988,8 +1945,8 @@ i915_gem_retire_work_handler(struct work_struct *work)
1988 1945
1989 if (!dev_priv->mm.suspended && 1946 if (!dev_priv->mm.suspended &&
1990 (!list_empty(&dev_priv->render_ring.request_list) || 1947 (!list_empty(&dev_priv->render_ring.request_list) ||
1991 (HAS_BSD(dev) && 1948 !list_empty(&dev_priv->bsd_ring.request_list) ||
1992 !list_empty(&dev_priv->bsd_ring.request_list)))) 1949 !list_empty(&dev_priv->blt_ring.request_list)))
1993 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); 1950 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1994 mutex_unlock(&dev->struct_mutex); 1951 mutex_unlock(&dev->struct_mutex);
1995} 1952}
@@ -2108,6 +2065,10 @@ i915_gem_flush(struct drm_device *dev,
2108 i915_gem_flush_ring(dev, file_priv, 2065 i915_gem_flush_ring(dev, file_priv,
2109 &dev_priv->bsd_ring, 2066 &dev_priv->bsd_ring,
2110 invalidate_domains, flush_domains); 2067 invalidate_domains, flush_domains);
2068 if (flush_rings & RING_BLT)
2069 i915_gem_flush_ring(dev, file_priv,
2070 &dev_priv->blt_ring,
2071 invalidate_domains, flush_domains);
2111 } 2072 }
2112} 2073}
2113 2074
@@ -2194,10 +2155,11 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2194 BUG_ON(obj_priv->pages_refcount); 2155 BUG_ON(obj_priv->pages_refcount);
2195 2156
2196 i915_gem_info_remove_gtt(dev_priv, obj->size); 2157 i915_gem_info_remove_gtt(dev_priv, obj->size);
2197 list_del_init(&obj_priv->list); 2158 list_del_init(&obj_priv->mm_list);
2198 2159
2199 drm_mm_put_block(obj_priv->gtt_space); 2160 drm_mm_put_block(obj_priv->gtt_space);
2200 obj_priv->gtt_space = NULL; 2161 obj_priv->gtt_space = NULL;
2162 obj_priv->gtt_offset = 0;
2201 2163
2202 if (i915_gem_object_is_purgeable(obj_priv)) 2164 if (i915_gem_object_is_purgeable(obj_priv))
2203 i915_gem_object_truncate(obj); 2165 i915_gem_object_truncate(obj);
@@ -2210,6 +2172,9 @@ i915_gem_object_unbind(struct drm_gem_object *obj)
2210static int i915_ring_idle(struct drm_device *dev, 2172static int i915_ring_idle(struct drm_device *dev,
2211 struct intel_ring_buffer *ring) 2173 struct intel_ring_buffer *ring)
2212{ 2174{
2175 if (list_empty(&ring->gpu_write_list))
2176 return 0;
2177
2213 i915_gem_flush_ring(dev, NULL, ring, 2178 i915_gem_flush_ring(dev, NULL, ring,
2214 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); 2179 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2215 return i915_wait_request(dev, 2180 return i915_wait_request(dev,
@@ -2226,8 +2191,8 @@ i915_gpu_idle(struct drm_device *dev)
2226 2191
2227 lists_empty = (list_empty(&dev_priv->mm.flushing_list) && 2192 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2228 list_empty(&dev_priv->render_ring.active_list) && 2193 list_empty(&dev_priv->render_ring.active_list) &&
2229 (!HAS_BSD(dev) || 2194 list_empty(&dev_priv->bsd_ring.active_list) &&
2230 list_empty(&dev_priv->bsd_ring.active_list))); 2195 list_empty(&dev_priv->blt_ring.active_list));
2231 if (lists_empty) 2196 if (lists_empty)
2232 return 0; 2197 return 0;
2233 2198
@@ -2236,11 +2201,13 @@ i915_gpu_idle(struct drm_device *dev)
2236 if (ret) 2201 if (ret)
2237 return ret; 2202 return ret;
2238 2203
2239 if (HAS_BSD(dev)) { 2204 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2240 ret = i915_ring_idle(dev, &dev_priv->bsd_ring); 2205 if (ret)
2241 if (ret) 2206 return ret;
2242 return ret; 2207
2243 } 2208 ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2209 if (ret)
2210 return ret;
2244 2211
2245 return 0; 2212 return 0;
2246} 2213}
@@ -2691,12 +2658,9 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2691 search_free: 2658 search_free:
2692 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2659 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2693 obj->size, alignment, 0); 2660 obj->size, alignment, 0);
2694 if (free_space != NULL) { 2661 if (free_space != NULL)
2695 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, 2662 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2696 alignment); 2663 alignment);
2697 if (obj_priv->gtt_space != NULL)
2698 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2699 }
2700 if (obj_priv->gtt_space == NULL) { 2664 if (obj_priv->gtt_space == NULL) {
2701 /* If the gtt is empty and we're still having trouble 2665 /* If the gtt is empty and we're still having trouble
2702 * fitting our object in, we're out of memory. 2666 * fitting our object in, we're out of memory.
@@ -2739,7 +2703,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2739 obj_priv->agp_mem = drm_agp_bind_pages(dev, 2703 obj_priv->agp_mem = drm_agp_bind_pages(dev,
2740 obj_priv->pages, 2704 obj_priv->pages,
2741 obj->size >> PAGE_SHIFT, 2705 obj->size >> PAGE_SHIFT,
2742 obj_priv->gtt_offset, 2706 obj_priv->gtt_space->start,
2743 obj_priv->agp_type); 2707 obj_priv->agp_type);
2744 if (obj_priv->agp_mem == NULL) { 2708 if (obj_priv->agp_mem == NULL) {
2745 i915_gem_object_put_pages(obj); 2709 i915_gem_object_put_pages(obj);
@@ -2754,7 +2718,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2754 } 2718 }
2755 2719
2756 /* keep track of bounds object by adding it to the inactive list */ 2720 /* keep track of bounds object by adding it to the inactive list */
2757 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list); 2721 list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2758 i915_gem_info_add_gtt(dev_priv, obj->size); 2722 i915_gem_info_add_gtt(dev_priv, obj->size);
2759 2723
2760 /* Assert that the object is not currently in any GPU domain. As it 2724 /* Assert that the object is not currently in any GPU domain. As it
@@ -2764,6 +2728,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2764 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); 2728 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2765 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); 2729 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2766 2730
2731 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2767 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset); 2732 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2768 2733
2769 return 0; 2734 return 0;
@@ -3115,7 +3080,8 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3115 * drm_agp_chipset_flush 3080 * drm_agp_chipset_flush
3116 */ 3081 */
3117static void 3082static void
3118i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj) 3083i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3084 struct intel_ring_buffer *ring)
3119{ 3085{
3120 struct drm_device *dev = obj->dev; 3086 struct drm_device *dev = obj->dev;
3121 struct drm_i915_private *dev_priv = dev->dev_private; 3087 struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3124,9 +3090,6 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3124 uint32_t flush_domains = 0; 3090 uint32_t flush_domains = 0;
3125 uint32_t old_read_domains; 3091 uint32_t old_read_domains;
3126 3092
3127 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3128 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3129
3130 intel_mark_busy(dev, obj); 3093 intel_mark_busy(dev, obj);
3131 3094
3132 /* 3095 /*
@@ -3172,8 +3135,10 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3172 3135
3173 dev->invalidate_domains |= invalidate_domains; 3136 dev->invalidate_domains |= invalidate_domains;
3174 dev->flush_domains |= flush_domains; 3137 dev->flush_domains |= flush_domains;
3175 if (obj_priv->ring) 3138 if (flush_domains & I915_GEM_GPU_DOMAINS)
3176 dev_priv->mm.flush_rings |= obj_priv->ring->id; 3139 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3140 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3141 dev_priv->mm.flush_rings |= ring->id;
3177 3142
3178 trace_i915_gem_object_change_domain(obj, 3143 trace_i915_gem_object_change_domain(obj,
3179 old_read_domains, 3144 old_read_domains,
@@ -3289,68 +3254,42 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3289 * Pin an object to the GTT and evaluate the relocations landing in it. 3254 * Pin an object to the GTT and evaluate the relocations landing in it.
3290 */ 3255 */
3291static int 3256static int
3292i915_gem_object_pin_and_relocate(struct drm_gem_object *obj, 3257i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3293 struct drm_file *file_priv, 3258 struct drm_file *file_priv,
3294 struct drm_i915_gem_exec_object2 *entry, 3259 struct drm_i915_gem_exec_object2 *entry)
3295 struct drm_i915_gem_relocation_entry *relocs)
3296{ 3260{
3297 struct drm_device *dev = obj->dev; 3261 struct drm_device *dev = obj->base.dev;
3298 drm_i915_private_t *dev_priv = dev->dev_private; 3262 drm_i915_private_t *dev_priv = dev->dev_private;
3299 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); 3263 struct drm_i915_gem_relocation_entry __user *user_relocs;
3300 int i, ret; 3264 struct drm_gem_object *target_obj = NULL;
3301 void __iomem *reloc_page; 3265 uint32_t target_handle = 0;
3302 bool need_fence; 3266 int i, ret = 0;
3303
3304 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3305 obj_priv->tiling_mode != I915_TILING_NONE;
3306
3307 /* Check fence reg constraints and rebind if necessary */
3308 if (need_fence &&
3309 !i915_gem_object_fence_offset_ok(obj,
3310 obj_priv->tiling_mode)) {
3311 ret = i915_gem_object_unbind(obj);
3312 if (ret)
3313 return ret;
3314 }
3315 3267
3316 /* Choose the GTT offset for our buffer and put it there. */ 3268 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3317 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment); 3269 for (i = 0; i < entry->relocation_count; i++) {
3318 if (ret) 3270 struct drm_i915_gem_relocation_entry reloc;
3319 return ret; 3271 uint32_t target_offset;
3320 3272
3321 /* 3273 if (__copy_from_user_inatomic(&reloc,
3322 * Pre-965 chips need a fence register set up in order to 3274 user_relocs+i,
3323 * properly handle blits to/from tiled surfaces. 3275 sizeof(reloc))) {
3324 */ 3276 ret = -EFAULT;
3325 if (need_fence) { 3277 break;
3326 ret = i915_gem_object_get_fence_reg(obj, true);
3327 if (ret != 0) {
3328 i915_gem_object_unpin(obj);
3329 return ret;
3330 } 3278 }
3331 3279
3332 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true; 3280 if (reloc.target_handle != target_handle) {
3333 } 3281 drm_gem_object_unreference(target_obj);
3334 3282
3335 entry->offset = obj_priv->gtt_offset; 3283 target_obj = drm_gem_object_lookup(dev, file_priv,
3284 reloc.target_handle);
3285 if (target_obj == NULL) {
3286 ret = -ENOENT;
3287 break;
3288 }
3336 3289
3337 /* Apply the relocations, using the GTT aperture to avoid cache 3290 target_handle = reloc.target_handle;
3338 * flushing requirements.
3339 */
3340 for (i = 0; i < entry->relocation_count; i++) {
3341 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3342 struct drm_gem_object *target_obj;
3343 struct drm_i915_gem_object *target_obj_priv;
3344 uint32_t reloc_val, reloc_offset;
3345 uint32_t __iomem *reloc_entry;
3346
3347 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3348 reloc->target_handle);
3349 if (target_obj == NULL) {
3350 i915_gem_object_unpin(obj);
3351 return -ENOENT;
3352 } 3291 }
3353 target_obj_priv = to_intel_bo(target_obj); 3292 target_offset = to_intel_bo(target_obj)->gtt_offset;
3354 3293
3355#if WATCH_RELOC 3294#if WATCH_RELOC
3356 DRM_INFO("%s: obj %p offset %08x target %d " 3295 DRM_INFO("%s: obj %p offset %08x target %d "
@@ -3358,136 +3297,202 @@ i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3358 "presumed %08x delta %08x\n", 3297 "presumed %08x delta %08x\n",
3359 __func__, 3298 __func__,
3360 obj, 3299 obj,
3361 (int) reloc->offset, 3300 (int) reloc.offset,
3362 (int) reloc->target_handle, 3301 (int) reloc.target_handle,
3363 (int) reloc->read_domains, 3302 (int) reloc.read_domains,
3364 (int) reloc->write_domain, 3303 (int) reloc.write_domain,
3365 (int) target_obj_priv->gtt_offset, 3304 (int) target_offset,
3366 (int) reloc->presumed_offset, 3305 (int) reloc.presumed_offset,
3367 reloc->delta); 3306 reloc.delta);
3368#endif 3307#endif
3369 3308
3370 /* The target buffer should have appeared before us in the 3309 /* The target buffer should have appeared before us in the
3371 * exec_object list, so it should have a GTT space bound by now. 3310 * exec_object list, so it should have a GTT space bound by now.
3372 */ 3311 */
3373 if (target_obj_priv->gtt_space == NULL) { 3312 if (target_offset == 0) {
3374 DRM_ERROR("No GTT space found for object %d\n", 3313 DRM_ERROR("No GTT space found for object %d\n",
3375 reloc->target_handle); 3314 reloc.target_handle);
3376 drm_gem_object_unreference(target_obj); 3315 ret = -EINVAL;
3377 i915_gem_object_unpin(obj); 3316 break;
3378 return -EINVAL;
3379 } 3317 }
3380 3318
3381 /* Validate that the target is in a valid r/w GPU domain */ 3319 /* Validate that the target is in a valid r/w GPU domain */
3382 if (reloc->write_domain & (reloc->write_domain - 1)) { 3320 if (reloc.write_domain & (reloc.write_domain - 1)) {
3383 DRM_ERROR("reloc with multiple write domains: " 3321 DRM_ERROR("reloc with multiple write domains: "
3384 "obj %p target %d offset %d " 3322 "obj %p target %d offset %d "
3385 "read %08x write %08x", 3323 "read %08x write %08x",
3386 obj, reloc->target_handle, 3324 obj, reloc.target_handle,
3387 (int) reloc->offset, 3325 (int) reloc.offset,
3388 reloc->read_domains, 3326 reloc.read_domains,
3389 reloc->write_domain); 3327 reloc.write_domain);
3390 drm_gem_object_unreference(target_obj); 3328 ret = -EINVAL;
3391 i915_gem_object_unpin(obj); 3329 break;
3392 return -EINVAL;
3393 } 3330 }
3394 if (reloc->write_domain & I915_GEM_DOMAIN_CPU || 3331 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3395 reloc->read_domains & I915_GEM_DOMAIN_CPU) { 3332 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3396 DRM_ERROR("reloc with read/write CPU domains: " 3333 DRM_ERROR("reloc with read/write CPU domains: "
3397 "obj %p target %d offset %d " 3334 "obj %p target %d offset %d "
3398 "read %08x write %08x", 3335 "read %08x write %08x",
3399 obj, reloc->target_handle, 3336 obj, reloc.target_handle,
3400 (int) reloc->offset, 3337 (int) reloc.offset,
3401 reloc->read_domains, 3338 reloc.read_domains,
3402 reloc->write_domain); 3339 reloc.write_domain);
3403 drm_gem_object_unreference(target_obj); 3340 ret = -EINVAL;
3404 i915_gem_object_unpin(obj); 3341 break;
3405 return -EINVAL;
3406 } 3342 }
3407 if (reloc->write_domain && target_obj->pending_write_domain && 3343 if (reloc.write_domain && target_obj->pending_write_domain &&
3408 reloc->write_domain != target_obj->pending_write_domain) { 3344 reloc.write_domain != target_obj->pending_write_domain) {
3409 DRM_ERROR("Write domain conflict: " 3345 DRM_ERROR("Write domain conflict: "
3410 "obj %p target %d offset %d " 3346 "obj %p target %d offset %d "
3411 "new %08x old %08x\n", 3347 "new %08x old %08x\n",
3412 obj, reloc->target_handle, 3348 obj, reloc.target_handle,
3413 (int) reloc->offset, 3349 (int) reloc.offset,
3414 reloc->write_domain, 3350 reloc.write_domain,
3415 target_obj->pending_write_domain); 3351 target_obj->pending_write_domain);
3416 drm_gem_object_unreference(target_obj); 3352 ret = -EINVAL;
3417 i915_gem_object_unpin(obj); 3353 break;
3418 return -EINVAL;
3419 } 3354 }
3420 3355
3421 target_obj->pending_read_domains |= reloc->read_domains; 3356 target_obj->pending_read_domains |= reloc.read_domains;
3422 target_obj->pending_write_domain |= reloc->write_domain; 3357 target_obj->pending_write_domain |= reloc.write_domain;
3423 3358
3424 /* If the relocation already has the right value in it, no 3359 /* If the relocation already has the right value in it, no
3425 * more work needs to be done. 3360 * more work needs to be done.
3426 */ 3361 */
3427 if (target_obj_priv->gtt_offset == reloc->presumed_offset) { 3362 if (target_offset == reloc.presumed_offset)
3428 drm_gem_object_unreference(target_obj);
3429 continue; 3363 continue;
3430 }
3431 3364
3432 /* Check that the relocation address is valid... */ 3365 /* Check that the relocation address is valid... */
3433 if (reloc->offset > obj->size - 4) { 3366 if (reloc.offset > obj->base.size - 4) {
3434 DRM_ERROR("Relocation beyond object bounds: " 3367 DRM_ERROR("Relocation beyond object bounds: "
3435 "obj %p target %d offset %d size %d.\n", 3368 "obj %p target %d offset %d size %d.\n",
3436 obj, reloc->target_handle, 3369 obj, reloc.target_handle,
3437 (int) reloc->offset, (int) obj->size); 3370 (int) reloc.offset, (int) obj->base.size);
3438 drm_gem_object_unreference(target_obj); 3371 ret = -EINVAL;
3439 i915_gem_object_unpin(obj); 3372 break;
3440 return -EINVAL;
3441 } 3373 }
3442 if (reloc->offset & 3) { 3374 if (reloc.offset & 3) {
3443 DRM_ERROR("Relocation not 4-byte aligned: " 3375 DRM_ERROR("Relocation not 4-byte aligned: "
3444 "obj %p target %d offset %d.\n", 3376 "obj %p target %d offset %d.\n",
3445 obj, reloc->target_handle, 3377 obj, reloc.target_handle,
3446 (int) reloc->offset); 3378 (int) reloc.offset);
3447 drm_gem_object_unreference(target_obj); 3379 ret = -EINVAL;
3448 i915_gem_object_unpin(obj); 3380 break;
3449 return -EINVAL;
3450 } 3381 }
3451 3382
3452 /* and points to somewhere within the target object. */ 3383 /* and points to somewhere within the target object. */
3453 if (reloc->delta >= target_obj->size) { 3384 if (reloc.delta >= target_obj->size) {
3454 DRM_ERROR("Relocation beyond target object bounds: " 3385 DRM_ERROR("Relocation beyond target object bounds: "
3455 "obj %p target %d delta %d size %d.\n", 3386 "obj %p target %d delta %d size %d.\n",
3456 obj, reloc->target_handle, 3387 obj, reloc.target_handle,
3457 (int) reloc->delta, (int) target_obj->size); 3388 (int) reloc.delta, (int) target_obj->size);
3458 drm_gem_object_unreference(target_obj); 3389 ret = -EINVAL;
3459 i915_gem_object_unpin(obj); 3390 break;
3460 return -EINVAL;
3461 } 3391 }
3462 3392
3463 ret = i915_gem_object_set_to_gtt_domain(obj, 1); 3393 reloc.delta += target_offset;
3464 if (ret != 0) { 3394 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3465 drm_gem_object_unreference(target_obj); 3395 uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3466 i915_gem_object_unpin(obj); 3396 char *vaddr;
3467 return ret; 3397
3398 vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT], KM_USER0);
3399 *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3400 kunmap_atomic(vaddr, KM_USER0);
3401 } else {
3402 uint32_t __iomem *reloc_entry;
3403 void __iomem *reloc_page;
3404
3405 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3406 if (ret)
3407 break;
3408
3409 /* Map the page containing the relocation we're going to perform. */
3410 reloc.offset += obj->gtt_offset;
3411 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3412 reloc.offset & PAGE_MASK,
3413 KM_USER0);
3414 reloc_entry = (uint32_t __iomem *)
3415 (reloc_page + (reloc.offset & ~PAGE_MASK));
3416 iowrite32(reloc.delta, reloc_entry);
3417 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3468 } 3418 }
3469 3419
3470 /* Map the page containing the relocation we're going to 3420 /* and update the user's relocation entry */
3471 * perform. 3421 reloc.presumed_offset = target_offset;
3472 */ 3422 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3473 reloc_offset = obj_priv->gtt_offset + reloc->offset; 3423 &reloc.presumed_offset,
3474 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping, 3424 sizeof(reloc.presumed_offset))) {
3475 (reloc_offset & 3425 ret = -EFAULT;
3476 ~(PAGE_SIZE - 1)), 3426 break;
3477 KM_USER0); 3427 }
3478 reloc_entry = (uint32_t __iomem *)(reloc_page + 3428 }
3479 (reloc_offset & (PAGE_SIZE - 1))); 3429
3480 reloc_val = target_obj_priv->gtt_offset + reloc->delta; 3430 drm_gem_object_unreference(target_obj);
3481 3431 return ret;
3482 writel(reloc_val, reloc_entry); 3432}
3483 io_mapping_unmap_atomic(reloc_page, KM_USER0); 3433
3484 3434static int
3485 /* The updated presumed offset for this entry will be 3435i915_gem_execbuffer_pin(struct drm_device *dev,
3486 * copied back out to the user. 3436 struct drm_file *file,
3487 */ 3437 struct drm_gem_object **object_list,
3488 reloc->presumed_offset = target_obj_priv->gtt_offset; 3438 struct drm_i915_gem_exec_object2 *exec_list,
3439 int count)
3440{
3441 struct drm_i915_private *dev_priv = dev->dev_private;
3442 int ret, i, retry;
3443
3444 /* attempt to pin all of the buffers into the GTT */
3445 for (retry = 0; retry < 2; retry++) {
3446 ret = 0;
3447 for (i = 0; i < count; i++) {
3448 struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3449 struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3450 bool need_fence =
3451 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3452 obj->tiling_mode != I915_TILING_NONE;
3453
3454 /* Check fence reg constraints and rebind if necessary */
3455 if (need_fence &&
3456 !i915_gem_object_fence_offset_ok(&obj->base,
3457 obj->tiling_mode)) {
3458 ret = i915_gem_object_unbind(&obj->base);
3459 if (ret)
3460 break;
3461 }
3462
3463 ret = i915_gem_object_pin(&obj->base, entry->alignment);
3464 if (ret)
3465 break;
3466
3467 /*
3468 * Pre-965 chips need a fence register set up in order
3469 * to properly handle blits to/from tiled surfaces.
3470 */
3471 if (need_fence) {
3472 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3473 if (ret) {
3474 i915_gem_object_unpin(&obj->base);
3475 break;
3476 }
3489 3477
3490 drm_gem_object_unreference(target_obj); 3478 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3479 }
3480
3481 entry->offset = obj->gtt_offset;
3482 }
3483
3484 while (i--)
3485 i915_gem_object_unpin(object_list[i]);
3486
3487 if (ret == 0)
3488 break;
3489
3490 if (ret != -ENOSPC || retry)
3491 return ret;
3492
3493 ret = i915_gem_evict_everything(dev);
3494 if (ret)
3495 return ret;
3491 } 3496 }
3492 3497
3493 return 0; 3498 return 0;
@@ -3551,86 +3556,8 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3551} 3556}
3552 3557
3553static int 3558static int
3554i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list, 3559i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3555 uint32_t buffer_count, 3560 uint64_t exec_offset)
3556 struct drm_i915_gem_relocation_entry **relocs)
3557{
3558 uint32_t reloc_count = 0, reloc_index = 0, i;
3559 int ret;
3560
3561 *relocs = NULL;
3562 for (i = 0; i < buffer_count; i++) {
3563 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3564 return -EINVAL;
3565 reloc_count += exec_list[i].relocation_count;
3566 }
3567
3568 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3569 if (*relocs == NULL) {
3570 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3571 return -ENOMEM;
3572 }
3573
3574 for (i = 0; i < buffer_count; i++) {
3575 struct drm_i915_gem_relocation_entry __user *user_relocs;
3576
3577 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3578
3579 ret = copy_from_user(&(*relocs)[reloc_index],
3580 user_relocs,
3581 exec_list[i].relocation_count *
3582 sizeof(**relocs));
3583 if (ret != 0) {
3584 drm_free_large(*relocs);
3585 *relocs = NULL;
3586 return -EFAULT;
3587 }
3588
3589 reloc_index += exec_list[i].relocation_count;
3590 }
3591
3592 return 0;
3593}
3594
3595static int
3596i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3597 uint32_t buffer_count,
3598 struct drm_i915_gem_relocation_entry *relocs)
3599{
3600 uint32_t reloc_count = 0, i;
3601 int ret = 0;
3602
3603 if (relocs == NULL)
3604 return 0;
3605
3606 for (i = 0; i < buffer_count; i++) {
3607 struct drm_i915_gem_relocation_entry __user *user_relocs;
3608 int unwritten;
3609
3610 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3611
3612 unwritten = copy_to_user(user_relocs,
3613 &relocs[reloc_count],
3614 exec_list[i].relocation_count *
3615 sizeof(*relocs));
3616
3617 if (unwritten) {
3618 ret = -EFAULT;
3619 goto err;
3620 }
3621
3622 reloc_count += exec_list[i].relocation_count;
3623 }
3624
3625err:
3626 drm_free_large(relocs);
3627
3628 return ret;
3629}
3630
3631static int
3632i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3633 uint64_t exec_offset)
3634{ 3561{
3635 uint32_t exec_start, exec_len; 3562 uint32_t exec_start, exec_len;
3636 3563
@@ -3647,43 +3574,32 @@ i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3647} 3574}
3648 3575
3649static int 3576static int
3650i915_gem_wait_for_pending_flip(struct drm_device *dev, 3577validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3651 struct drm_gem_object **object_list, 3578 int count)
3652 int count)
3653{ 3579{
3654 drm_i915_private_t *dev_priv = dev->dev_private; 3580 int i;
3655 struct drm_i915_gem_object *obj_priv;
3656 DEFINE_WAIT(wait);
3657 int i, ret = 0;
3658 3581
3659 for (;;) { 3582 for (i = 0; i < count; i++) {
3660 prepare_to_wait(&dev_priv->pending_flip_queue, 3583 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3661 &wait, TASK_INTERRUPTIBLE); 3584 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3662 for (i = 0; i < count; i++) {
3663 obj_priv = to_intel_bo(object_list[i]);
3664 if (atomic_read(&obj_priv->pending_flip) > 0)
3665 break;
3666 }
3667 if (i == count)
3668 break;
3669 3585
3670 if (!signal_pending(current)) { 3586 if (!access_ok(VERIFY_READ, ptr, length))
3671 mutex_unlock(&dev->struct_mutex); 3587 return -EFAULT;
3672 schedule(); 3588
3673 mutex_lock(&dev->struct_mutex); 3589 /* we may also need to update the presumed offsets */
3674 continue; 3590 if (!access_ok(VERIFY_WRITE, ptr, length))
3675 } 3591 return -EFAULT;
3676 ret = -ERESTARTSYS; 3592
3677 break; 3593 if (fault_in_pages_readable(ptr, length))
3594 return -EFAULT;
3678 } 3595 }
3679 finish_wait(&dev_priv->pending_flip_queue, &wait);
3680 3596
3681 return ret; 3597 return 0;
3682} 3598}
3683 3599
3684static int 3600static int
3685i915_gem_do_execbuffer(struct drm_device *dev, void *data, 3601i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3686 struct drm_file *file_priv, 3602 struct drm_file *file,
3687 struct drm_i915_gem_execbuffer2 *args, 3603 struct drm_i915_gem_execbuffer2 *args,
3688 struct drm_i915_gem_exec_object2 *exec_list) 3604 struct drm_i915_gem_exec_object2 *exec_list)
3689{ 3605{
@@ -3692,12 +3608,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3692 struct drm_gem_object *batch_obj; 3608 struct drm_gem_object *batch_obj;
3693 struct drm_i915_gem_object *obj_priv; 3609 struct drm_i915_gem_object *obj_priv;
3694 struct drm_clip_rect *cliprects = NULL; 3610 struct drm_clip_rect *cliprects = NULL;
3695 struct drm_i915_gem_relocation_entry *relocs = NULL;
3696 struct drm_i915_gem_request *request = NULL; 3611 struct drm_i915_gem_request *request = NULL;
3697 int ret, ret2, i, pinned = 0; 3612 int ret, i, flips;
3698 uint64_t exec_offset; 3613 uint64_t exec_offset;
3699 uint32_t reloc_index;
3700 int pin_tries, flips;
3701 3614
3702 struct intel_ring_buffer *ring = NULL; 3615 struct intel_ring_buffer *ring = NULL;
3703 3616
@@ -3705,18 +3618,37 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3705 if (ret) 3618 if (ret)
3706 return ret; 3619 return ret;
3707 3620
3621 ret = validate_exec_list(exec_list, args->buffer_count);
3622 if (ret)
3623 return ret;
3624
3708#if WATCH_EXEC 3625#if WATCH_EXEC
3709 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", 3626 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3710 (int) args->buffers_ptr, args->buffer_count, args->batch_len); 3627 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3711#endif 3628#endif
3712 if (args->flags & I915_EXEC_BSD) { 3629 switch (args->flags & I915_EXEC_RING_MASK) {
3630 case I915_EXEC_DEFAULT:
3631 case I915_EXEC_RENDER:
3632 ring = &dev_priv->render_ring;
3633 break;
3634 case I915_EXEC_BSD:
3713 if (!HAS_BSD(dev)) { 3635 if (!HAS_BSD(dev)) {
3714 DRM_ERROR("execbuf with wrong flag\n"); 3636 DRM_ERROR("execbuf with invalid ring (BSD)\n");
3715 return -EINVAL; 3637 return -EINVAL;
3716 } 3638 }
3717 ring = &dev_priv->bsd_ring; 3639 ring = &dev_priv->bsd_ring;
3718 } else { 3640 break;
3719 ring = &dev_priv->render_ring; 3641 case I915_EXEC_BLT:
3642 if (!HAS_BLT(dev)) {
3643 DRM_ERROR("execbuf with invalid ring (BLT)\n");
3644 return -EINVAL;
3645 }
3646 ring = &dev_priv->blt_ring;
3647 break;
3648 default:
3649 DRM_ERROR("execbuf with unknown ring: %d\n",
3650 (int)(args->flags & I915_EXEC_RING_MASK));
3651 return -EINVAL;
3720 } 3652 }
3721 3653
3722 if (args->buffer_count < 1) { 3654 if (args->buffer_count < 1) {
@@ -3757,11 +3689,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3757 goto pre_mutex_err; 3689 goto pre_mutex_err;
3758 } 3690 }
3759 3691
3760 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3761 &relocs);
3762 if (ret != 0)
3763 goto pre_mutex_err;
3764
3765 ret = i915_mutex_lock_interruptible(dev); 3692 ret = i915_mutex_lock_interruptible(dev);
3766 if (ret) 3693 if (ret)
3767 goto pre_mutex_err; 3694 goto pre_mutex_err;
@@ -3773,9 +3700,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3773 } 3700 }
3774 3701
3775 /* Look up object handles */ 3702 /* Look up object handles */
3776 flips = 0;
3777 for (i = 0; i < args->buffer_count; i++) { 3703 for (i = 0; i < args->buffer_count; i++) {
3778 object_list[i] = drm_gem_object_lookup(dev, file_priv, 3704 object_list[i] = drm_gem_object_lookup(dev, file,
3779 exec_list[i].handle); 3705 exec_list[i].handle);
3780 if (object_list[i] == NULL) { 3706 if (object_list[i] == NULL) {
3781 DRM_ERROR("Invalid object handle %d at index %d\n", 3707 DRM_ERROR("Invalid object handle %d at index %d\n",
@@ -3796,76 +3722,22 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3796 goto err; 3722 goto err;
3797 } 3723 }
3798 obj_priv->in_execbuffer = true; 3724 obj_priv->in_execbuffer = true;
3799 flips += atomic_read(&obj_priv->pending_flip);
3800 } 3725 }
3801 3726
3802 if (flips > 0) { 3727 /* Move the objects en-masse into the GTT, evicting if necessary. */
3803 ret = i915_gem_wait_for_pending_flip(dev, object_list, 3728 ret = i915_gem_execbuffer_pin(dev, file,
3804 args->buffer_count); 3729 object_list, exec_list,
3805 if (ret) 3730 args->buffer_count);
3806 goto err; 3731 if (ret)
3807 } 3732 goto err;
3808
3809 /* Pin and relocate */
3810 for (pin_tries = 0; ; pin_tries++) {
3811 ret = 0;
3812 reloc_index = 0;
3813
3814 for (i = 0; i < args->buffer_count; i++) {
3815 object_list[i]->pending_read_domains = 0;
3816 object_list[i]->pending_write_domain = 0;
3817 ret = i915_gem_object_pin_and_relocate(object_list[i],
3818 file_priv,
3819 &exec_list[i],
3820 &relocs[reloc_index]);
3821 if (ret)
3822 break;
3823 pinned = i + 1;
3824 reloc_index += exec_list[i].relocation_count;
3825 }
3826 /* success */
3827 if (ret == 0)
3828 break;
3829
3830 /* error other than GTT full, or we've already tried again */
3831 if (ret != -ENOSPC || pin_tries >= 1) {
3832 if (ret != -ERESTARTSYS) {
3833 unsigned long long total_size = 0;
3834 int num_fences = 0;
3835 for (i = 0; i < args->buffer_count; i++) {
3836 obj_priv = to_intel_bo(object_list[i]);
3837
3838 total_size += object_list[i]->size;
3839 num_fences +=
3840 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3841 obj_priv->tiling_mode != I915_TILING_NONE;
3842 }
3843 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3844 pinned+1, args->buffer_count,
3845 total_size, num_fences,
3846 ret);
3847 DRM_ERROR("%u objects [%u pinned, %u GTT], "
3848 "%zu object bytes [%zu pinned], "
3849 "%zu /%zu gtt bytes\n",
3850 dev_priv->mm.object_count,
3851 dev_priv->mm.pin_count,
3852 dev_priv->mm.gtt_count,
3853 dev_priv->mm.object_memory,
3854 dev_priv->mm.pin_memory,
3855 dev_priv->mm.gtt_memory,
3856 dev_priv->mm.gtt_total);
3857 }
3858 goto err;
3859 }
3860
3861 /* unpin all of our buffers */
3862 for (i = 0; i < pinned; i++)
3863 i915_gem_object_unpin(object_list[i]);
3864 pinned = 0;
3865 3733
3866 /* evict everyone we can from the aperture */ 3734 /* The objects are in their final locations, apply the relocations. */
3867 ret = i915_gem_evict_everything(dev); 3735 for (i = 0; i < args->buffer_count; i++) {
3868 if (ret && ret != -ENOSPC) 3736 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3737 obj->base.pending_read_domains = 0;
3738 obj->base.pending_write_domain = 0;
3739 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3740 if (ret)
3869 goto err; 3741 goto err;
3870 } 3742 }
3871 3743
@@ -3878,9 +3750,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3878 } 3750 }
3879 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND; 3751 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3880 3752
3881 /* Sanity check the batch buffer, prior to moving objects */ 3753 /* Sanity check the batch buffer */
3882 exec_offset = exec_list[args->buffer_count - 1].offset; 3754 exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3883 ret = i915_gem_check_execbuffer (args, exec_offset); 3755 ret = i915_gem_check_execbuffer(args, exec_offset);
3884 if (ret != 0) { 3756 if (ret != 0) {
3885 DRM_ERROR("execbuf with invalid offset/length\n"); 3757 DRM_ERROR("execbuf with invalid offset/length\n");
3886 goto err; 3758 goto err;
@@ -3898,7 +3770,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3898 struct drm_gem_object *obj = object_list[i]; 3770 struct drm_gem_object *obj = object_list[i];
3899 3771
3900 /* Compute new gpu domains and update invalidate/flush */ 3772 /* Compute new gpu domains and update invalidate/flush */
3901 i915_gem_object_set_to_gpu_domain(obj); 3773 i915_gem_object_set_to_gpu_domain(obj, ring);
3902 } 3774 }
3903 3775
3904 if (dev->invalidate_domains | dev->flush_domains) { 3776 if (dev->invalidate_domains | dev->flush_domains) {
@@ -3908,7 +3780,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3908 dev->invalidate_domains, 3780 dev->invalidate_domains,
3909 dev->flush_domains); 3781 dev->flush_domains);
3910#endif 3782#endif
3911 i915_gem_flush(dev, file_priv, 3783 i915_gem_flush(dev, file,
3912 dev->invalidate_domains, 3784 dev->invalidate_domains,
3913 dev->flush_domains, 3785 dev->flush_domains,
3914 dev_priv->mm.flush_rings); 3786 dev_priv->mm.flush_rings);
@@ -3916,14 +3788,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3916 3788
3917 for (i = 0; i < args->buffer_count; i++) { 3789 for (i = 0; i < args->buffer_count; i++) {
3918 struct drm_gem_object *obj = object_list[i]; 3790 struct drm_gem_object *obj = object_list[i];
3919 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3920 uint32_t old_write_domain = obj->write_domain; 3791 uint32_t old_write_domain = obj->write_domain;
3921
3922 obj->write_domain = obj->pending_write_domain; 3792 obj->write_domain = obj->pending_write_domain;
3923 if (obj->write_domain)
3924 list_move_tail(&obj_priv->gpu_write_list,
3925 &dev_priv->mm.gpu_write_list);
3926
3927 trace_i915_gem_object_change_domain(obj, 3793 trace_i915_gem_object_change_domain(obj,
3928 obj->read_domains, 3794 obj->read_domains,
3929 old_write_domain); 3795 old_write_domain);
@@ -3943,9 +3809,38 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3943 ~0); 3809 ~0);
3944#endif 3810#endif
3945 3811
3812 /* Check for any pending flips. As we only maintain a flip queue depth
3813 * of 1, we can simply insert a WAIT for the next display flip prior
3814 * to executing the batch and avoid stalling the CPU.
3815 */
3816 flips = 0;
3817 for (i = 0; i < args->buffer_count; i++) {
3818 if (object_list[i]->write_domain)
3819 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3820 }
3821 if (flips) {
3822 int plane, flip_mask;
3823
3824 for (plane = 0; flips >> plane; plane++) {
3825 if (((flips >> plane) & 1) == 0)
3826 continue;
3827
3828 if (plane)
3829 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3830 else
3831 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3832
3833 intel_ring_begin(dev, ring, 2);
3834 intel_ring_emit(dev, ring,
3835 MI_WAIT_FOR_EVENT | flip_mask);
3836 intel_ring_emit(dev, ring, MI_NOOP);
3837 intel_ring_advance(dev, ring);
3838 }
3839 }
3840
3946 /* Exec the batchbuffer */ 3841 /* Exec the batchbuffer */
3947 ret = ring->dispatch_gem_execbuffer(dev, ring, args, 3842 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3948 cliprects, exec_offset); 3843 cliprects, exec_offset);
3949 if (ret) { 3844 if (ret) {
3950 DRM_ERROR("dispatch failed %d\n", ret); 3845 DRM_ERROR("dispatch failed %d\n", ret);
3951 goto err; 3846 goto err;
@@ -3959,18 +3854,17 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3959 3854
3960 for (i = 0; i < args->buffer_count; i++) { 3855 for (i = 0; i < args->buffer_count; i++) {
3961 struct drm_gem_object *obj = object_list[i]; 3856 struct drm_gem_object *obj = object_list[i];
3962 obj_priv = to_intel_bo(obj);
3963 3857
3964 i915_gem_object_move_to_active(obj, ring); 3858 i915_gem_object_move_to_active(obj, ring);
3859 if (obj->write_domain)
3860 list_move_tail(&to_intel_bo(obj)->gpu_write_list,
3861 &ring->gpu_write_list);
3965 } 3862 }
3966 3863
3967 i915_add_request(dev, file_priv, request, ring); 3864 i915_add_request(dev, file, request, ring);
3968 request = NULL; 3865 request = NULL;
3969 3866
3970err: 3867err:
3971 for (i = 0; i < pinned; i++)
3972 i915_gem_object_unpin(object_list[i]);
3973
3974 for (i = 0; i < args->buffer_count; i++) { 3868 for (i = 0; i < args->buffer_count; i++) {
3975 if (object_list[i]) { 3869 if (object_list[i]) {
3976 obj_priv = to_intel_bo(object_list[i]); 3870 obj_priv = to_intel_bo(object_list[i]);
@@ -3982,20 +3876,6 @@ err:
3982 mutex_unlock(&dev->struct_mutex); 3876 mutex_unlock(&dev->struct_mutex);
3983 3877
3984pre_mutex_err: 3878pre_mutex_err:
3985 /* Copy the updated relocations out regardless of current error
3986 * state. Failure to update the relocs would mean that the next
3987 * time userland calls execbuf, it would do so with presumed offset
3988 * state that didn't match the actual object state.
3989 */
3990 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3991 relocs);
3992 if (ret2 != 0) {
3993 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3994
3995 if (ret == 0)
3996 ret = ret2;
3997 }
3998
3999 drm_free_large(object_list); 3879 drm_free_large(object_list);
4000 kfree(cliprects); 3880 kfree(cliprects);
4001 kfree(request); 3881 kfree(request);
@@ -4187,7 +4067,7 @@ i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4187 if (obj_priv->pin_count == 1) { 4067 if (obj_priv->pin_count == 1) {
4188 i915_gem_info_add_pin(dev_priv, obj->size); 4068 i915_gem_info_add_pin(dev_priv, obj->size);
4189 if (!obj_priv->active) 4069 if (!obj_priv->active)
4190 list_move_tail(&obj_priv->list, 4070 list_move_tail(&obj_priv->mm_list,
4191 &dev_priv->mm.pinned_list); 4071 &dev_priv->mm.pinned_list);
4192 } 4072 }
4193 4073
@@ -4213,7 +4093,7 @@ i915_gem_object_unpin(struct drm_gem_object *obj)
4213 */ 4093 */
4214 if (obj_priv->pin_count == 0) { 4094 if (obj_priv->pin_count == 0) {
4215 if (!obj_priv->active) 4095 if (!obj_priv->active)
4216 list_move_tail(&obj_priv->list, 4096 list_move_tail(&obj_priv->mm_list,
4217 &dev_priv->mm.inactive_list); 4097 &dev_priv->mm.inactive_list);
4218 i915_gem_info_remove_pin(dev_priv, obj->size); 4098 i915_gem_info_remove_pin(dev_priv, obj->size);
4219 } 4099 }
@@ -4229,44 +4109,36 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4229 struct drm_i915_gem_object *obj_priv; 4109 struct drm_i915_gem_object *obj_priv;
4230 int ret; 4110 int ret;
4231 4111
4112 ret = i915_mutex_lock_interruptible(dev);
4113 if (ret)
4114 return ret;
4115
4232 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 4116 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4233 if (obj == NULL) { 4117 if (obj == NULL) {
4234 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n", 4118 ret = -ENOENT;
4235 args->handle); 4119 goto unlock;
4236 return -ENOENT;
4237 } 4120 }
4238 obj_priv = to_intel_bo(obj); 4121 obj_priv = to_intel_bo(obj);
4239 4122
4240 ret = i915_mutex_lock_interruptible(dev);
4241 if (ret) {
4242 drm_gem_object_unreference_unlocked(obj);
4243 return ret;
4244 }
4245
4246 if (obj_priv->madv != I915_MADV_WILLNEED) { 4123 if (obj_priv->madv != I915_MADV_WILLNEED) {
4247 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 4124 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4248 drm_gem_object_unreference(obj); 4125 ret = -EINVAL;
4249 mutex_unlock(&dev->struct_mutex); 4126 goto out;
4250 return -EINVAL;
4251 } 4127 }
4252 4128
4253 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { 4129 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4254 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 4130 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4255 args->handle); 4131 args->handle);
4256 drm_gem_object_unreference(obj); 4132 ret = -EINVAL;
4257 mutex_unlock(&dev->struct_mutex); 4133 goto out;
4258 return -EINVAL;
4259 } 4134 }
4260 4135
4261 obj_priv->user_pin_count++; 4136 obj_priv->user_pin_count++;
4262 obj_priv->pin_filp = file_priv; 4137 obj_priv->pin_filp = file_priv;
4263 if (obj_priv->user_pin_count == 1) { 4138 if (obj_priv->user_pin_count == 1) {
4264 ret = i915_gem_object_pin(obj, args->alignment); 4139 ret = i915_gem_object_pin(obj, args->alignment);
4265 if (ret != 0) { 4140 if (ret)
4266 drm_gem_object_unreference(obj); 4141 goto out;
4267 mutex_unlock(&dev->struct_mutex);
4268 return ret;
4269 }
4270 } 4142 }
4271 4143
4272 /* XXX - flush the CPU caches for pinned objects 4144 /* XXX - flush the CPU caches for pinned objects
@@ -4274,10 +4146,11 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4274 */ 4146 */
4275 i915_gem_object_flush_cpu_write_domain(obj); 4147 i915_gem_object_flush_cpu_write_domain(obj);
4276 args->offset = obj_priv->gtt_offset; 4148 args->offset = obj_priv->gtt_offset;
4149out:
4277 drm_gem_object_unreference(obj); 4150 drm_gem_object_unreference(obj);
4151unlock:
4278 mutex_unlock(&dev->struct_mutex); 4152 mutex_unlock(&dev->struct_mutex);
4279 4153 return ret;
4280 return 0;
4281} 4154}
4282 4155
4283int 4156int
@@ -4289,27 +4162,22 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4289 struct drm_i915_gem_object *obj_priv; 4162 struct drm_i915_gem_object *obj_priv;
4290 int ret; 4163 int ret;
4291 4164
4165 ret = i915_mutex_lock_interruptible(dev);
4166 if (ret)
4167 return ret;
4168
4292 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 4169 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4293 if (obj == NULL) { 4170 if (obj == NULL) {
4294 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n", 4171 ret = -ENOENT;
4295 args->handle); 4172 goto unlock;
4296 return -ENOENT;
4297 } 4173 }
4298
4299 obj_priv = to_intel_bo(obj); 4174 obj_priv = to_intel_bo(obj);
4300 4175
4301 ret = i915_mutex_lock_interruptible(dev);
4302 if (ret) {
4303 drm_gem_object_unreference_unlocked(obj);
4304 return ret;
4305 }
4306
4307 if (obj_priv->pin_filp != file_priv) { 4176 if (obj_priv->pin_filp != file_priv) {
4308 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 4177 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4309 args->handle); 4178 args->handle);
4310 drm_gem_object_unreference(obj); 4179 ret = -EINVAL;
4311 mutex_unlock(&dev->struct_mutex); 4180 goto out;
4312 return -EINVAL;
4313 } 4181 }
4314 obj_priv->user_pin_count--; 4182 obj_priv->user_pin_count--;
4315 if (obj_priv->user_pin_count == 0) { 4183 if (obj_priv->user_pin_count == 0) {
@@ -4317,9 +4185,11 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4317 i915_gem_object_unpin(obj); 4185 i915_gem_object_unpin(obj);
4318 } 4186 }
4319 4187
4188out:
4320 drm_gem_object_unreference(obj); 4189 drm_gem_object_unreference(obj);
4190unlock:
4321 mutex_unlock(&dev->struct_mutex); 4191 mutex_unlock(&dev->struct_mutex);
4322 return 0; 4192 return ret;
4323} 4193}
4324 4194
4325int 4195int
@@ -4331,25 +4201,22 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4331 struct drm_i915_gem_object *obj_priv; 4201 struct drm_i915_gem_object *obj_priv;
4332 int ret; 4202 int ret;
4333 4203
4334 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4335 if (obj == NULL) {
4336 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4337 args->handle);
4338 return -ENOENT;
4339 }
4340
4341 ret = i915_mutex_lock_interruptible(dev); 4204 ret = i915_mutex_lock_interruptible(dev);
4342 if (ret) { 4205 if (ret)
4343 drm_gem_object_unreference_unlocked(obj);
4344 return ret; 4206 return ret;
4207
4208 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4209 if (obj == NULL) {
4210 ret = -ENOENT;
4211 goto unlock;
4345 } 4212 }
4213 obj_priv = to_intel_bo(obj);
4346 4214
4347 /* Count all active objects as busy, even if they are currently not used 4215 /* Count all active objects as busy, even if they are currently not used
4348 * by the gpu. Users of this interface expect objects to eventually 4216 * by the gpu. Users of this interface expect objects to eventually
4349 * become non-busy without any further actions, therefore emit any 4217 * become non-busy without any further actions, therefore emit any
4350 * necessary flushes here. 4218 * necessary flushes here.
4351 */ 4219 */
4352 obj_priv = to_intel_bo(obj);
4353 args->busy = obj_priv->active; 4220 args->busy = obj_priv->active;
4354 if (args->busy) { 4221 if (args->busy) {
4355 /* Unconditionally flush objects, even when the gpu still uses this 4222 /* Unconditionally flush objects, even when the gpu still uses this
@@ -4373,8 +4240,9 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4373 } 4240 }
4374 4241
4375 drm_gem_object_unreference(obj); 4242 drm_gem_object_unreference(obj);
4243unlock:
4376 mutex_unlock(&dev->struct_mutex); 4244 mutex_unlock(&dev->struct_mutex);
4377 return 0; 4245 return ret;
4378} 4246}
4379 4247
4380int 4248int
@@ -4401,26 +4269,20 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4401 return -EINVAL; 4269 return -EINVAL;
4402 } 4270 }
4403 4271
4272 ret = i915_mutex_lock_interruptible(dev);
4273 if (ret)
4274 return ret;
4275
4404 obj = drm_gem_object_lookup(dev, file_priv, args->handle); 4276 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4405 if (obj == NULL) { 4277 if (obj == NULL) {
4406 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n", 4278 ret = -ENOENT;
4407 args->handle); 4279 goto unlock;
4408 return -ENOENT;
4409 } 4280 }
4410 obj_priv = to_intel_bo(obj); 4281 obj_priv = to_intel_bo(obj);
4411 4282
4412 ret = i915_mutex_lock_interruptible(dev);
4413 if (ret) {
4414 drm_gem_object_unreference_unlocked(obj);
4415 return ret;
4416 }
4417
4418 if (obj_priv->pin_count) { 4283 if (obj_priv->pin_count) {
4419 drm_gem_object_unreference(obj); 4284 ret = -EINVAL;
4420 mutex_unlock(&dev->struct_mutex); 4285 goto out;
4421
4422 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4423 return -EINVAL;
4424 } 4286 }
4425 4287
4426 if (obj_priv->madv != __I915_MADV_PURGED) 4288 if (obj_priv->madv != __I915_MADV_PURGED)
@@ -4433,10 +4295,11 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4433 4295
4434 args->retained = obj_priv->madv != __I915_MADV_PURGED; 4296 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4435 4297
4298out:
4436 drm_gem_object_unreference(obj); 4299 drm_gem_object_unreference(obj);
4300unlock:
4437 mutex_unlock(&dev->struct_mutex); 4301 mutex_unlock(&dev->struct_mutex);
4438 4302 return ret;
4439 return 0;
4440} 4303}
4441 4304
4442struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, 4305struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
@@ -4462,12 +4325,11 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4462 obj->agp_type = AGP_USER_MEMORY; 4325 obj->agp_type = AGP_USER_MEMORY;
4463 obj->base.driver_private = NULL; 4326 obj->base.driver_private = NULL;
4464 obj->fence_reg = I915_FENCE_REG_NONE; 4327 obj->fence_reg = I915_FENCE_REG_NONE;
4465 INIT_LIST_HEAD(&obj->list); 4328 INIT_LIST_HEAD(&obj->mm_list);
4329 INIT_LIST_HEAD(&obj->ring_list);
4466 INIT_LIST_HEAD(&obj->gpu_write_list); 4330 INIT_LIST_HEAD(&obj->gpu_write_list);
4467 obj->madv = I915_MADV_WILLNEED; 4331 obj->madv = I915_MADV_WILLNEED;
4468 4332
4469 trace_i915_gem_object_create(&obj->base);
4470
4471 return &obj->base; 4333 return &obj->base;
4472} 4334}
4473 4335
@@ -4487,7 +4349,7 @@ static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4487 4349
4488 ret = i915_gem_object_unbind(obj); 4350 ret = i915_gem_object_unbind(obj);
4489 if (ret == -ERESTARTSYS) { 4351 if (ret == -ERESTARTSYS) {
4490 list_move(&obj_priv->list, 4352 list_move(&obj_priv->mm_list,
4491 &dev_priv->mm.deferred_free_list); 4353 &dev_priv->mm.deferred_free_list);
4492 return; 4354 return;
4493 } 4355 }
@@ -4527,10 +4389,7 @@ i915_gem_idle(struct drm_device *dev)
4527 4389
4528 mutex_lock(&dev->struct_mutex); 4390 mutex_lock(&dev->struct_mutex);
4529 4391
4530 if (dev_priv->mm.suspended || 4392 if (dev_priv->mm.suspended) {
4531 (dev_priv->render_ring.gem_object == NULL) ||
4532 (HAS_BSD(dev) &&
4533 dev_priv->bsd_ring.gem_object == NULL)) {
4534 mutex_unlock(&dev->struct_mutex); 4393 mutex_unlock(&dev->struct_mutex);
4535 return 0; 4394 return 0;
4536 } 4395 }
@@ -4651,10 +4510,18 @@ i915_gem_init_ringbuffer(struct drm_device *dev)
4651 goto cleanup_render_ring; 4510 goto cleanup_render_ring;
4652 } 4511 }
4653 4512
4513 if (HAS_BLT(dev)) {
4514 ret = intel_init_blt_ring_buffer(dev);
4515 if (ret)
4516 goto cleanup_bsd_ring;
4517 }
4518
4654 dev_priv->next_seqno = 1; 4519 dev_priv->next_seqno = 1;
4655 4520
4656 return 0; 4521 return 0;
4657 4522
4523cleanup_bsd_ring:
4524 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4658cleanup_render_ring: 4525cleanup_render_ring:
4659 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 4526 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4660cleanup_pipe_control: 4527cleanup_pipe_control:
@@ -4669,8 +4536,8 @@ i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4669 drm_i915_private_t *dev_priv = dev->dev_private; 4536 drm_i915_private_t *dev_priv = dev->dev_private;
4670 4537
4671 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); 4538 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4672 if (HAS_BSD(dev)) 4539 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4673 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); 4540 intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4674 if (HAS_PIPE_CONTROL(dev)) 4541 if (HAS_PIPE_CONTROL(dev))
4675 i915_gem_cleanup_pipe_control(dev); 4542 i915_gem_cleanup_pipe_control(dev);
4676} 4543}
@@ -4699,12 +4566,15 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4699 return ret; 4566 return ret;
4700 } 4567 }
4701 4568
4569 BUG_ON(!list_empty(&dev_priv->mm.active_list));
4702 BUG_ON(!list_empty(&dev_priv->render_ring.active_list)); 4570 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4703 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list)); 4571 BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4572 BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4704 BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); 4573 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4705 BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); 4574 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4706 BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); 4575 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4707 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list)); 4576 BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4577 BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4708 mutex_unlock(&dev->struct_mutex); 4578 mutex_unlock(&dev->struct_mutex);
4709 4579
4710 ret = drm_irq_install(dev); 4580 ret = drm_irq_install(dev);
@@ -4746,24 +4616,29 @@ i915_gem_lastclose(struct drm_device *dev)
4746 DRM_ERROR("failed to idle hardware: %d\n", ret); 4616 DRM_ERROR("failed to idle hardware: %d\n", ret);
4747} 4617}
4748 4618
4619static void
4620init_ring_lists(struct intel_ring_buffer *ring)
4621{
4622 INIT_LIST_HEAD(&ring->active_list);
4623 INIT_LIST_HEAD(&ring->request_list);
4624 INIT_LIST_HEAD(&ring->gpu_write_list);
4625}
4626
4749void 4627void
4750i915_gem_load(struct drm_device *dev) 4628i915_gem_load(struct drm_device *dev)
4751{ 4629{
4752 int i; 4630 int i;
4753 drm_i915_private_t *dev_priv = dev->dev_private; 4631 drm_i915_private_t *dev_priv = dev->dev_private;
4754 4632
4633 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4755 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 4634 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4756 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4757 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 4635 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4758 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 4636 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4759 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 4637 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4760 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 4638 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4761 INIT_LIST_HEAD(&dev_priv->render_ring.active_list); 4639 init_ring_lists(&dev_priv->render_ring);
4762 INIT_LIST_HEAD(&dev_priv->render_ring.request_list); 4640 init_ring_lists(&dev_priv->bsd_ring);
4763 if (HAS_BSD(dev)) { 4641 init_ring_lists(&dev_priv->blt_ring);
4764 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4765 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4766 }
4767 for (i = 0; i < 16; i++) 4642 for (i = 0; i < 16; i++)
4768 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 4643 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4769 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 4644 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
@@ -5026,9 +4901,9 @@ i915_gpu_is_active(struct drm_device *dev)
5026 int lists_empty; 4901 int lists_empty;
5027 4902
5028 lists_empty = list_empty(&dev_priv->mm.flushing_list) && 4903 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5029 list_empty(&dev_priv->render_ring.active_list); 4904 list_empty(&dev_priv->render_ring.active_list) &&
5030 if (HAS_BSD(dev)) 4905 list_empty(&dev_priv->bsd_ring.active_list) &&
5031 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list); 4906 list_empty(&dev_priv->blt_ring.active_list);
5032 4907
5033 return !lists_empty; 4908 return !lists_empty;
5034} 4909}
@@ -5050,7 +4925,7 @@ i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
5050 if (mutex_trylock(&dev->struct_mutex)) { 4925 if (mutex_trylock(&dev->struct_mutex)) {
5051 list_for_each_entry(obj_priv, 4926 list_for_each_entry(obj_priv,
5052 &dev_priv->mm.inactive_list, 4927 &dev_priv->mm.inactive_list,
5053 list) 4928 mm_list)
5054 cnt++; 4929 cnt++;
5055 mutex_unlock(&dev->struct_mutex); 4930 mutex_unlock(&dev->struct_mutex);
5056 } 4931 }
@@ -5076,7 +4951,7 @@ rescan:
5076 4951
5077 list_for_each_entry_safe(obj_priv, next_obj, 4952 list_for_each_entry_safe(obj_priv, next_obj,
5078 &dev_priv->mm.inactive_list, 4953 &dev_priv->mm.inactive_list,
5079 list) { 4954 mm_list) {
5080 if (i915_gem_object_is_purgeable(obj_priv)) { 4955 if (i915_gem_object_is_purgeable(obj_priv)) {
5081 i915_gem_object_unbind(&obj_priv->base); 4956 i915_gem_object_unbind(&obj_priv->base);
5082 if (--nr_to_scan <= 0) 4957 if (--nr_to_scan <= 0)
@@ -5105,7 +4980,7 @@ rescan:
5105 4980
5106 list_for_each_entry_safe(obj_priv, next_obj, 4981 list_for_each_entry_safe(obj_priv, next_obj,
5107 &dev_priv->mm.inactive_list, 4982 &dev_priv->mm.inactive_list,
5108 list) { 4983 mm_list) {
5109 if (nr_to_scan > 0) { 4984 if (nr_to_scan > 0) {
5110 i915_gem_object_unbind(&obj_priv->base); 4985 i915_gem_object_unbind(&obj_priv->base);
5111 nr_to_scan--; 4986 nr_to_scan--;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-22 17:12:55 -0500