aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c590
1 files changed, 360 insertions, 230 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ef188e39140..5e54821af99 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -38,8 +38,7 @@
38 38
39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
40 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, 41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
42 bool pipelined);
43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); 42static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); 43static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, 44static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
@@ -547,6 +546,19 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
547 struct drm_i915_gem_object *obj_priv; 546 struct drm_i915_gem_object *obj_priv;
548 int ret = 0; 547 int ret = 0;
549 548
549 if (args->size == 0)
550 return 0;
551
552 if (!access_ok(VERIFY_WRITE,
553 (char __user *)(uintptr_t)args->data_ptr,
554 args->size))
555 return -EFAULT;
556
557 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
558 args->size);
559 if (ret)
560 return -EFAULT;
561
550 ret = i915_mutex_lock_interruptible(dev); 562 ret = i915_mutex_lock_interruptible(dev);
551 if (ret) 563 if (ret)
552 return ret; 564 return ret;
@@ -564,23 +576,6 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
564 goto out; 576 goto out;
565 } 577 }
566 578
567 if (args->size == 0)
568 goto out;
569
570 if (!access_ok(VERIFY_WRITE,
571 (char __user *)(uintptr_t)args->data_ptr,
572 args->size)) {
573 ret = -EFAULT;
574 goto out;
575 }
576
577 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
578 args->size);
579 if (ret) {
580 ret = -EFAULT;
581 goto out;
582 }
583
584 ret = i915_gem_object_get_pages_or_evict(obj); 579 ret = i915_gem_object_get_pages_or_evict(obj);
585 if (ret) 580 if (ret)
586 goto out; 581 goto out;
@@ -981,7 +976,20 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
981 struct drm_i915_gem_pwrite *args = data; 976 struct drm_i915_gem_pwrite *args = data;
982 struct drm_gem_object *obj; 977 struct drm_gem_object *obj;
983 struct drm_i915_gem_object *obj_priv; 978 struct drm_i915_gem_object *obj_priv;
984 int ret = 0; 979 int ret;
980
981 if (args->size == 0)
982 return 0;
983
984 if (!access_ok(VERIFY_READ,
985 (char __user *)(uintptr_t)args->data_ptr,
986 args->size))
987 return -EFAULT;
988
989 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
990 args->size);
991 if (ret)
992 return -EFAULT;
985 993
986 ret = i915_mutex_lock_interruptible(dev); 994 ret = i915_mutex_lock_interruptible(dev);
987 if (ret) 995 if (ret)
@@ -994,30 +1002,12 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
994 } 1002 }
995 obj_priv = to_intel_bo(obj); 1003 obj_priv = to_intel_bo(obj);
996 1004
997
998 /* Bounds check destination. */ 1005 /* Bounds check destination. */
999 if (args->offset > obj->size || args->size > obj->size - args->offset) { 1006 if (args->offset > obj->size || args->size > obj->size - args->offset) {
1000 ret = -EINVAL; 1007 ret = -EINVAL;
1001 goto out; 1008 goto out;
1002 } 1009 }
1003 1010
1004 if (args->size == 0)
1005 goto out;
1006
1007 if (!access_ok(VERIFY_READ,
1008 (char __user *)(uintptr_t)args->data_ptr,
1009 args->size)) {
1010 ret = -EFAULT;
1011 goto out;
1012 }
1013
1014 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1015 args->size);
1016 if (ret) {
1017 ret = -EFAULT;
1018 goto out;
1019 }
1020
1021 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1011 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1022 * it would end up going through the fenced access, and we'll get 1012 * it would end up going through the fenced access, and we'll get
1023 * different detiling behavior between reading and writing. 1013 * different detiling behavior between reading and writing.
@@ -2603,7 +2593,7 @@ i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2603 if (reg->gpu) { 2593 if (reg->gpu) {
2604 int ret; 2594 int ret;
2605 2595
2606 ret = i915_gem_object_flush_gpu_write_domain(obj, true); 2596 ret = i915_gem_object_flush_gpu_write_domain(obj);
2607 if (ret) 2597 if (ret)
2608 return ret; 2598 return ret;
2609 2599
@@ -2751,8 +2741,7 @@ i915_gem_clflush_object(struct drm_gem_object *obj)
2751 2741
2752/** Flushes any GPU write domain for the object if it's dirty. */ 2742/** Flushes any GPU write domain for the object if it's dirty. */
2753static int 2743static int
2754i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj, 2744i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2755 bool pipelined)
2756{ 2745{
2757 struct drm_device *dev = obj->dev; 2746 struct drm_device *dev = obj->dev;
2758 uint32_t old_write_domain; 2747 uint32_t old_write_domain;
@@ -2771,10 +2760,7 @@ i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2771 obj->read_domains, 2760 obj->read_domains,
2772 old_write_domain); 2761 old_write_domain);
2773 2762
2774 if (pipelined) 2763 return 0;
2775 return 0;
2776
2777 return i915_gem_object_wait_rendering(obj, true);
2778} 2764}
2779 2765
2780/** Flushes the GTT write domain for the object if it's dirty. */ 2766/** Flushes the GTT write domain for the object if it's dirty. */
@@ -2835,18 +2821,15 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2835 if (obj_priv->gtt_space == NULL) 2821 if (obj_priv->gtt_space == NULL)
2836 return -EINVAL; 2822 return -EINVAL;
2837 2823
2838 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 2824 ret = i915_gem_object_flush_gpu_write_domain(obj);
2839 if (ret != 0) 2825 if (ret != 0)
2840 return ret; 2826 return ret;
2827 ret = i915_gem_object_wait_rendering(obj, true);
2828 if (ret)
2829 return ret;
2841 2830
2842 i915_gem_object_flush_cpu_write_domain(obj); 2831 i915_gem_object_flush_cpu_write_domain(obj);
2843 2832
2844 if (write) {
2845 ret = i915_gem_object_wait_rendering(obj, true);
2846 if (ret)
2847 return ret;
2848 }
2849
2850 old_write_domain = obj->write_domain; 2833 old_write_domain = obj->write_domain;
2851 old_read_domains = obj->read_domains; 2834 old_read_domains = obj->read_domains;
2852 2835
@@ -2884,7 +2867,7 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2884 if (obj_priv->gtt_space == NULL) 2867 if (obj_priv->gtt_space == NULL)
2885 return -EINVAL; 2868 return -EINVAL;
2886 2869
2887 ret = i915_gem_object_flush_gpu_write_domain(obj, true); 2870 ret = i915_gem_object_flush_gpu_write_domain(obj);
2888 if (ret) 2871 if (ret)
2889 return ret; 2872 return ret;
2890 2873
@@ -2907,6 +2890,20 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2907 return 0; 2890 return 0;
2908} 2891}
2909 2892
2893int
2894i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2895 bool interruptible)
2896{
2897 if (!obj->active)
2898 return 0;
2899
2900 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2901 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring,
2902 0, obj->base.write_domain);
2903
2904 return i915_gem_object_wait_rendering(&obj->base, interruptible);
2905}
2906
2910/** 2907/**
2911 * Moves a single object to the CPU read, and possibly write domain. 2908 * Moves a single object to the CPU read, and possibly write domain.
2912 * 2909 *
@@ -2919,9 +2916,12 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2919 uint32_t old_write_domain, old_read_domains; 2916 uint32_t old_write_domain, old_read_domains;
2920 int ret; 2917 int ret;
2921 2918
2922 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 2919 ret = i915_gem_object_flush_gpu_write_domain(obj);
2923 if (ret != 0) 2920 if (ret != 0)
2924 return ret; 2921 return ret;
2922 ret = i915_gem_object_wait_rendering(obj, true);
2923 if (ret)
2924 return ret;
2925 2925
2926 i915_gem_object_flush_gtt_write_domain(obj); 2926 i915_gem_object_flush_gtt_write_domain(obj);
2927 2927
@@ -2930,12 +2930,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2930 */ 2930 */
2931 i915_gem_object_set_to_full_cpu_read_domain(obj); 2931 i915_gem_object_set_to_full_cpu_read_domain(obj);
2932 2932
2933 if (write) {
2934 ret = i915_gem_object_wait_rendering(obj, true);
2935 if (ret)
2936 return ret;
2937 }
2938
2939 old_write_domain = obj->write_domain; 2933 old_write_domain = obj->write_domain;
2940 old_read_domains = obj->read_domains; 2934 old_read_domains = obj->read_domains;
2941 2935
@@ -3200,9 +3194,13 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3200 if (offset == 0 && size == obj->size) 3194 if (offset == 0 && size == obj->size)
3201 return i915_gem_object_set_to_cpu_domain(obj, 0); 3195 return i915_gem_object_set_to_cpu_domain(obj, 0);
3202 3196
3203 ret = i915_gem_object_flush_gpu_write_domain(obj, false); 3197 ret = i915_gem_object_flush_gpu_write_domain(obj);
3204 if (ret != 0) 3198 if (ret != 0)
3205 return ret; 3199 return ret;
3200 ret = i915_gem_object_wait_rendering(obj, true);
3201 if (ret)
3202 return ret;
3203
3206 i915_gem_object_flush_gtt_write_domain(obj); 3204 i915_gem_object_flush_gtt_write_domain(obj);
3207 3205
3208 /* If we're already fully in the CPU read domain, we're done. */ 3206 /* If we're already fully in the CPU read domain, we're done. */
@@ -3249,192 +3247,230 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3249 return 0; 3247 return 0;
3250} 3248}
3251 3249
3252/**
3253 * Pin an object to the GTT and evaluate the relocations landing in it.
3254 */
3255static int 3250static int
3256i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj, 3251i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3257 struct drm_file *file_priv, 3252 struct drm_file *file_priv,
3258 struct drm_i915_gem_exec_object2 *entry) 3253 struct drm_i915_gem_exec_object2 *entry,
3254 struct drm_i915_gem_relocation_entry *reloc)
3259{ 3255{
3260 struct drm_device *dev = obj->base.dev; 3256 struct drm_device *dev = obj->base.dev;
3261 drm_i915_private_t *dev_priv = dev->dev_private; 3257 struct drm_gem_object *target_obj;
3262 struct drm_i915_gem_relocation_entry __user *user_relocs; 3258 uint32_t target_offset;
3263 struct drm_gem_object *target_obj = NULL; 3259 int ret = -EINVAL;
3264 uint32_t target_handle = 0;
3265 int i, ret = 0;
3266 3260
3267 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr; 3261 target_obj = drm_gem_object_lookup(dev, file_priv,
3268 for (i = 0; i < entry->relocation_count; i++) { 3262 reloc->target_handle);
3269 struct drm_i915_gem_relocation_entry reloc; 3263 if (target_obj == NULL)
3270 uint32_t target_offset; 3264 return -ENOENT;
3271 3265
3272 if (__copy_from_user_inatomic(&reloc, 3266 target_offset = to_intel_bo(target_obj)->gtt_offset;
3273 user_relocs+i,
3274 sizeof(reloc))) {
3275 ret = -EFAULT;
3276 break;
3277 }
3278 3267
3279 if (reloc.target_handle != target_handle) { 3268#if WATCH_RELOC
3280 drm_gem_object_unreference(target_obj); 3269 DRM_INFO("%s: obj %p offset %08x target %d "
3270 "read %08x write %08x gtt %08x "
3271 "presumed %08x delta %08x\n",
3272 __func__,
3273 obj,
3274 (int) reloc->offset,
3275 (int) reloc->target_handle,
3276 (int) reloc->read_domains,
3277 (int) reloc->write_domain,
3278 (int) target_offset,
3279 (int) reloc->presumed_offset,
3280 reloc->delta);
3281#endif
3281 3282
3282 target_obj = drm_gem_object_lookup(dev, file_priv, 3283 /* The target buffer should have appeared before us in the
3283 reloc.target_handle); 3284 * exec_object list, so it should have a GTT space bound by now.
3284 if (target_obj == NULL) { 3285 */
3285 ret = -ENOENT; 3286 if (target_offset == 0) {
3286 break; 3287 DRM_ERROR("No GTT space found for object %d\n",
3287 } 3288 reloc->target_handle);
3289 goto err;
3290 }
3288 3291
3289 target_handle = reloc.target_handle; 3292 /* Validate that the target is in a valid r/w GPU domain */
3290 } 3293 if (reloc->write_domain & (reloc->write_domain - 1)) {
3291 target_offset = to_intel_bo(target_obj)->gtt_offset; 3294 DRM_ERROR("reloc with multiple write domains: "
3295 "obj %p target %d offset %d "
3296 "read %08x write %08x",
3297 obj, reloc->target_handle,
3298 (int) reloc->offset,
3299 reloc->read_domains,
3300 reloc->write_domain);
3301 goto err;
3302 }
3303 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3304 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3305 DRM_ERROR("reloc with read/write CPU domains: "
3306 "obj %p target %d offset %d "
3307 "read %08x write %08x",
3308 obj, reloc->target_handle,
3309 (int) reloc->offset,
3310 reloc->read_domains,
3311 reloc->write_domain);
3312 goto err;
3313 }
3314 if (reloc->write_domain && target_obj->pending_write_domain &&
3315 reloc->write_domain != target_obj->pending_write_domain) {
3316 DRM_ERROR("Write domain conflict: "
3317 "obj %p target %d offset %d "
3318 "new %08x old %08x\n",
3319 obj, reloc->target_handle,
3320 (int) reloc->offset,
3321 reloc->write_domain,
3322 target_obj->pending_write_domain);
3323 goto err;
3324 }
3292 3325
3293#if WATCH_RELOC 3326 target_obj->pending_read_domains |= reloc->read_domains;
3294 DRM_INFO("%s: obj %p offset %08x target %d " 3327 target_obj->pending_write_domain |= reloc->write_domain;
3295 "read %08x write %08x gtt %08x "
3296 "presumed %08x delta %08x\n",
3297 __func__,
3298 obj,
3299 (int) reloc.offset,
3300 (int) reloc.target_handle,
3301 (int) reloc.read_domains,
3302 (int) reloc.write_domain,
3303 (int) target_offset,
3304 (int) reloc.presumed_offset,
3305 reloc.delta);
3306#endif
3307 3328
3308 /* The target buffer should have appeared before us in the 3329 /* If the relocation already has the right value in it, no
3309 * exec_object list, so it should have a GTT space bound by now. 3330 * more work needs to be done.
3310 */ 3331 */
3311 if (target_offset == 0) { 3332 if (target_offset == reloc->presumed_offset)
3312 DRM_ERROR("No GTT space found for object %d\n", 3333 goto out;
3313 reloc.target_handle);
3314 ret = -EINVAL;
3315 break;
3316 }
3317 3334
3318 /* Validate that the target is in a valid r/w GPU domain */ 3335 /* Check that the relocation address is valid... */
3319 if (reloc.write_domain & (reloc.write_domain - 1)) { 3336 if (reloc->offset > obj->base.size - 4) {
3320 DRM_ERROR("reloc with multiple write domains: " 3337 DRM_ERROR("Relocation beyond object bounds: "
3321 "obj %p target %d offset %d " 3338 "obj %p target %d offset %d size %d.\n",
3322 "read %08x write %08x", 3339 obj, reloc->target_handle,
3323 obj, reloc.target_handle, 3340 (int) reloc->offset,
3324 (int) reloc.offset, 3341 (int) obj->base.size);
3325 reloc.read_domains, 3342 goto err;
3326 reloc.write_domain); 3343 }
3327 ret = -EINVAL; 3344 if (reloc->offset & 3) {
3328 break; 3345 DRM_ERROR("Relocation not 4-byte aligned: "
3329 } 3346 "obj %p target %d offset %d.\n",
3330 if (reloc.write_domain & I915_GEM_DOMAIN_CPU || 3347 obj, reloc->target_handle,
3331 reloc.read_domains & I915_GEM_DOMAIN_CPU) { 3348 (int) reloc->offset);
3332 DRM_ERROR("reloc with read/write CPU domains: " 3349 goto err;
3333 "obj %p target %d offset %d " 3350 }
3334 "read %08x write %08x",
3335 obj, reloc.target_handle,
3336 (int) reloc.offset,
3337 reloc.read_domains,
3338 reloc.write_domain);
3339 ret = -EINVAL;
3340 break;
3341 }
3342 if (reloc.write_domain && target_obj->pending_write_domain &&
3343 reloc.write_domain != target_obj->pending_write_domain) {
3344 DRM_ERROR("Write domain conflict: "
3345 "obj %p target %d offset %d "
3346 "new %08x old %08x\n",
3347 obj, reloc.target_handle,
3348 (int) reloc.offset,
3349 reloc.write_domain,
3350 target_obj->pending_write_domain);
3351 ret = -EINVAL;
3352 break;
3353 }
3354 3351
3355 target_obj->pending_read_domains |= reloc.read_domains; 3352 /* and points to somewhere within the target object. */
3356 target_obj->pending_write_domain |= reloc.write_domain; 3353 if (reloc->delta >= target_obj->size) {
3354 DRM_ERROR("Relocation beyond target object bounds: "
3355 "obj %p target %d delta %d size %d.\n",
3356 obj, reloc->target_handle,
3357 (int) reloc->delta,
3358 (int) target_obj->size);
3359 goto err;
3360 }
3357 3361
3358 /* If the relocation already has the right value in it, no 3362 reloc->delta += target_offset;
3359 * more work needs to be done. 3363 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3360 */ 3364 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3361 if (target_offset == reloc.presumed_offset) 3365 char *vaddr;
3362 continue;
3363 3366
3364 /* Check that the relocation address is valid... */ 3367 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3365 if (reloc.offset > obj->base.size - 4) { 3368 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3366 DRM_ERROR("Relocation beyond object bounds: " 3369 kunmap_atomic(vaddr);
3367 "obj %p target %d offset %d size %d.\n", 3370 } else {
3368 obj, reloc.target_handle, 3371 struct drm_i915_private *dev_priv = dev->dev_private;
3369 (int) reloc.offset, (int) obj->base.size); 3372 uint32_t __iomem *reloc_entry;
3370 ret = -EINVAL; 3373 void __iomem *reloc_page;
3371 break;
3372 }
3373 if (reloc.offset & 3) {
3374 DRM_ERROR("Relocation not 4-byte aligned: "
3375 "obj %p target %d offset %d.\n",
3376 obj, reloc.target_handle,
3377 (int) reloc.offset);
3378 ret = -EINVAL;
3379 break;
3380 }
3381 3374
3382 /* and points to somewhere within the target object. */ 3375 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3383 if (reloc.delta >= target_obj->size) { 3376 if (ret)
3384 DRM_ERROR("Relocation beyond target object bounds: " 3377 goto err;
3385 "obj %p target %d delta %d size %d.\n",
3386 obj, reloc.target_handle,
3387 (int) reloc.delta, (int) target_obj->size);
3388 ret = -EINVAL;
3389 break;
3390 }
3391 3378
3392 reloc.delta += target_offset; 3379 /* Map the page containing the relocation we're going to perform. */
3393 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) { 3380 reloc->offset += obj->gtt_offset;
3394 uint32_t page_offset = reloc.offset & ~PAGE_MASK; 3381 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3395 char *vaddr; 3382 reloc->offset & PAGE_MASK);
3383 reloc_entry = (uint32_t __iomem *)
3384 (reloc_page + (reloc->offset & ~PAGE_MASK));
3385 iowrite32(reloc->delta, reloc_entry);
3386 io_mapping_unmap_atomic(reloc_page);
3387 }
3396 3388
3397 vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]); 3389 /* and update the user's relocation entry */
3398 *(uint32_t *)(vaddr + page_offset) = reloc.delta; 3390 reloc->presumed_offset = target_offset;
3399 kunmap_atomic(vaddr);
3400 } else {
3401 uint32_t __iomem *reloc_entry;
3402 void __iomem *reloc_page;
3403 3391
3404 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1); 3392out:
3405 if (ret) 3393 ret = 0;
3406 break; 3394err:
3395 drm_gem_object_unreference(target_obj);
3396 return ret;
3397}
3407 3398
3408 /* Map the page containing the relocation we're going to perform. */ 3399static int
3409 reloc.offset += obj->gtt_offset; 3400i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3410 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping, 3401 struct drm_file *file_priv,
3411 reloc.offset & PAGE_MASK); 3402 struct drm_i915_gem_exec_object2 *entry)
3412 reloc_entry = (uint32_t __iomem *) 3403{
3413 (reloc_page + (reloc.offset & ~PAGE_MASK)); 3404 struct drm_i915_gem_relocation_entry __user *user_relocs;
3414 iowrite32(reloc.delta, reloc_entry); 3405 int i, ret;
3415 io_mapping_unmap_atomic(reloc_page); 3406
3416 } 3407 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3408 for (i = 0; i < entry->relocation_count; i++) {
3409 struct drm_i915_gem_relocation_entry reloc;
3410
3411 if (__copy_from_user_inatomic(&reloc,
3412 user_relocs+i,
3413 sizeof(reloc)))
3414 return -EFAULT;
3415
3416 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3417 if (ret)
3418 return ret;
3417 3419
3418 /* and update the user's relocation entry */
3419 reloc.presumed_offset = target_offset;
3420 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset, 3420 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3421 &reloc.presumed_offset, 3421 &reloc.presumed_offset,
3422 sizeof(reloc.presumed_offset))) { 3422 sizeof(reloc.presumed_offset)))
3423 ret = -EFAULT; 3423 return -EFAULT;
3424 break;
3425 }
3426 } 3424 }
3427 3425
3428 drm_gem_object_unreference(target_obj); 3426 return 0;
3429 return ret; 3427}
3428
3429static int
3430i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3431 struct drm_file *file_priv,
3432 struct drm_i915_gem_exec_object2 *entry,
3433 struct drm_i915_gem_relocation_entry *relocs)
3434{
3435 int i, ret;
3436
3437 for (i = 0; i < entry->relocation_count; i++) {
3438 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3439 if (ret)
3440 return ret;
3441 }
3442
3443 return 0;
3444}
3445
3446static int
3447i915_gem_execbuffer_relocate(struct drm_device *dev,
3448 struct drm_file *file,
3449 struct drm_gem_object **object_list,
3450 struct drm_i915_gem_exec_object2 *exec_list,
3451 int count)
3452{
3453 int i, ret;
3454
3455 for (i = 0; i < count; i++) {
3456 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3457 obj->base.pending_read_domains = 0;
3458 obj->base.pending_write_domain = 0;
3459 ret = i915_gem_execbuffer_relocate_object(obj, file,
3460 &exec_list[i]);
3461 if (ret)
3462 return ret;
3463 }
3464
3465 return 0;
3430} 3466}
3431 3467
3432static int 3468static int
3433i915_gem_execbuffer_pin(struct drm_device *dev, 3469i915_gem_execbuffer_reserve(struct drm_device *dev,
3434 struct drm_file *file, 3470 struct drm_file *file,
3435 struct drm_gem_object **object_list, 3471 struct drm_gem_object **object_list,
3436 struct drm_i915_gem_exec_object2 *exec_list, 3472 struct drm_i915_gem_exec_object2 *exec_list,
3437 int count) 3473 int count)
3438{ 3474{
3439 struct drm_i915_private *dev_priv = dev->dev_private; 3475 struct drm_i915_private *dev_priv = dev->dev_private;
3440 int ret, i, retry; 3476 int ret, i, retry;
@@ -3497,6 +3533,87 @@ i915_gem_execbuffer_pin(struct drm_device *dev,
3497} 3533}
3498 3534
3499static int 3535static int
3536i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3537 struct drm_file *file,
3538 struct drm_gem_object **object_list,
3539 struct drm_i915_gem_exec_object2 *exec_list,
3540 int count)
3541{
3542 struct drm_i915_gem_relocation_entry *reloc;
3543 int i, total, ret;
3544
3545 for (i = 0; i < count; i++) {
3546 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3547 obj->in_execbuffer = false;
3548 }
3549
3550 mutex_unlock(&dev->struct_mutex);
3551
3552 total = 0;
3553 for (i = 0; i < count; i++)
3554 total += exec_list[i].relocation_count;
3555
3556 reloc = drm_malloc_ab(total, sizeof(*reloc));
3557 if (reloc == NULL) {
3558 mutex_lock(&dev->struct_mutex);
3559 return -ENOMEM;
3560 }
3561
3562 total = 0;
3563 for (i = 0; i < count; i++) {
3564 struct drm_i915_gem_relocation_entry __user *user_relocs;
3565
3566 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3567
3568 if (copy_from_user(reloc+total, user_relocs,
3569 exec_list[i].relocation_count *
3570 sizeof(*reloc))) {
3571 ret = -EFAULT;
3572 mutex_lock(&dev->struct_mutex);
3573 goto err;
3574 }
3575
3576 total += exec_list[i].relocation_count;
3577 }
3578
3579 ret = i915_mutex_lock_interruptible(dev);
3580 if (ret) {
3581 mutex_lock(&dev->struct_mutex);
3582 goto err;
3583 }
3584
3585 ret = i915_gem_execbuffer_reserve(dev, file,
3586 object_list, exec_list,
3587 count);
3588 if (ret)
3589 goto err;
3590
3591 total = 0;
3592 for (i = 0; i < count; i++) {
3593 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3594 obj->base.pending_read_domains = 0;
3595 obj->base.pending_write_domain = 0;
3596 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3597 &exec_list[i],
3598 reloc + total);
3599 if (ret)
3600 goto err;
3601
3602 total += exec_list[i].relocation_count;
3603 }
3604
3605 /* Leave the user relocations as are, this is the painfully slow path,
3606 * and we want to avoid the complication of dropping the lock whilst
3607 * having buffers reserved in the aperture and so causing spurious
3608 * ENOSPC for random operations.
3609 */
3610
3611err:
3612 drm_free_large(reloc);
3613 return ret;
3614}
3615
3616static int
3500i915_gem_execbuffer_move_to_gpu(struct drm_device *dev, 3617i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3501 struct drm_file *file, 3618 struct drm_file *file,
3502 struct intel_ring_buffer *ring, 3619 struct intel_ring_buffer *ring,
@@ -3625,8 +3742,15 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3625 3742
3626 for (i = 0; i < count; i++) { 3743 for (i = 0; i < count; i++) {
3627 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr; 3744 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3628 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry); 3745 int length; /* limited by fault_in_pages_readable() */
3629 3746
3747 /* First check for malicious input causing overflow */
3748 if (exec[i].relocation_count >
3749 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3750 return -EINVAL;
3751
3752 length = exec[i].relocation_count *
3753 sizeof(struct drm_i915_gem_relocation_entry);
3630 if (!access_ok(VERIFY_READ, ptr, length)) 3754 if (!access_ok(VERIFY_READ, ptr, length))
3631 return -EFAULT; 3755 return -EFAULT;
3632 3756
@@ -3769,18 +3893,24 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3769 } 3893 }
3770 3894
3771 /* Move the objects en-masse into the GTT, evicting if necessary. */ 3895 /* Move the objects en-masse into the GTT, evicting if necessary. */
3772 ret = i915_gem_execbuffer_pin(dev, file, 3896 ret = i915_gem_execbuffer_reserve(dev, file,
3773 object_list, exec_list, 3897 object_list, exec_list,
3774 args->buffer_count); 3898 args->buffer_count);
3775 if (ret) 3899 if (ret)
3776 goto err; 3900 goto err;
3777 3901
3778 /* The objects are in their final locations, apply the relocations. */ 3902 /* The objects are in their final locations, apply the relocations. */
3779 for (i = 0; i < args->buffer_count; i++) { 3903 ret = i915_gem_execbuffer_relocate(dev, file,
3780 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]); 3904 object_list, exec_list,
3781 obj->base.pending_read_domains = 0; 3905 args->buffer_count);
3782 obj->base.pending_write_domain = 0; 3906 if (ret) {
3783 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]); 3907 if (ret == -EFAULT) {
3908 ret = i915_gem_execbuffer_relocate_slow(dev, file,
3909 object_list,
3910 exec_list,
3911 args->buffer_count);
3912 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3913 }
3784 if (ret) 3914 if (ret)
3785 goto err; 3915 goto err;
3786 } 3916 }
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-22 11:54:12 -0400