aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/ttm/ttm_bo.c
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
commitc71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch)
treeecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /drivers/gpu/drm/ttm/ttm_bo.c
parentea53c912f8a86a8567697115b6a0d8152beee5c8 (diff)
parent6a00f206debf8a5c8899055726ad127dbeeed098 (diff)
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts: litmus/sched_cedf.c
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
-rw-r--r--drivers/gpu/drm/ttm/ttm_bo.c550
1 files changed, 266 insertions, 284 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index db809e034cc4..2e618b5ac465 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -27,14 +27,6 @@
27/* 27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */ 29 */
30/* Notes:
31 *
32 * We store bo pointer in drm_mm_node struct so we know which bo own a
33 * specific node. There is no protection on the pointer, thus to make
34 * sure things don't go berserk you have to access this pointer while
35 * holding the global lru lock and make sure anytime you free a node you
36 * reset the pointer to NULL.
37 */
38 30
39#include "ttm/ttm_module.h" 31#include "ttm/ttm_module.h"
40#include "ttm/ttm_bo_driver.h" 32#include "ttm/ttm_bo_driver.h"
@@ -45,6 +37,7 @@
45#include <linux/mm.h> 37#include <linux/mm.h>
46#include <linux/file.h> 38#include <linux/file.h>
47#include <linux/module.h> 39#include <linux/module.h>
40#include <asm/atomic.h>
48 41
49#define TTM_ASSERT_LOCKED(param) 42#define TTM_ASSERT_LOCKED(param)
50#define TTM_DEBUG(fmt, arg...) 43#define TTM_DEBUG(fmt, arg...)
@@ -84,11 +77,8 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
84 man->available_caching); 77 man->available_caching);
85 printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", 78 printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n",
86 man->default_caching); 79 man->default_caching);
87 if (mem_type != TTM_PL_SYSTEM) { 80 if (mem_type != TTM_PL_SYSTEM)
88 spin_lock(&bdev->glob->lru_lock); 81 (*man->func->debug)(man, TTM_PFX);
89 drm_mm_debug_table(&man->manager, TTM_PFX);
90 spin_unlock(&bdev->glob->lru_lock);
91 }
92} 82}
93 83
94static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 84static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
@@ -169,22 +159,17 @@ static void ttm_bo_release_list(struct kref *list_kref)
169 159
170int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) 160int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
171{ 161{
172
173 if (interruptible) { 162 if (interruptible) {
174 int ret = 0; 163 return wait_event_interruptible(bo->event_queue,
175
176 ret = wait_event_interruptible(bo->event_queue,
177 atomic_read(&bo->reserved) == 0); 164 atomic_read(&bo->reserved) == 0);
178 if (unlikely(ret != 0))
179 return ret;
180 } else { 165 } else {
181 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); 166 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
167 return 0;
182 } 168 }
183 return 0;
184} 169}
185EXPORT_SYMBOL(ttm_bo_wait_unreserved); 170EXPORT_SYMBOL(ttm_bo_wait_unreserved);
186 171
187static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 172void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
188{ 173{
189 struct ttm_bo_device *bdev = bo->bdev; 174 struct ttm_bo_device *bdev = bo->bdev;
190 struct ttm_mem_type_manager *man; 175 struct ttm_mem_type_manager *man;
@@ -206,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
206 } 191 }
207} 192}
208 193
209/** 194int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
210 * Call with the lru_lock held.
211 */
212
213static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
214{ 195{
215 int put_count = 0; 196 int put_count = 0;
216 197
@@ -239,9 +220,21 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
239 int ret; 220 int ret;
240 221
241 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { 222 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
242 if (use_sequence && bo->seq_valid && 223 /**
243 (sequence - bo->val_seq < (1 << 31))) { 224 * Deadlock avoidance for multi-bo reserving.
244 return -EAGAIN; 225 */
226 if (use_sequence && bo->seq_valid) {
227 /**
228 * We've already reserved this one.
229 */
230 if (unlikely(sequence == bo->val_seq))
231 return -EDEADLK;
232 /**
233 * Already reserved by a thread that will not back
234 * off for us. We need to back off.
235 */
236 if (unlikely(sequence - bo->val_seq < (1 << 31)))
237 return -EAGAIN;
245 } 238 }
246 239
247 if (no_wait) 240 if (no_wait)
@@ -256,6 +249,14 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
256 } 249 }
257 250
258 if (use_sequence) { 251 if (use_sequence) {
252 /**
253 * Wake up waiters that may need to recheck for deadlock,
254 * if we decreased the sequence number.
255 */
256 if (unlikely((bo->val_seq - sequence < (1 << 31))
257 || !bo->seq_valid))
258 wake_up_all(&bo->event_queue);
259
259 bo->val_seq = sequence; 260 bo->val_seq = sequence;
260 bo->seq_valid = true; 261 bo->seq_valid = true;
261 } else { 262 } else {
@@ -271,6 +272,13 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
271 BUG(); 272 BUG();
272} 273}
273 274
275void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
276 bool never_free)
277{
278 kref_sub(&bo->list_kref, count,
279 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
280}
281
274int ttm_bo_reserve(struct ttm_buffer_object *bo, 282int ttm_bo_reserve(struct ttm_buffer_object *bo,
275 bool interruptible, 283 bool interruptible,
276 bool no_wait, bool use_sequence, uint32_t sequence) 284 bool no_wait, bool use_sequence, uint32_t sequence)
@@ -286,20 +294,24 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
286 put_count = ttm_bo_del_from_lru(bo); 294 put_count = ttm_bo_del_from_lru(bo);
287 spin_unlock(&glob->lru_lock); 295 spin_unlock(&glob->lru_lock);
288 296
289 while (put_count--) 297 ttm_bo_list_ref_sub(bo, put_count, true);
290 kref_put(&bo->list_kref, ttm_bo_ref_bug);
291 298
292 return ret; 299 return ret;
293} 300}
294 301
302void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
303{
304 ttm_bo_add_to_lru(bo);
305 atomic_set(&bo->reserved, 0);
306 wake_up_all(&bo->event_queue);
307}
308
295void ttm_bo_unreserve(struct ttm_buffer_object *bo) 309void ttm_bo_unreserve(struct ttm_buffer_object *bo)
296{ 310{
297 struct ttm_bo_global *glob = bo->glob; 311 struct ttm_bo_global *glob = bo->glob;
298 312
299 spin_lock(&glob->lru_lock); 313 spin_lock(&glob->lru_lock);
300 ttm_bo_add_to_lru(bo); 314 ttm_bo_unreserve_locked(bo);
301 atomic_set(&bo->reserved, 0);
302 wake_up_all(&bo->event_queue);
303 spin_unlock(&glob->lru_lock); 315 spin_unlock(&glob->lru_lock);
304} 316}
305EXPORT_SYMBOL(ttm_bo_unreserve); 317EXPORT_SYMBOL(ttm_bo_unreserve);
@@ -366,8 +378,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
366 int ret = 0; 378 int ret = 0;
367 379
368 if (old_is_pci || new_is_pci || 380 if (old_is_pci || new_is_pci ||
369 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) 381 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
370 ttm_bo_unmap_virtual(bo); 382 ret = ttm_mem_io_lock(old_man, true);
383 if (unlikely(ret != 0))
384 goto out_err;
385 ttm_bo_unmap_virtual_locked(bo);
386 ttm_mem_io_unlock(old_man);
387 }
371 388
372 /* 389 /*
373 * Create and bind a ttm if required. 390 * Create and bind a ttm if required.
@@ -389,11 +406,12 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
389 } 406 }
390 407
391 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 408 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
409 if (bdev->driver->move_notify)
410 bdev->driver->move_notify(bo, mem);
392 bo->mem = *mem; 411 bo->mem = *mem;
393 mem->mm_node = NULL; 412 mem->mm_node = NULL;
394 goto moved; 413 goto moved;
395 } 414 }
396
397 } 415 }
398 416
399 if (bdev->driver->move_notify) 417 if (bdev->driver->move_notify)
@@ -420,11 +438,9 @@ moved:
420 } 438 }
421 439
422 if (bo->mem.mm_node) { 440 if (bo->mem.mm_node) {
423 spin_lock(&bo->lock); 441 bo->offset = (bo->mem.start << PAGE_SHIFT) +
424 bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
425 bdev->man[bo->mem.mem_type].gpu_offset; 442 bdev->man[bo->mem.mem_type].gpu_offset;
426 bo->cur_placement = bo->mem.placement; 443 bo->cur_placement = bo->mem.placement;
427 spin_unlock(&bo->lock);
428 } else 444 } else
429 bo->offset = 0; 445 bo->offset = 0;
430 446
@@ -442,135 +458,150 @@ out_err:
442} 458}
443 459
444/** 460/**
445 * Call bo::reserved and with the lru lock held. 461 * Call bo::reserved.
446 * Will release GPU memory type usage on destruction. 462 * Will release GPU memory type usage on destruction.
447 * This is the place to put in driver specific hooks. 463 * This is the place to put in driver specific hooks to release
448 * Will release the bo::reserved lock and the 464 * driver private resources.
449 * lru lock on exit. 465 * Will release the bo::reserved lock.
450 */ 466 */
451 467
452static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 468static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
453{ 469{
454 struct ttm_bo_global *glob = bo->glob;
455
456 if (bo->ttm) { 470 if (bo->ttm) {
457
458 /**
459 * Release the lru_lock, since we don't want to have
460 * an atomic requirement on ttm_tt[unbind|destroy].
461 */
462
463 spin_unlock(&glob->lru_lock);
464 ttm_tt_unbind(bo->ttm); 471 ttm_tt_unbind(bo->ttm);
465 ttm_tt_destroy(bo->ttm); 472 ttm_tt_destroy(bo->ttm);
466 bo->ttm = NULL; 473 bo->ttm = NULL;
467 spin_lock(&glob->lru_lock);
468 }
469
470 if (bo->mem.mm_node) {
471 drm_mm_put_block(bo->mem.mm_node);
472 bo->mem.mm_node = NULL;
473 } 474 }
475 ttm_bo_mem_put(bo, &bo->mem);
474 476
475 atomic_set(&bo->reserved, 0); 477 atomic_set(&bo->reserved, 0);
478
479 /*
480 * Make processes trying to reserve really pick it up.
481 */
482 smp_mb__after_atomic_dec();
476 wake_up_all(&bo->event_queue); 483 wake_up_all(&bo->event_queue);
477 spin_unlock(&glob->lru_lock);
478} 484}
479 485
480 486static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
481/**
482 * If bo idle, remove from delayed- and lru lists, and unref.
483 * If not idle, and already on delayed list, do nothing.
484 * If not idle, and not on delayed list, put on delayed list,
485 * up the list_kref and schedule a delayed list check.
486 */
487
488static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
489{ 487{
490 struct ttm_bo_device *bdev = bo->bdev; 488 struct ttm_bo_device *bdev = bo->bdev;
491 struct ttm_bo_global *glob = bo->glob; 489 struct ttm_bo_global *glob = bo->glob;
492 struct ttm_bo_driver *driver = bdev->driver; 490 struct ttm_bo_driver *driver;
491 void *sync_obj = NULL;
492 void *sync_obj_arg;
493 int put_count;
493 int ret; 494 int ret;
494 495
495 spin_lock(&bo->lock); 496 spin_lock(&bdev->fence_lock);
496retry: 497 (void) ttm_bo_wait(bo, false, false, true);
497 (void) ttm_bo_wait(bo, false, false, !remove_all);
498
499 if (!bo->sync_obj) { 498 if (!bo->sync_obj) {
500 int put_count;
501
502 spin_unlock(&bo->lock);
503 499
504 spin_lock(&glob->lru_lock); 500 spin_lock(&glob->lru_lock);
505 ret = ttm_bo_reserve_locked(bo, false, !remove_all, false, 0);
506 501
507 /** 502 /**
508 * Someone else has the object reserved. Bail and retry. 503 * Lock inversion between bo:reserve and bdev::fence_lock here,
504 * but that's OK, since we're only trylocking.
509 */ 505 */
510 506
511 if (unlikely(ret == -EBUSY)) { 507 ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
512 spin_unlock(&glob->lru_lock);
513 spin_lock(&bo->lock);
514 goto requeue;
515 }
516
517 /**
518 * We can re-check for sync object without taking
519 * the bo::lock since setting the sync object requires
520 * also bo::reserved. A busy object at this point may
521 * be caused by another thread starting an accelerated
522 * eviction.
523 */
524 508
525 if (unlikely(bo->sync_obj)) { 509 if (unlikely(ret == -EBUSY))
526 atomic_set(&bo->reserved, 0); 510 goto queue;
527 wake_up_all(&bo->event_queue);
528 spin_unlock(&glob->lru_lock);
529 spin_lock(&bo->lock);
530 if (remove_all)
531 goto retry;
532 else
533 goto requeue;
534 }
535 511
512 spin_unlock(&bdev->fence_lock);
536 put_count = ttm_bo_del_from_lru(bo); 513 put_count = ttm_bo_del_from_lru(bo);
537 514
538 if (!list_empty(&bo->ddestroy)) { 515 spin_unlock(&glob->lru_lock);
539 list_del_init(&bo->ddestroy);
540 ++put_count;
541 }
542
543 ttm_bo_cleanup_memtype_use(bo); 516 ttm_bo_cleanup_memtype_use(bo);
544 517
545 while (put_count--) 518 ttm_bo_list_ref_sub(bo, put_count, true);
546 kref_put(&bo->list_kref, ttm_bo_ref_bug);
547 519
548 return 0; 520 return;
521 } else {
522 spin_lock(&glob->lru_lock);
523 }
524queue:
525 driver = bdev->driver;
526 if (bo->sync_obj)
527 sync_obj = driver->sync_obj_ref(bo->sync_obj);
528 sync_obj_arg = bo->sync_obj_arg;
529
530 kref_get(&bo->list_kref);
531 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
532 spin_unlock(&glob->lru_lock);
533 spin_unlock(&bdev->fence_lock);
534
535 if (sync_obj) {
536 driver->sync_obj_flush(sync_obj, sync_obj_arg);
537 driver->sync_obj_unref(&sync_obj);
549 } 538 }
550requeue: 539 schedule_delayed_work(&bdev->wq,
540 ((HZ / 100) < 1) ? 1 : HZ / 100);
541}
542
543/**
544 * function ttm_bo_cleanup_refs
545 * If bo idle, remove from delayed- and lru lists, and unref.
546 * If not idle, do nothing.
547 *
548 * @interruptible Any sleeps should occur interruptibly.
549 * @no_wait_reserve Never wait for reserve. Return -EBUSY instead.
550 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
551 */
552
553static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
554 bool interruptible,
555 bool no_wait_reserve,
556 bool no_wait_gpu)
557{
558 struct ttm_bo_device *bdev = bo->bdev;
559 struct ttm_bo_global *glob = bo->glob;
560 int put_count;
561 int ret = 0;
562
563retry:
564 spin_lock(&bdev->fence_lock);
565 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
566 spin_unlock(&bdev->fence_lock);
567
568 if (unlikely(ret != 0))
569 return ret;
570
551 spin_lock(&glob->lru_lock); 571 spin_lock(&glob->lru_lock);
552 if (list_empty(&bo->ddestroy)) { 572 ret = ttm_bo_reserve_locked(bo, interruptible,
553 void *sync_obj = bo->sync_obj; 573 no_wait_reserve, false, 0);
554 void *sync_obj_arg = bo->sync_obj_arg;
555 574
556 kref_get(&bo->list_kref); 575 if (unlikely(ret != 0) || list_empty(&bo->ddestroy)) {
557 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
558 spin_unlock(&glob->lru_lock); 576 spin_unlock(&glob->lru_lock);
559 spin_unlock(&bo->lock); 577 return ret;
578 }
560 579
561 if (sync_obj) 580 /**
562 driver->sync_obj_flush(sync_obj, sync_obj_arg); 581 * We can re-check for sync object without taking
563 schedule_delayed_work(&bdev->wq, 582 * the bo::lock since setting the sync object requires
564 ((HZ / 100) < 1) ? 1 : HZ / 100); 583 * also bo::reserved. A busy object at this point may
565 ret = 0; 584 * be caused by another thread recently starting an accelerated
585 * eviction.
586 */
566 587
567 } else { 588 if (unlikely(bo->sync_obj)) {
589 atomic_set(&bo->reserved, 0);
590 wake_up_all(&bo->event_queue);
568 spin_unlock(&glob->lru_lock); 591 spin_unlock(&glob->lru_lock);
569 spin_unlock(&bo->lock); 592 goto retry;
570 ret = -EBUSY;
571 } 593 }
572 594
573 return ret; 595 put_count = ttm_bo_del_from_lru(bo);
596 list_del_init(&bo->ddestroy);
597 ++put_count;
598
599 spin_unlock(&glob->lru_lock);
600 ttm_bo_cleanup_memtype_use(bo);
601
602 ttm_bo_list_ref_sub(bo, put_count, true);
603
604 return 0;
574} 605}
575 606
576/** 607/**
@@ -602,7 +633,8 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
602 } 633 }
603 634
604 spin_unlock(&glob->lru_lock); 635 spin_unlock(&glob->lru_lock);
605 ret = ttm_bo_cleanup_refs(entry, remove_all); 636 ret = ttm_bo_cleanup_refs(entry, false, !remove_all,
637 !remove_all);
606 kref_put(&entry->list_kref, ttm_bo_release_list); 638 kref_put(&entry->list_kref, ttm_bo_release_list);
607 entry = nentry; 639 entry = nentry;
608 640
@@ -638,6 +670,7 @@ static void ttm_bo_release(struct kref *kref)
638 struct ttm_buffer_object *bo = 670 struct ttm_buffer_object *bo =
639 container_of(kref, struct ttm_buffer_object, kref); 671 container_of(kref, struct ttm_buffer_object, kref);
640 struct ttm_bo_device *bdev = bo->bdev; 672 struct ttm_bo_device *bdev = bo->bdev;
673 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
641 674
642 if (likely(bo->vm_node != NULL)) { 675 if (likely(bo->vm_node != NULL)) {
643 rb_erase(&bo->vm_rb, &bdev->addr_space_rb); 676 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
@@ -645,7 +678,10 @@ static void ttm_bo_release(struct kref *kref)
645 bo->vm_node = NULL; 678 bo->vm_node = NULL;
646 } 679 }
647 write_unlock(&bdev->vm_lock); 680 write_unlock(&bdev->vm_lock);
648 ttm_bo_cleanup_refs(bo, false); 681 ttm_mem_io_lock(man, false);
682 ttm_mem_io_free_vm(bo);
683 ttm_mem_io_unlock(man);
684 ttm_bo_cleanup_refs_or_queue(bo);
649 kref_put(&bo->list_kref, ttm_bo_release_list); 685 kref_put(&bo->list_kref, ttm_bo_release_list);
650 write_lock(&bdev->vm_lock); 686 write_lock(&bdev->vm_lock);
651} 687}
@@ -680,14 +716,13 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
680 bool no_wait_reserve, bool no_wait_gpu) 716 bool no_wait_reserve, bool no_wait_gpu)
681{ 717{
682 struct ttm_bo_device *bdev = bo->bdev; 718 struct ttm_bo_device *bdev = bo->bdev;
683 struct ttm_bo_global *glob = bo->glob;
684 struct ttm_mem_reg evict_mem; 719 struct ttm_mem_reg evict_mem;
685 struct ttm_placement placement; 720 struct ttm_placement placement;
686 int ret = 0; 721 int ret = 0;
687 722
688 spin_lock(&bo->lock); 723 spin_lock(&bdev->fence_lock);
689 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 724 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
690 spin_unlock(&bo->lock); 725 spin_unlock(&bdev->fence_lock);
691 726
692 if (unlikely(ret != 0)) { 727 if (unlikely(ret != 0)) {
693 if (ret != -ERESTARTSYS) { 728 if (ret != -ERESTARTSYS) {
@@ -702,7 +737,8 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
702 737
703 evict_mem = bo->mem; 738 evict_mem = bo->mem;
704 evict_mem.mm_node = NULL; 739 evict_mem.mm_node = NULL;
705 evict_mem.bus.io_reserved = false; 740 evict_mem.bus.io_reserved_vm = false;
741 evict_mem.bus.io_reserved_count = 0;
706 742
707 placement.fpfn = 0; 743 placement.fpfn = 0;
708 placement.lpfn = 0; 744 placement.lpfn = 0;
@@ -726,12 +762,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
726 if (ret) { 762 if (ret) {
727 if (ret != -ERESTARTSYS) 763 if (ret != -ERESTARTSYS)
728 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); 764 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
729 spin_lock(&glob->lru_lock); 765 ttm_bo_mem_put(bo, &evict_mem);
730 if (evict_mem.mm_node) {
731 drm_mm_put_block(evict_mem.mm_node);
732 evict_mem.mm_node = NULL;
733 }
734 spin_unlock(&glob->lru_lock);
735 goto out; 766 goto out;
736 } 767 }
737 bo->evicted = true; 768 bo->evicted = true;
@@ -759,6 +790,18 @@ retry:
759 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); 790 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
760 kref_get(&bo->list_kref); 791 kref_get(&bo->list_kref);
761 792
793 if (!list_empty(&bo->ddestroy)) {
794 spin_unlock(&glob->lru_lock);
795 ret = ttm_bo_cleanup_refs(bo, interruptible,
796 no_wait_reserve, no_wait_gpu);
797 kref_put(&bo->list_kref, ttm_bo_release_list);
798
799 if (likely(ret == 0 || ret == -ERESTARTSYS))
800 return ret;
801
802 goto retry;
803 }
804
762 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); 805 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
763 806
764 if (unlikely(ret == -EBUSY)) { 807 if (unlikely(ret == -EBUSY)) {
@@ -782,8 +825,7 @@ retry:
782 825
783 BUG_ON(ret != 0); 826 BUG_ON(ret != 0);
784 827
785 while (put_count--) 828 ttm_bo_list_ref_sub(bo, put_count, true);
786 kref_put(&bo->list_kref, ttm_bo_ref_bug);
787 829
788 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); 830 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
789 ttm_bo_unreserve(bo); 831 ttm_bo_unreserve(bo);
@@ -792,41 +834,14 @@ retry:
792 return ret; 834 return ret;
793} 835}
794 836
795static int ttm_bo_man_get_node(struct ttm_buffer_object *bo, 837void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
796 struct ttm_mem_type_manager *man,
797 struct ttm_placement *placement,
798 struct ttm_mem_reg *mem,
799 struct drm_mm_node **node)
800{ 838{
801 struct ttm_bo_global *glob = bo->glob; 839 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
802 unsigned long lpfn;
803 int ret;
804
805 lpfn = placement->lpfn;
806 if (!lpfn)
807 lpfn = man->size;
808 *node = NULL;
809 do {
810 ret = drm_mm_pre_get(&man->manager);
811 if (unlikely(ret))
812 return ret;
813 840
814 spin_lock(&glob->lru_lock); 841 if (mem->mm_node)
815 *node = drm_mm_search_free_in_range(&man->manager, 842 (*man->func->put_node)(man, mem);
816 mem->num_pages, mem->page_alignment,
817 placement->fpfn, lpfn, 1);
818 if (unlikely(*node == NULL)) {
819 spin_unlock(&glob->lru_lock);
820 return 0;
821 }
822 *node = drm_mm_get_block_atomic_range(*node, mem->num_pages,
823 mem->page_alignment,
824 placement->fpfn,
825 lpfn);
826 spin_unlock(&glob->lru_lock);
827 } while (*node == NULL);
828 return 0;
829} 843}
844EXPORT_SYMBOL(ttm_bo_mem_put);
830 845
831/** 846/**
832 * Repeatedly evict memory from the LRU for @mem_type until we create enough 847 * Repeatedly evict memory from the LRU for @mem_type until we create enough
@@ -841,31 +856,22 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
841 bool no_wait_gpu) 856 bool no_wait_gpu)
842{ 857{
843 struct ttm_bo_device *bdev = bo->bdev; 858 struct ttm_bo_device *bdev = bo->bdev;
844 struct ttm_bo_global *glob = bdev->glob;
845 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 859 struct ttm_mem_type_manager *man = &bdev->man[mem_type];
846 struct drm_mm_node *node;
847 int ret; 860 int ret;
848 861
849 do { 862 do {
850 ret = ttm_bo_man_get_node(bo, man, placement, mem, &node); 863 ret = (*man->func->get_node)(man, bo, placement, mem);
851 if (unlikely(ret != 0)) 864 if (unlikely(ret != 0))
852 return ret; 865 return ret;
853 if (node) 866 if (mem->mm_node)
854 break;
855 spin_lock(&glob->lru_lock);
856 if (list_empty(&man->lru)) {
857 spin_unlock(&glob->lru_lock);
858 break; 867 break;
859 }
860 spin_unlock(&glob->lru_lock);
861 ret = ttm_mem_evict_first(bdev, mem_type, interruptible, 868 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
862 no_wait_reserve, no_wait_gpu); 869 no_wait_reserve, no_wait_gpu);
863 if (unlikely(ret != 0)) 870 if (unlikely(ret != 0))
864 return ret; 871 return ret;
865 } while (1); 872 } while (1);
866 if (node == NULL) 873 if (mem->mm_node == NULL)
867 return -ENOMEM; 874 return -ENOMEM;
868 mem->mm_node = node;
869 mem->mem_type = mem_type; 875 mem->mem_type = mem_type;
870 return 0; 876 return 0;
871} 877}
@@ -939,7 +945,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
939 bool type_found = false; 945 bool type_found = false;
940 bool type_ok = false; 946 bool type_ok = false;
941 bool has_erestartsys = false; 947 bool has_erestartsys = false;
942 struct drm_mm_node *node = NULL;
943 int i, ret; 948 int i, ret;
944 949
945 mem->mm_node = NULL; 950 mem->mm_node = NULL;
@@ -973,17 +978,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
973 978
974 if (man->has_type && man->use_type) { 979 if (man->has_type && man->use_type) {
975 type_found = true; 980 type_found = true;
976 ret = ttm_bo_man_get_node(bo, man, placement, mem, 981 ret = (*man->func->get_node)(man, bo, placement, mem);
977 &node);
978 if (unlikely(ret)) 982 if (unlikely(ret))
979 return ret; 983 return ret;
980 } 984 }
981 if (node) 985 if (mem->mm_node)
982 break; 986 break;
983 } 987 }
984 988
985 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) { 989 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
986 mem->mm_node = node;
987 mem->mem_type = mem_type; 990 mem->mem_type = mem_type;
988 mem->placement = cur_flags; 991 mem->placement = cur_flags;
989 return 0; 992 return 0;
@@ -1053,9 +1056,9 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1053 bool interruptible, bool no_wait_reserve, 1056 bool interruptible, bool no_wait_reserve,
1054 bool no_wait_gpu) 1057 bool no_wait_gpu)
1055{ 1058{
1056 struct ttm_bo_global *glob = bo->glob;
1057 int ret = 0; 1059 int ret = 0;
1058 struct ttm_mem_reg mem; 1060 struct ttm_mem_reg mem;
1061 struct ttm_bo_device *bdev = bo->bdev;
1059 1062
1060 BUG_ON(!atomic_read(&bo->reserved)); 1063 BUG_ON(!atomic_read(&bo->reserved));
1061 1064
@@ -1064,15 +1067,16 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1064 * Have the driver move function wait for idle when necessary, 1067 * Have the driver move function wait for idle when necessary,
1065 * instead of doing it here. 1068 * instead of doing it here.
1066 */ 1069 */
1067 spin_lock(&bo->lock); 1070 spin_lock(&bdev->fence_lock);
1068 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1071 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
1069 spin_unlock(&bo->lock); 1072 spin_unlock(&bdev->fence_lock);
1070 if (ret) 1073 if (ret)
1071 return ret; 1074 return ret;
1072 mem.num_pages = bo->num_pages; 1075 mem.num_pages = bo->num_pages;
1073 mem.size = mem.num_pages << PAGE_SHIFT; 1076 mem.size = mem.num_pages << PAGE_SHIFT;
1074 mem.page_alignment = bo->mem.page_alignment; 1077 mem.page_alignment = bo->mem.page_alignment;
1075 mem.bus.io_reserved = false; 1078 mem.bus.io_reserved_vm = false;
1079 mem.bus.io_reserved_count = 0;
1076 /* 1080 /*
1077 * Determine where to move the buffer. 1081 * Determine where to move the buffer.
1078 */ 1082 */
@@ -1081,11 +1085,8 @@ int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1081 goto out_unlock; 1085 goto out_unlock;
1082 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); 1086 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
1083out_unlock: 1087out_unlock:
1084 if (ret && mem.mm_node) { 1088 if (ret && mem.mm_node)
1085 spin_lock(&glob->lru_lock); 1089 ttm_bo_mem_put(bo, &mem);
1086 drm_mm_put_block(mem.mm_node);
1087 spin_unlock(&glob->lru_lock);
1088 }
1089 return ret; 1090 return ret;
1090} 1091}
1091 1092
@@ -1093,11 +1094,10 @@ static int ttm_bo_mem_compat(struct ttm_placement *placement,
1093 struct ttm_mem_reg *mem) 1094 struct ttm_mem_reg *mem)
1094{ 1095{
1095 int i; 1096 int i;
1096 struct drm_mm_node *node = mem->mm_node;
1097 1097
1098 if (node && placement->lpfn != 0 && 1098 if (mem->mm_node && placement->lpfn != 0 &&
1099 (node->start < placement->fpfn || 1099 (mem->start < placement->fpfn ||
1100 node->start + node->size > placement->lpfn)) 1100 mem->start + mem->num_pages > placement->lpfn))
1101 return -1; 1101 return -1;
1102 1102
1103 for (i = 0; i < placement->num_placement; i++) { 1103 for (i = 0; i < placement->num_placement; i++) {
@@ -1154,35 +1154,9 @@ EXPORT_SYMBOL(ttm_bo_validate);
1154int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1154int ttm_bo_check_placement(struct ttm_buffer_object *bo,
1155 struct ttm_placement *placement) 1155 struct ttm_placement *placement)
1156{ 1156{
1157 int i; 1157 BUG_ON((placement->fpfn || placement->lpfn) &&
1158 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
1158 1159
1159 if (placement->fpfn || placement->lpfn) {
1160 if (bo->mem.num_pages > (placement->lpfn - placement->fpfn)) {
1161 printk(KERN_ERR TTM_PFX "Page number range to small "
1162 "Need %lu pages, range is [%u, %u]\n",
1163 bo->mem.num_pages, placement->fpfn,
1164 placement->lpfn);
1165 return -EINVAL;
1166 }
1167 }
1168 for (i = 0; i < placement->num_placement; i++) {
1169 if (!capable(CAP_SYS_ADMIN)) {
1170 if (placement->placement[i] & TTM_PL_FLAG_NO_EVICT) {
1171 printk(KERN_ERR TTM_PFX "Need to be root to "
1172 "modify NO_EVICT status.\n");
1173 return -EINVAL;
1174 }
1175 }
1176 }
1177 for (i = 0; i < placement->num_busy_placement; i++) {
1178 if (!capable(CAP_SYS_ADMIN)) {
1179 if (placement->busy_placement[i] & TTM_PL_FLAG_NO_EVICT) {
1180 printk(KERN_ERR TTM_PFX "Need to be root to "
1181 "modify NO_EVICT status.\n");
1182 return -EINVAL;
1183 }
1184 }
1185 }
1186 return 0; 1160 return 0;
1187} 1161}
1188 1162
@@ -1194,7 +1168,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1194 uint32_t page_alignment, 1168 uint32_t page_alignment,
1195 unsigned long buffer_start, 1169 unsigned long buffer_start,
1196 bool interruptible, 1170 bool interruptible,
1197 struct file *persistant_swap_storage, 1171 struct file *persistent_swap_storage,
1198 size_t acc_size, 1172 size_t acc_size,
1199 void (*destroy) (struct ttm_buffer_object *)) 1173 void (*destroy) (struct ttm_buffer_object *))
1200{ 1174{
@@ -1205,11 +1179,14 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1205 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1179 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1206 if (num_pages == 0) { 1180 if (num_pages == 0) {
1207 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n"); 1181 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
1182 if (destroy)
1183 (*destroy)(bo);
1184 else
1185 kfree(bo);
1208 return -EINVAL; 1186 return -EINVAL;
1209 } 1187 }
1210 bo->destroy = destroy; 1188 bo->destroy = destroy;
1211 1189
1212 spin_lock_init(&bo->lock);
1213 kref_init(&bo->kref); 1190 kref_init(&bo->kref);
1214 kref_init(&bo->list_kref); 1191 kref_init(&bo->list_kref);
1215 atomic_set(&bo->cpu_writers, 0); 1192 atomic_set(&bo->cpu_writers, 0);
@@ -1218,6 +1195,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1218 INIT_LIST_HEAD(&bo->lru); 1195 INIT_LIST_HEAD(&bo->lru);
1219 INIT_LIST_HEAD(&bo->ddestroy); 1196 INIT_LIST_HEAD(&bo->ddestroy);
1220 INIT_LIST_HEAD(&bo->swap); 1197 INIT_LIST_HEAD(&bo->swap);
1198 INIT_LIST_HEAD(&bo->io_reserve_lru);
1221 bo->bdev = bdev; 1199 bo->bdev = bdev;
1222 bo->glob = bdev->glob; 1200 bo->glob = bdev->glob;
1223 bo->type = type; 1201 bo->type = type;
@@ -1227,12 +1205,13 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
1227 bo->mem.num_pages = bo->num_pages; 1205 bo->mem.num_pages = bo->num_pages;
1228 bo->mem.mm_node = NULL; 1206 bo->mem.mm_node = NULL;
1229 bo->mem.page_alignment = page_alignment; 1207 bo->mem.page_alignment = page_alignment;
1230 bo->mem.bus.io_reserved = false; 1208 bo->mem.bus.io_reserved_vm = false;
1209 bo->mem.bus.io_reserved_count = 0;
1231 bo->buffer_start = buffer_start & PAGE_MASK; 1210 bo->buffer_start = buffer_start & PAGE_MASK;
1232 bo->priv_flags = 0; 1211 bo->priv_flags = 0;
1233 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1212 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1234 bo->seq_valid = false; 1213 bo->seq_valid = false;
1235 bo->persistant_swap_storage = persistant_swap_storage; 1214 bo->persistent_swap_storage = persistent_swap_storage;
1236 bo->acc_size = acc_size; 1215 bo->acc_size = acc_size;
1237 atomic_inc(&bo->glob->bo_count); 1216 atomic_inc(&bo->glob->bo_count);
1238 1217
@@ -1281,7 +1260,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
1281 uint32_t page_alignment, 1260 uint32_t page_alignment,
1282 unsigned long buffer_start, 1261 unsigned long buffer_start,
1283 bool interruptible, 1262 bool interruptible,
1284 struct file *persistant_swap_storage, 1263 struct file *persistent_swap_storage,
1285 struct ttm_buffer_object **p_bo) 1264 struct ttm_buffer_object **p_bo)
1286{ 1265{
1287 struct ttm_buffer_object *bo; 1266 struct ttm_buffer_object *bo;
@@ -1303,7 +1282,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev,
1303 1282
1304 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1283 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1305 buffer_start, interruptible, 1284 buffer_start, interruptible,
1306 persistant_swap_storage, acc_size, NULL); 1285 persistent_swap_storage, acc_size, NULL);
1307 if (likely(ret == 0)) 1286 if (likely(ret == 0))
1308 *p_bo = bo; 1287 *p_bo = bo;
1309 1288
@@ -1341,7 +1320,6 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1341 1320
1342int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1321int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1343{ 1322{
1344 struct ttm_bo_global *glob = bdev->glob;
1345 struct ttm_mem_type_manager *man; 1323 struct ttm_mem_type_manager *man;
1346 int ret = -EINVAL; 1324 int ret = -EINVAL;
1347 1325
@@ -1364,13 +1342,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1364 if (mem_type > 0) { 1342 if (mem_type > 0) {
1365 ttm_bo_force_list_clean(bdev, mem_type, false); 1343 ttm_bo_force_list_clean(bdev, mem_type, false);
1366 1344
1367 spin_lock(&glob->lru_lock); 1345 ret = (*man->func->takedown)(man);
1368 if (drm_mm_clean(&man->manager))
1369 drm_mm_takedown(&man->manager);
1370 else
1371 ret = -EBUSY;
1372
1373 spin_unlock(&glob->lru_lock);
1374 } 1346 }
1375 1347
1376 return ret; 1348 return ret;
@@ -1405,32 +1377,22 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1405 int ret = -EINVAL; 1377 int ret = -EINVAL;
1406 struct ttm_mem_type_manager *man; 1378 struct ttm_mem_type_manager *man;
1407 1379
1408 if (type >= TTM_NUM_MEM_TYPES) { 1380 BUG_ON(type >= TTM_NUM_MEM_TYPES);
1409 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type);
1410 return ret;
1411 }
1412
1413 man = &bdev->man[type]; 1381 man = &bdev->man[type];
1414 if (man->has_type) { 1382 BUG_ON(man->has_type);
1415 printk(KERN_ERR TTM_PFX 1383 man->io_reserve_fastpath = true;
1416 "Memory manager already initialized for type %d\n", 1384 man->use_io_reserve_lru = false;
1417 type); 1385 mutex_init(&man->io_reserve_mutex);
1418 return ret; 1386 INIT_LIST_HEAD(&man->io_reserve_lru);
1419 }
1420 1387
1421 ret = bdev->driver->init_mem_type(bdev, type, man); 1388 ret = bdev->driver->init_mem_type(bdev, type, man);
1422 if (ret) 1389 if (ret)
1423 return ret; 1390 return ret;
1391 man->bdev = bdev;
1424 1392
1425 ret = 0; 1393 ret = 0;
1426 if (type != TTM_PL_SYSTEM) { 1394 if (type != TTM_PL_SYSTEM) {
1427 if (!p_size) { 1395 ret = (*man->func->init)(man, p_size);
1428 printk(KERN_ERR TTM_PFX
1429 "Zero size memory manager type %d\n",
1430 type);
1431 return ret;
1432 }
1433 ret = drm_mm_init(&man->manager, 0, p_size);
1434 if (ret) 1396 if (ret)
1435 return ret; 1397 return ret;
1436 } 1398 }
@@ -1539,8 +1501,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
1539 list_del(&bdev->device_list); 1501 list_del(&bdev->device_list);
1540 mutex_unlock(&glob->device_list_mutex); 1502 mutex_unlock(&glob->device_list_mutex);
1541 1503
1542 if (!cancel_delayed_work(&bdev->wq)) 1504 cancel_delayed_work_sync(&bdev->wq);
1543 flush_scheduled_work();
1544 1505
1545 while (ttm_bo_delayed_delete(bdev, true)) 1506 while (ttm_bo_delayed_delete(bdev, true))
1546 ; 1507 ;
@@ -1594,7 +1555,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
1594 bdev->dev_mapping = NULL; 1555 bdev->dev_mapping = NULL;
1595 bdev->glob = glob; 1556 bdev->glob = glob;
1596 bdev->need_dma32 = need_dma32; 1557 bdev->need_dma32 = need_dma32;
1597 1558 bdev->val_seq = 0;
1559 spin_lock_init(&bdev->fence_lock);
1598 mutex_lock(&glob->device_list_mutex); 1560 mutex_lock(&glob->device_list_mutex);
1599 list_add_tail(&bdev->device_list, &glob->device_list); 1561 list_add_tail(&bdev->device_list, &glob->device_list);
1600 mutex_unlock(&glob->device_list_mutex); 1562 mutex_unlock(&glob->device_list_mutex);
@@ -1628,7 +1590,7 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1628 return true; 1590 return true;
1629} 1591}
1630 1592
1631void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1593void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1632{ 1594{
1633 struct ttm_bo_device *bdev = bo->bdev; 1595 struct ttm_bo_device *bdev = bo->bdev;
1634 loff_t offset = (loff_t) bo->addr_space_offset; 1596 loff_t offset = (loff_t) bo->addr_space_offset;
@@ -1637,8 +1599,20 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1637 if (!bdev->dev_mapping) 1599 if (!bdev->dev_mapping)
1638 return; 1600 return;
1639 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1601 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
1640 ttm_mem_io_free(bdev, &bo->mem); 1602 ttm_mem_io_free_vm(bo);
1603}
1604
1605void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1606{
1607 struct ttm_bo_device *bdev = bo->bdev;
1608 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1609
1610 ttm_mem_io_lock(man, false);
1611 ttm_bo_unmap_virtual_locked(bo);
1612 ttm_mem_io_unlock(man);
1641} 1613}
1614
1615
1642EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1616EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1643 1617
1644static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1618static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
@@ -1718,6 +1692,7 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1718 bool lazy, bool interruptible, bool no_wait) 1692 bool lazy, bool interruptible, bool no_wait)
1719{ 1693{
1720 struct ttm_bo_driver *driver = bo->bdev->driver; 1694 struct ttm_bo_driver *driver = bo->bdev->driver;
1695 struct ttm_bo_device *bdev = bo->bdev;
1721 void *sync_obj; 1696 void *sync_obj;
1722 void *sync_obj_arg; 1697 void *sync_obj_arg;
1723 int ret = 0; 1698 int ret = 0;
@@ -1731,9 +1706,9 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1731 void *tmp_obj = bo->sync_obj; 1706 void *tmp_obj = bo->sync_obj;
1732 bo->sync_obj = NULL; 1707 bo->sync_obj = NULL;
1733 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1708 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1734 spin_unlock(&bo->lock); 1709 spin_unlock(&bdev->fence_lock);
1735 driver->sync_obj_unref(&tmp_obj); 1710 driver->sync_obj_unref(&tmp_obj);
1736 spin_lock(&bo->lock); 1711 spin_lock(&bdev->fence_lock);
1737 continue; 1712 continue;
1738 } 1713 }
1739 1714
@@ -1742,29 +1717,29 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,
1742 1717
1743 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1718 sync_obj = driver->sync_obj_ref(bo->sync_obj);
1744 sync_obj_arg = bo->sync_obj_arg; 1719 sync_obj_arg = bo->sync_obj_arg;
1745 spin_unlock(&bo->lock); 1720 spin_unlock(&bdev->fence_lock);
1746 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, 1721 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
1747 lazy, interruptible); 1722 lazy, interruptible);
1748 if (unlikely(ret != 0)) { 1723 if (unlikely(ret != 0)) {
1749 driver->sync_obj_unref(&sync_obj); 1724 driver->sync_obj_unref(&sync_obj);
1750 spin_lock(&bo->lock); 1725 spin_lock(&bdev->fence_lock);
1751 return ret; 1726 return ret;
1752 } 1727 }
1753 spin_lock(&bo->lock); 1728 spin_lock(&bdev->fence_lock);
1754 if (likely(bo->sync_obj == sync_obj && 1729 if (likely(bo->sync_obj == sync_obj &&
1755 bo->sync_obj_arg == sync_obj_arg)) { 1730 bo->sync_obj_arg == sync_obj_arg)) {
1756 void *tmp_obj = bo->sync_obj; 1731 void *tmp_obj = bo->sync_obj;
1757 bo->sync_obj = NULL; 1732 bo->sync_obj = NULL;
1758 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1733 clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1759 &bo->priv_flags); 1734 &bo->priv_flags);
1760 spin_unlock(&bo->lock); 1735 spin_unlock(&bdev->fence_lock);
1761 driver->sync_obj_unref(&sync_obj); 1736 driver->sync_obj_unref(&sync_obj);
1762 driver->sync_obj_unref(&tmp_obj); 1737 driver->sync_obj_unref(&tmp_obj);
1763 spin_lock(&bo->lock); 1738 spin_lock(&bdev->fence_lock);
1764 } else { 1739 } else {
1765 spin_unlock(&bo->lock); 1740 spin_unlock(&bdev->fence_lock);
1766 driver->sync_obj_unref(&sync_obj); 1741 driver->sync_obj_unref(&sync_obj);
1767 spin_lock(&bo->lock); 1742 spin_lock(&bdev->fence_lock);
1768 } 1743 }
1769 } 1744 }
1770 return 0; 1745 return 0;
@@ -1773,6 +1748,7 @@ EXPORT_SYMBOL(ttm_bo_wait);
1773 1748
1774int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1749int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1775{ 1750{
1751 struct ttm_bo_device *bdev = bo->bdev;
1776 int ret = 0; 1752 int ret = 0;
1777 1753
1778 /* 1754 /*
@@ -1782,9 +1758,9 @@ int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1782 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1758 ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1783 if (unlikely(ret != 0)) 1759 if (unlikely(ret != 0))
1784 return ret; 1760 return ret;
1785 spin_lock(&bo->lock); 1761 spin_lock(&bdev->fence_lock);
1786 ret = ttm_bo_wait(bo, false, true, no_wait); 1762 ret = ttm_bo_wait(bo, false, true, no_wait);
1787 spin_unlock(&bo->lock); 1763 spin_unlock(&bdev->fence_lock);
1788 if (likely(ret == 0)) 1764 if (likely(ret == 0))
1789 atomic_inc(&bo->cpu_writers); 1765 atomic_inc(&bo->cpu_writers);
1790 ttm_bo_unreserve(bo); 1766 ttm_bo_unreserve(bo);
@@ -1824,6 +1800,13 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1824 struct ttm_buffer_object, swap); 1800 struct ttm_buffer_object, swap);
1825 kref_get(&bo->list_kref); 1801 kref_get(&bo->list_kref);
1826 1802
1803 if (!list_empty(&bo->ddestroy)) {
1804 spin_unlock(&glob->lru_lock);
1805 (void) ttm_bo_cleanup_refs(bo, false, false, false);
1806 kref_put(&bo->list_kref, ttm_bo_release_list);
1807 continue;
1808 }
1809
1827 /** 1810 /**
1828 * Reserve buffer. Since we unlock while sleeping, we need 1811 * Reserve buffer. Since we unlock while sleeping, we need
1829 * to re-check that nobody removed us from the swap-list while 1812 * to re-check that nobody removed us from the swap-list while
@@ -1843,16 +1826,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1843 put_count = ttm_bo_del_from_lru(bo); 1826 put_count = ttm_bo_del_from_lru(bo);
1844 spin_unlock(&glob->lru_lock); 1827 spin_unlock(&glob->lru_lock);
1845 1828
1846 while (put_count--) 1829 ttm_bo_list_ref_sub(bo, put_count, true);
1847 kref_put(&bo->list_kref, ttm_bo_ref_bug);
1848 1830
1849 /** 1831 /**
1850 * Wait for GPU, then move to system cached. 1832 * Wait for GPU, then move to system cached.
1851 */ 1833 */
1852 1834
1853 spin_lock(&bo->lock); 1835 spin_lock(&bo->bdev->fence_lock);
1854 ret = ttm_bo_wait(bo, false, false, false); 1836 ret = ttm_bo_wait(bo, false, false, false);
1855 spin_unlock(&bo->lock); 1837 spin_unlock(&bo->bdev->fence_lock);
1856 1838
1857 if (unlikely(ret != 0)) 1839 if (unlikely(ret != 0))
1858 goto out; 1840 goto out;
@@ -1881,7 +1863,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1881 if (bo->bdev->driver->swap_notify) 1863 if (bo->bdev->driver->swap_notify)
1882 bo->bdev->driver->swap_notify(bo); 1864 bo->bdev->driver->swap_notify(bo);
1883 1865
1884 ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage); 1866 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1885out: 1867out:
1886 1868
1887 /** 1869 /**