diff options
author | Dave Airlie <airlied@redhat.com> | 2010-11-21 22:24:40 -0500 |
---|---|---|
committer | Dave Airlie <airlied@redhat.com> | 2010-11-21 22:24:40 -0500 |
commit | d6ea88865d3e5b0c62040531310c1f2c6a994f46 (patch) | |
tree | b80a7cbc6eeab003b412e3037fd335ce9d572f67 /drivers/gpu/drm/ttm | |
parent | 27641c3f003e7f3b6585c01d8a788883603eb262 (diff) |
drm/ttm: Add a bo list reserve fastpath (v2)
Makes it possible to reserve a list of buffer objects with a single
spin lock / unlock if there is no contention.
Should improve cpu usage on SMP kernels.
v2: Initialize private list members on reserve and don't call
ttm_bo_list_ref_sub() with zero put_count.
Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'drivers/gpu/drm/ttm')
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 32 | ||||
-rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 124 |
2 files changed, 129 insertions, 27 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 148a322d8f5d..a586378b1b2b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c | |||
@@ -169,7 +169,7 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) | |||
169 | } | 169 | } |
170 | EXPORT_SYMBOL(ttm_bo_wait_unreserved); | 170 | EXPORT_SYMBOL(ttm_bo_wait_unreserved); |
171 | 171 | ||
172 | static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) | 172 | void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) |
173 | { | 173 | { |
174 | struct ttm_bo_device *bdev = bo->bdev; | 174 | struct ttm_bo_device *bdev = bo->bdev; |
175 | struct ttm_mem_type_manager *man; | 175 | struct ttm_mem_type_manager *man; |
@@ -191,11 +191,7 @@ static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) | |||
191 | } | 191 | } |
192 | } | 192 | } |
193 | 193 | ||
194 | /** | 194 | int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) |
195 | * Call with the lru_lock held. | ||
196 | */ | ||
197 | |||
198 | static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) | ||
199 | { | 195 | { |
200 | int put_count = 0; | 196 | int put_count = 0; |
201 | 197 | ||
@@ -267,6 +263,15 @@ static void ttm_bo_ref_bug(struct kref *list_kref) | |||
267 | BUG(); | 263 | BUG(); |
268 | } | 264 | } |
269 | 265 | ||
266 | void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, | ||
267 | bool never_free) | ||
268 | { | ||
269 | while (count--) | ||
270 | kref_put(&bo->list_kref, | ||
271 | (never_free || (count >= 0)) ? ttm_bo_ref_bug : | ||
272 | ttm_bo_release_list); | ||
273 | } | ||
274 | |||
270 | int ttm_bo_reserve(struct ttm_buffer_object *bo, | 275 | int ttm_bo_reserve(struct ttm_buffer_object *bo, |
271 | bool interruptible, | 276 | bool interruptible, |
272 | bool no_wait, bool use_sequence, uint32_t sequence) | 277 | bool no_wait, bool use_sequence, uint32_t sequence) |
@@ -282,8 +287,7 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo, | |||
282 | put_count = ttm_bo_del_from_lru(bo); | 287 | put_count = ttm_bo_del_from_lru(bo); |
283 | spin_unlock(&glob->lru_lock); | 288 | spin_unlock(&glob->lru_lock); |
284 | 289 | ||
285 | while (put_count--) | 290 | ttm_bo_list_ref_sub(bo, put_count, true); |
286 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | ||
287 | 291 | ||
288 | return ret; | 292 | return ret; |
289 | } | 293 | } |
@@ -496,8 +500,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) | |||
496 | spin_unlock(&glob->lru_lock); | 500 | spin_unlock(&glob->lru_lock); |
497 | ttm_bo_cleanup_memtype_use(bo); | 501 | ttm_bo_cleanup_memtype_use(bo); |
498 | 502 | ||
499 | while (put_count--) | 503 | ttm_bo_list_ref_sub(bo, put_count, true); |
500 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | ||
501 | 504 | ||
502 | return; | 505 | return; |
503 | } else { | 506 | } else { |
@@ -580,8 +583,7 @@ retry: | |||
580 | spin_unlock(&glob->lru_lock); | 583 | spin_unlock(&glob->lru_lock); |
581 | ttm_bo_cleanup_memtype_use(bo); | 584 | ttm_bo_cleanup_memtype_use(bo); |
582 | 585 | ||
583 | while (put_count--) | 586 | ttm_bo_list_ref_sub(bo, put_count, true); |
584 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | ||
585 | 587 | ||
586 | return 0; | 588 | return 0; |
587 | } | 589 | } |
@@ -802,8 +804,7 @@ retry: | |||
802 | 804 | ||
803 | BUG_ON(ret != 0); | 805 | BUG_ON(ret != 0); |
804 | 806 | ||
805 | while (put_count--) | 807 | ttm_bo_list_ref_sub(bo, put_count, true); |
806 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | ||
807 | 808 | ||
808 | ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); | 809 | ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); |
809 | ttm_bo_unreserve(bo); | 810 | ttm_bo_unreserve(bo); |
@@ -1783,8 +1784,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) | |||
1783 | put_count = ttm_bo_del_from_lru(bo); | 1784 | put_count = ttm_bo_del_from_lru(bo); |
1784 | spin_unlock(&glob->lru_lock); | 1785 | spin_unlock(&glob->lru_lock); |
1785 | 1786 | ||
1786 | while (put_count--) | 1787 | ttm_bo_list_ref_sub(bo, put_count, true); |
1787 | kref_put(&bo->list_kref, ttm_bo_ref_bug); | ||
1788 | 1788 | ||
1789 | /** | 1789 | /** |
1790 | * Wait for GPU, then move to system cached. | 1790 | * Wait for GPU, then move to system cached. |
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index c285c2902d15..201a71d111ec 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c | |||
@@ -32,6 +32,72 @@ | |||
32 | #include <linux/sched.h> | 32 | #include <linux/sched.h> |
33 | #include <linux/module.h> | 33 | #include <linux/module.h> |
34 | 34 | ||
35 | static void ttm_eu_backoff_reservation_locked(struct list_head *list) | ||
36 | { | ||
37 | struct ttm_validate_buffer *entry; | ||
38 | |||
39 | list_for_each_entry(entry, list, head) { | ||
40 | struct ttm_buffer_object *bo = entry->bo; | ||
41 | if (!entry->reserved) | ||
42 | continue; | ||
43 | |||
44 | if (entry->removed) { | ||
45 | ttm_bo_add_to_lru(bo); | ||
46 | entry->removed = false; | ||
47 | |||
48 | } | ||
49 | entry->reserved = false; | ||
50 | atomic_set(&bo->reserved, 0); | ||
51 | wake_up_all(&bo->event_queue); | ||
52 | } | ||
53 | } | ||
54 | |||
55 | static void ttm_eu_del_from_lru_locked(struct list_head *list) | ||
56 | { | ||
57 | struct ttm_validate_buffer *entry; | ||
58 | |||
59 | list_for_each_entry(entry, list, head) { | ||
60 | struct ttm_buffer_object *bo = entry->bo; | ||
61 | if (!entry->reserved) | ||
62 | continue; | ||
63 | |||
64 | if (!entry->removed) { | ||
65 | entry->put_count = ttm_bo_del_from_lru(bo); | ||
66 | entry->removed = true; | ||
67 | } | ||
68 | } | ||
69 | } | ||
70 | |||
71 | static void ttm_eu_list_ref_sub(struct list_head *list) | ||
72 | { | ||
73 | struct ttm_validate_buffer *entry; | ||
74 | |||
75 | list_for_each_entry(entry, list, head) { | ||
76 | struct ttm_buffer_object *bo = entry->bo; | ||
77 | |||
78 | if (entry->put_count) { | ||
79 | ttm_bo_list_ref_sub(bo, entry->put_count, true); | ||
80 | entry->put_count = 0; | ||
81 | } | ||
82 | } | ||
83 | } | ||
84 | |||
85 | static int ttm_eu_wait_unreserved_locked(struct list_head *list, | ||
86 | struct ttm_buffer_object *bo) | ||
87 | { | ||
88 | struct ttm_bo_global *glob = bo->glob; | ||
89 | int ret; | ||
90 | |||
91 | ttm_eu_del_from_lru_locked(list); | ||
92 | spin_unlock(&glob->lru_lock); | ||
93 | ret = ttm_bo_wait_unreserved(bo, true); | ||
94 | spin_lock(&glob->lru_lock); | ||
95 | if (unlikely(ret != 0)) | ||
96 | ttm_eu_backoff_reservation_locked(list); | ||
97 | return ret; | ||
98 | } | ||
99 | |||
100 | |||
35 | void ttm_eu_backoff_reservation(struct list_head *list) | 101 | void ttm_eu_backoff_reservation(struct list_head *list) |
36 | { | 102 | { |
37 | struct ttm_validate_buffer *entry; | 103 | struct ttm_validate_buffer *entry; |
@@ -61,35 +127,71 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); | |||
61 | 127 | ||
62 | int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq) | 128 | int ttm_eu_reserve_buffers(struct list_head *list, uint32_t val_seq) |
63 | { | 129 | { |
130 | struct ttm_bo_global *glob; | ||
64 | struct ttm_validate_buffer *entry; | 131 | struct ttm_validate_buffer *entry; |
65 | int ret; | 132 | int ret; |
66 | 133 | ||
134 | if (list_empty(list)) | ||
135 | return 0; | ||
136 | |||
137 | list_for_each_entry(entry, list, head) { | ||
138 | entry->reserved = false; | ||
139 | entry->put_count = 0; | ||
140 | entry->removed = false; | ||
141 | } | ||
142 | |||
143 | entry = list_first_entry(list, struct ttm_validate_buffer, head); | ||
144 | glob = entry->bo->glob; | ||
145 | |||
67 | retry: | 146 | retry: |
147 | spin_lock(&glob->lru_lock); | ||
68 | list_for_each_entry(entry, list, head) { | 148 | list_for_each_entry(entry, list, head) { |
69 | struct ttm_buffer_object *bo = entry->bo; | 149 | struct ttm_buffer_object *bo = entry->bo; |
70 | 150 | ||
71 | entry->reserved = false; | 151 | retry_this_bo: |
72 | ret = ttm_bo_reserve(bo, true, false, true, val_seq); | 152 | ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); |
73 | if (ret != 0) { | 153 | switch (ret) { |
74 | ttm_eu_backoff_reservation(list); | 154 | case 0: |
75 | if (ret == -EAGAIN) { | 155 | break; |
76 | ret = ttm_bo_wait_unreserved(bo, true); | 156 | case -EBUSY: |
77 | if (unlikely(ret != 0)) | 157 | ret = ttm_eu_wait_unreserved_locked(list, bo); |
78 | return ret; | 158 | if (unlikely(ret != 0)) { |
79 | goto retry; | 159 | spin_unlock(&glob->lru_lock); |
80 | } else | 160 | ttm_eu_list_ref_sub(list); |
161 | return ret; | ||
162 | } | ||
163 | goto retry_this_bo; | ||
164 | case -EAGAIN: | ||
165 | ttm_eu_backoff_reservation_locked(list); | ||
166 | spin_unlock(&glob->lru_lock); | ||
167 | ttm_eu_list_ref_sub(list); | ||
168 | ret = ttm_bo_wait_unreserved(bo, true); | ||
169 | if (unlikely(ret != 0)) | ||
81 | return ret; | 170 | return ret; |
171 | goto retry; | ||
172 | default: | ||
173 | ttm_eu_backoff_reservation_locked(list); | ||
174 | spin_unlock(&glob->lru_lock); | ||
175 | ttm_eu_list_ref_sub(list); | ||
176 | return ret; | ||
82 | } | 177 | } |
83 | 178 | ||
84 | entry->reserved = true; | 179 | entry->reserved = true; |
85 | if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { | 180 | if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { |
86 | ttm_eu_backoff_reservation(list); | 181 | ttm_eu_backoff_reservation_locked(list); |
182 | spin_unlock(&glob->lru_lock); | ||
183 | ttm_eu_list_ref_sub(list); | ||
87 | ret = ttm_bo_wait_cpu(bo, false); | 184 | ret = ttm_bo_wait_cpu(bo, false); |
88 | if (ret) | 185 | if (ret) |
89 | return ret; | 186 | return ret; |
90 | goto retry; | 187 | goto retry; |
91 | } | 188 | } |
92 | } | 189 | } |
190 | |||
191 | ttm_eu_del_from_lru_locked(list); | ||
192 | spin_unlock(&glob->lru_lock); | ||
193 | ttm_eu_list_ref_sub(list); | ||
194 | |||
93 | return 0; | 195 | return 0; |
94 | } | 196 | } |
95 | EXPORT_SYMBOL(ttm_eu_reserve_buffers); | 197 | EXPORT_SYMBOL(ttm_eu_reserve_buffers); |