diff options
Diffstat (limited to 'drivers/gpu/drm')
-rw-r--r-- | drivers/gpu/drm/radeon/radeon.h | 6 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_device.c | 8 | ||||
-rw-r--r-- | drivers/gpu/drm/radeon/radeon_fence.c | 299 |
3 files changed, 119 insertions, 194 deletions
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index cdf46bc6dcc4..7c8711793421 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h | |||
@@ -263,15 +263,12 @@ struct radeon_fence_driver { | |||
263 | atomic64_t last_seq; | 263 | atomic64_t last_seq; |
264 | unsigned long last_activity; | 264 | unsigned long last_activity; |
265 | wait_queue_head_t queue; | 265 | wait_queue_head_t queue; |
266 | struct list_head emitted; | ||
267 | struct list_head signaled; | ||
268 | bool initialized; | 266 | bool initialized; |
269 | }; | 267 | }; |
270 | 268 | ||
271 | struct radeon_fence { | 269 | struct radeon_fence { |
272 | struct radeon_device *rdev; | 270 | struct radeon_device *rdev; |
273 | struct kref kref; | 271 | struct kref kref; |
274 | struct list_head list; | ||
275 | /* protected by radeon_fence.lock */ | 272 | /* protected by radeon_fence.lock */ |
276 | uint64_t seq; | 273 | uint64_t seq; |
277 | /* RB, DMA, etc. */ | 274 | /* RB, DMA, etc. */ |
@@ -291,7 +288,7 @@ int radeon_fence_wait_next(struct radeon_device *rdev, int ring); | |||
291 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring); | 288 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring); |
292 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence); | 289 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence); |
293 | void radeon_fence_unref(struct radeon_fence **fence); | 290 | void radeon_fence_unref(struct radeon_fence **fence); |
294 | int radeon_fence_count_emitted(struct radeon_device *rdev, int ring); | 291 | unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring); |
295 | 292 | ||
296 | /* | 293 | /* |
297 | * Tiling registers | 294 | * Tiling registers |
@@ -1534,7 +1531,6 @@ struct radeon_device { | |||
1534 | struct radeon_mode_info mode_info; | 1531 | struct radeon_mode_info mode_info; |
1535 | struct radeon_scratch scratch; | 1532 | struct radeon_scratch scratch; |
1536 | struct radeon_mman mman; | 1533 | struct radeon_mman mman; |
1537 | rwlock_t fence_lock; | ||
1538 | struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS]; | 1534 | struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS]; |
1539 | struct radeon_semaphore_driver semaphore_drv; | 1535 | struct radeon_semaphore_driver semaphore_drv; |
1540 | struct mutex ring_lock; | 1536 | struct mutex ring_lock; |
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 3f6ff2a0bce2..0e7b72a0ed35 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c | |||
@@ -225,9 +225,9 @@ int radeon_wb_init(struct radeon_device *rdev) | |||
225 | /* disable event_write fences */ | 225 | /* disable event_write fences */ |
226 | rdev->wb.use_event = false; | 226 | rdev->wb.use_event = false; |
227 | /* disabled via module param */ | 227 | /* disabled via module param */ |
228 | if (radeon_no_wb == 1) | 228 | if (radeon_no_wb == 1) { |
229 | rdev->wb.enabled = false; | 229 | rdev->wb.enabled = false; |
230 | else { | 230 | } else { |
231 | if (rdev->flags & RADEON_IS_AGP) { | 231 | if (rdev->flags & RADEON_IS_AGP) { |
232 | /* often unreliable on AGP */ | 232 | /* often unreliable on AGP */ |
233 | rdev->wb.enabled = false; | 233 | rdev->wb.enabled = false; |
@@ -237,8 +237,9 @@ int radeon_wb_init(struct radeon_device *rdev) | |||
237 | } else { | 237 | } else { |
238 | rdev->wb.enabled = true; | 238 | rdev->wb.enabled = true; |
239 | /* event_write fences are only available on r600+ */ | 239 | /* event_write fences are only available on r600+ */ |
240 | if (rdev->family >= CHIP_R600) | 240 | if (rdev->family >= CHIP_R600) { |
241 | rdev->wb.use_event = true; | 241 | rdev->wb.use_event = true; |
242 | } | ||
242 | } | 243 | } |
243 | } | 244 | } |
244 | /* always use writeback/events on NI, APUs */ | 245 | /* always use writeback/events on NI, APUs */ |
@@ -731,7 +732,6 @@ int radeon_device_init(struct radeon_device *rdev, | |||
731 | mutex_init(&rdev->gem.mutex); | 732 | mutex_init(&rdev->gem.mutex); |
732 | mutex_init(&rdev->pm.mutex); | 733 | mutex_init(&rdev->pm.mutex); |
733 | mutex_init(&rdev->vram_mutex); | 734 | mutex_init(&rdev->vram_mutex); |
734 | rwlock_init(&rdev->fence_lock); | ||
735 | rwlock_init(&rdev->semaphore_drv.lock); | 735 | rwlock_init(&rdev->semaphore_drv.lock); |
736 | INIT_LIST_HEAD(&rdev->gem.objects); | 736 | INIT_LIST_HEAD(&rdev->gem.objects); |
737 | init_waitqueue_head(&rdev->irq.vblank_queue); | 737 | init_waitqueue_head(&rdev->irq.vblank_queue); |
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index feb2bbc6ef6d..ed202255ac76 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c | |||
@@ -63,30 +63,18 @@ static u32 radeon_fence_read(struct radeon_device *rdev, int ring) | |||
63 | 63 | ||
64 | int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) | 64 | int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence) |
65 | { | 65 | { |
66 | unsigned long irq_flags; | 66 | /* we are protected by the ring emission mutex */ |
67 | |||
68 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
69 | if (fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { | 67 | if (fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) { |
70 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
71 | return 0; | 68 | return 0; |
72 | } | 69 | } |
73 | /* we are protected by the ring emission mutex */ | ||
74 | fence->seq = ++rdev->fence_drv[fence->ring].seq; | 70 | fence->seq = ++rdev->fence_drv[fence->ring].seq; |
75 | radeon_fence_ring_emit(rdev, fence->ring, fence); | 71 | radeon_fence_ring_emit(rdev, fence->ring, fence); |
76 | trace_radeon_fence_emit(rdev->ddev, fence->seq); | 72 | trace_radeon_fence_emit(rdev->ddev, fence->seq); |
77 | /* are we the first fence on a previusly idle ring? */ | ||
78 | if (list_empty(&rdev->fence_drv[fence->ring].emitted)) { | ||
79 | rdev->fence_drv[fence->ring].last_activity = jiffies; | ||
80 | } | ||
81 | list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted); | ||
82 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
83 | return 0; | 73 | return 0; |
84 | } | 74 | } |
85 | 75 | ||
86 | static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) | 76 | void radeon_fence_process(struct radeon_device *rdev, int ring) |
87 | { | 77 | { |
88 | struct radeon_fence *fence; | ||
89 | struct list_head *i, *n; | ||
90 | uint64_t seq, last_seq; | 78 | uint64_t seq, last_seq; |
91 | unsigned count_loop = 0; | 79 | unsigned count_loop = 0; |
92 | bool wake = false; | 80 | bool wake = false; |
@@ -120,14 +108,15 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) | |||
120 | seq += 0x100000000LL; | 108 | seq += 0x100000000LL; |
121 | } | 109 | } |
122 | 110 | ||
123 | if (!wake && seq == last_seq) { | 111 | if (seq == last_seq) { |
124 | return false; | 112 | break; |
125 | } | 113 | } |
126 | /* If we loop over we don't want to return without | 114 | /* If we loop over we don't want to return without |
127 | * checking if a fence is signaled as it means that the | 115 | * checking if a fence is signaled as it means that the |
128 | * seq we just read is different from the previous on. | 116 | * seq we just read is different from the previous on. |
129 | */ | 117 | */ |
130 | wake = true; | 118 | wake = true; |
119 | last_seq = seq; | ||
131 | if ((count_loop++) > 10) { | 120 | if ((count_loop++) > 10) { |
132 | /* We looped over too many time leave with the | 121 | /* We looped over too many time leave with the |
133 | * fact that we might have set an older fence | 122 | * fact that we might have set an older fence |
@@ -136,46 +125,20 @@ static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring) | |||
136 | */ | 125 | */ |
137 | break; | 126 | break; |
138 | } | 127 | } |
139 | last_seq = seq; | ||
140 | } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); | 128 | } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); |
141 | 129 | ||
142 | /* reset wake to false */ | 130 | if (wake) { |
143 | wake = false; | 131 | rdev->fence_drv[ring].last_activity = jiffies; |
144 | rdev->fence_drv[ring].last_activity = jiffies; | 132 | wake_up_all(&rdev->fence_drv[ring].queue); |
145 | |||
146 | n = NULL; | ||
147 | list_for_each(i, &rdev->fence_drv[ring].emitted) { | ||
148 | fence = list_entry(i, struct radeon_fence, list); | ||
149 | if (fence->seq == seq) { | ||
150 | n = i; | ||
151 | break; | ||
152 | } | ||
153 | } | ||
154 | /* all fence previous to this one are considered as signaled */ | ||
155 | if (n) { | ||
156 | i = n; | ||
157 | do { | ||
158 | n = i->prev; | ||
159 | list_move_tail(i, &rdev->fence_drv[ring].signaled); | ||
160 | fence = list_entry(i, struct radeon_fence, list); | ||
161 | fence->seq = RADEON_FENCE_SIGNALED_SEQ; | ||
162 | i = n; | ||
163 | } while (i != &rdev->fence_drv[ring].emitted); | ||
164 | wake = true; | ||
165 | } | 133 | } |
166 | return wake; | ||
167 | } | 134 | } |
168 | 135 | ||
169 | static void radeon_fence_destroy(struct kref *kref) | 136 | static void radeon_fence_destroy(struct kref *kref) |
170 | { | 137 | { |
171 | unsigned long irq_flags; | 138 | struct radeon_fence *fence; |
172 | struct radeon_fence *fence; | ||
173 | 139 | ||
174 | fence = container_of(kref, struct radeon_fence, kref); | 140 | fence = container_of(kref, struct radeon_fence, kref); |
175 | write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); | ||
176 | list_del(&fence->list); | ||
177 | fence->seq = RADEON_FENCE_NOTEMITED_SEQ; | 141 | fence->seq = RADEON_FENCE_NOTEMITED_SEQ; |
178 | write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); | ||
179 | if (fence->semaphore) | 142 | if (fence->semaphore) |
180 | radeon_semaphore_free(fence->rdev, fence->semaphore); | 143 | radeon_semaphore_free(fence->rdev, fence->semaphore); |
181 | kfree(fence); | 144 | kfree(fence); |
@@ -194,80 +157,82 @@ int radeon_fence_create(struct radeon_device *rdev, | |||
194 | (*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ; | 157 | (*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ; |
195 | (*fence)->ring = ring; | 158 | (*fence)->ring = ring; |
196 | (*fence)->semaphore = NULL; | 159 | (*fence)->semaphore = NULL; |
197 | INIT_LIST_HEAD(&(*fence)->list); | ||
198 | return 0; | 160 | return 0; |
199 | } | 161 | } |
200 | 162 | ||
201 | bool radeon_fence_signaled(struct radeon_fence *fence) | 163 | static bool radeon_fence_seq_signaled(struct radeon_device *rdev, |
164 | u64 seq, unsigned ring) | ||
202 | { | 165 | { |
203 | unsigned long irq_flags; | 166 | if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { |
204 | bool signaled = false; | 167 | return true; |
205 | 168 | } | |
206 | if (!fence) | 169 | /* poll new last sequence at least once */ |
170 | radeon_fence_process(rdev, ring); | ||
171 | if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { | ||
207 | return true; | 172 | return true; |
173 | } | ||
174 | return false; | ||
175 | } | ||
208 | 176 | ||
209 | write_lock_irqsave(&fence->rdev->fence_lock, irq_flags); | 177 | bool radeon_fence_signaled(struct radeon_fence *fence) |
210 | signaled = (fence->seq == RADEON_FENCE_SIGNALED_SEQ); | 178 | { |
211 | /* if we are shuting down report all fence as signaled */ | 179 | if (!fence) { |
212 | if (fence->rdev->shutdown) { | 180 | return true; |
213 | signaled = true; | ||
214 | } | 181 | } |
215 | if (fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { | 182 | if (fence->seq == RADEON_FENCE_NOTEMITED_SEQ) { |
216 | WARN(1, "Querying an unemitted fence : %p !\n", fence); | 183 | WARN(1, "Querying an unemitted fence : %p !\n", fence); |
217 | signaled = true; | 184 | return true; |
218 | } | 185 | } |
219 | if (!signaled) { | 186 | if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) { |
220 | radeon_fence_poll_locked(fence->rdev, fence->ring); | 187 | return true; |
221 | signaled = (fence->seq == RADEON_FENCE_SIGNALED_SEQ); | ||
222 | } | 188 | } |
223 | write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags); | 189 | if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { |
224 | return signaled; | 190 | fence->seq = RADEON_FENCE_SIGNALED_SEQ; |
191 | return true; | ||
192 | } | ||
193 | return false; | ||
225 | } | 194 | } |
226 | 195 | ||
227 | int radeon_fence_wait(struct radeon_fence *fence, bool intr) | 196 | static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq, |
197 | unsigned ring, bool intr) | ||
228 | { | 198 | { |
229 | struct radeon_device *rdev; | 199 | unsigned long timeout, last_activity; |
230 | unsigned long irq_flags, timeout, last_activity; | ||
231 | uint64_t seq; | 200 | uint64_t seq; |
232 | int i, r; | 201 | unsigned i; |
233 | bool signaled; | 202 | bool signaled; |
203 | int r; | ||
234 | 204 | ||
235 | if (fence == NULL) { | 205 | while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) { |
236 | WARN(1, "Querying an invalid fence : %p !\n", fence); | 206 | if (!rdev->ring[ring].ready) { |
237 | return -EINVAL; | 207 | return -EBUSY; |
238 | } | 208 | } |
239 | 209 | ||
240 | rdev = fence->rdev; | ||
241 | signaled = radeon_fence_signaled(fence); | ||
242 | while (!signaled) { | ||
243 | read_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
244 | timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT; | 210 | timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT; |
245 | if (time_after(rdev->fence_drv[fence->ring].last_activity, timeout)) { | 211 | if (time_after(rdev->fence_drv[ring].last_activity, timeout)) { |
246 | /* the normal case, timeout is somewhere before last_activity */ | 212 | /* the normal case, timeout is somewhere before last_activity */ |
247 | timeout = rdev->fence_drv[fence->ring].last_activity - timeout; | 213 | timeout = rdev->fence_drv[ring].last_activity - timeout; |
248 | } else { | 214 | } else { |
249 | /* either jiffies wrapped around, or no fence was signaled in the last 500ms | 215 | /* either jiffies wrapped around, or no fence was signaled in the last 500ms |
250 | * anyway we will just wait for the minimum amount and then check for a lockup */ | 216 | * anyway we will just wait for the minimum amount and then check for a lockup |
217 | */ | ||
251 | timeout = 1; | 218 | timeout = 1; |
252 | } | 219 | } |
253 | /* save current sequence value used to check for GPU lockups */ | 220 | seq = atomic64_read(&rdev->fence_drv[ring].last_seq); |
254 | seq = atomic64_read(&rdev->fence_drv[fence->ring].last_seq); | ||
255 | /* Save current last activity valuee, used to check for GPU lockups */ | 221 | /* Save current last activity valuee, used to check for GPU lockups */ |
256 | last_activity = rdev->fence_drv[fence->ring].last_activity; | 222 | last_activity = rdev->fence_drv[ring].last_activity; |
257 | read_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
258 | 223 | ||
259 | trace_radeon_fence_wait_begin(rdev->ddev, seq); | 224 | trace_radeon_fence_wait_begin(rdev->ddev, seq); |
260 | radeon_irq_kms_sw_irq_get(rdev, fence->ring); | 225 | radeon_irq_kms_sw_irq_get(rdev, ring); |
261 | if (intr) { | 226 | if (intr) { |
262 | r = wait_event_interruptible_timeout( | 227 | r = wait_event_interruptible_timeout(rdev->fence_drv[ring].queue, |
263 | rdev->fence_drv[fence->ring].queue, | 228 | (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)), |
264 | (signaled = radeon_fence_signaled(fence)), timeout); | 229 | timeout); |
265 | } else { | 230 | } else { |
266 | r = wait_event_timeout( | 231 | r = wait_event_timeout(rdev->fence_drv[ring].queue, |
267 | rdev->fence_drv[fence->ring].queue, | 232 | (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)), |
268 | (signaled = radeon_fence_signaled(fence)), timeout); | 233 | timeout); |
269 | } | 234 | } |
270 | radeon_irq_kms_sw_irq_put(rdev, fence->ring); | 235 | radeon_irq_kms_sw_irq_put(rdev, ring); |
271 | if (unlikely(r < 0)) { | 236 | if (unlikely(r < 0)) { |
272 | return r; | 237 | return r; |
273 | } | 238 | } |
@@ -280,19 +245,24 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) | |||
280 | continue; | 245 | continue; |
281 | } | 246 | } |
282 | 247 | ||
283 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | 248 | /* check if sequence value has changed since last_activity */ |
249 | if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) { | ||
250 | continue; | ||
251 | } | ||
284 | /* test if somebody else has already decided that this is a lockup */ | 252 | /* test if somebody else has already decided that this is a lockup */ |
285 | if (last_activity != rdev->fence_drv[fence->ring].last_activity) { | 253 | if (last_activity != rdev->fence_drv[ring].last_activity) { |
286 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
287 | continue; | 254 | continue; |
288 | } | 255 | } |
289 | 256 | ||
290 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 257 | if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { |
291 | |||
292 | if (radeon_ring_is_lockup(rdev, fence->ring, &rdev->ring[fence->ring])) { | ||
293 | /* good news we believe it's a lockup */ | 258 | /* good news we believe it's a lockup */ |
294 | dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", | 259 | dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n", |
295 | fence->seq, seq); | 260 | target_seq, seq); |
261 | |||
262 | /* change last activity so nobody else think there is a lockup */ | ||
263 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { | ||
264 | rdev->fence_drv[i].last_activity = jiffies; | ||
265 | } | ||
296 | 266 | ||
297 | /* change last activity so nobody else think there is a lockup */ | 267 | /* change last activity so nobody else think there is a lockup */ |
298 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { | 268 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
@@ -300,7 +270,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) | |||
300 | } | 270 | } |
301 | 271 | ||
302 | /* mark the ring as not ready any more */ | 272 | /* mark the ring as not ready any more */ |
303 | rdev->ring[fence->ring].ready = false; | 273 | rdev->ring[ring].ready = false; |
304 | return -EDEADLK; | 274 | return -EDEADLK; |
305 | } | 275 | } |
306 | } | 276 | } |
@@ -308,52 +278,47 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) | |||
308 | return 0; | 278 | return 0; |
309 | } | 279 | } |
310 | 280 | ||
311 | int radeon_fence_wait_next(struct radeon_device *rdev, int ring) | 281 | int radeon_fence_wait(struct radeon_fence *fence, bool intr) |
312 | { | 282 | { |
313 | unsigned long irq_flags; | ||
314 | struct radeon_fence *fence; | ||
315 | int r; | 283 | int r; |
316 | 284 | ||
317 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | 285 | if (fence == NULL) { |
318 | if (!rdev->ring[ring].ready) { | 286 | WARN(1, "Querying an invalid fence : %p !\n", fence); |
319 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 287 | return -EINVAL; |
320 | return -EBUSY; | ||
321 | } | 288 | } |
322 | if (list_empty(&rdev->fence_drv[ring].emitted)) { | 289 | |
323 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 290 | r = radeon_fence_wait_seq(fence->rdev, fence->seq, fence->ring, intr); |
324 | return -ENOENT; | 291 | if (r) { |
292 | return r; | ||
325 | } | 293 | } |
326 | fence = list_entry(rdev->fence_drv[ring].emitted.next, | 294 | fence->seq = RADEON_FENCE_SIGNALED_SEQ; |
327 | struct radeon_fence, list); | 295 | return 0; |
328 | radeon_fence_ref(fence); | ||
329 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
330 | r = radeon_fence_wait(fence, false); | ||
331 | radeon_fence_unref(&fence); | ||
332 | return r; | ||
333 | } | 296 | } |
334 | 297 | ||
335 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) | 298 | int radeon_fence_wait_next(struct radeon_device *rdev, int ring) |
336 | { | 299 | { |
337 | unsigned long irq_flags; | 300 | uint64_t seq; |
338 | struct radeon_fence *fence; | ||
339 | int r; | ||
340 | 301 | ||
341 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | 302 | /* We are not protected by ring lock when reading current seq but |
342 | if (!rdev->ring[ring].ready) { | 303 | * it's ok as worst case is we return to early while we could have |
343 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 304 | * wait. |
344 | return -EBUSY; | 305 | */ |
345 | } | 306 | seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; |
346 | if (list_empty(&rdev->fence_drv[ring].emitted)) { | 307 | if (seq >= rdev->fence_drv[ring].seq) { |
347 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 308 | /* nothing to wait for, last_seq is already the last emited fence */ |
348 | return 0; | 309 | return 0; |
349 | } | 310 | } |
350 | fence = list_entry(rdev->fence_drv[ring].emitted.prev, | 311 | return radeon_fence_wait_seq(rdev, seq, ring, false); |
351 | struct radeon_fence, list); | 312 | } |
352 | radeon_fence_ref(fence); | 313 | |
353 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 314 | int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) |
354 | r = radeon_fence_wait(fence, false); | 315 | { |
355 | radeon_fence_unref(&fence); | 316 | /* We are not protected by ring lock when reading current seq |
356 | return r; | 317 | * but it's ok as wait empty is call from place where no more |
318 | * activity can be scheduled so there won't be concurrent access | ||
319 | * to seq value. | ||
320 | */ | ||
321 | return radeon_fence_wait_seq(rdev, rdev->fence_drv[ring].seq, ring, false); | ||
357 | } | 322 | } |
358 | 323 | ||
359 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) | 324 | struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) |
@@ -372,49 +337,27 @@ void radeon_fence_unref(struct radeon_fence **fence) | |||
372 | } | 337 | } |
373 | } | 338 | } |
374 | 339 | ||
375 | void radeon_fence_process(struct radeon_device *rdev, int ring) | 340 | unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) |
376 | { | 341 | { |
377 | unsigned long irq_flags; | 342 | uint64_t emitted; |
378 | bool wake; | ||
379 | |||
380 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
381 | wake = radeon_fence_poll_locked(rdev, ring); | ||
382 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
383 | if (wake) { | ||
384 | wake_up_all(&rdev->fence_drv[ring].queue); | ||
385 | } | ||
386 | } | ||
387 | 343 | ||
388 | int radeon_fence_count_emitted(struct radeon_device *rdev, int ring) | 344 | radeon_fence_process(rdev, ring); |
389 | { | 345 | /* We are not protected by ring lock when reading the last sequence |
390 | unsigned long irq_flags; | 346 | * but it's ok to report slightly wrong fence count here. |
391 | int not_processed = 0; | 347 | */ |
392 | 348 | emitted = rdev->fence_drv[ring].seq - atomic64_read(&rdev->fence_drv[ring].last_seq); | |
393 | read_lock_irqsave(&rdev->fence_lock, irq_flags); | 349 | /* to avoid 32bits warp around */ |
394 | if (!rdev->fence_drv[ring].initialized) { | 350 | if (emitted > 0x10000000) { |
395 | read_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 351 | emitted = 0x10000000; |
396 | return 0; | ||
397 | } | ||
398 | |||
399 | if (!list_empty(&rdev->fence_drv[ring].emitted)) { | ||
400 | struct list_head *ptr; | ||
401 | list_for_each(ptr, &rdev->fence_drv[ring].emitted) { | ||
402 | /* count up to 3, that's enought info */ | ||
403 | if (++not_processed >= 3) | ||
404 | break; | ||
405 | } | ||
406 | } | 352 | } |
407 | read_unlock_irqrestore(&rdev->fence_lock, irq_flags); | 353 | return (unsigned)emitted; |
408 | return not_processed; | ||
409 | } | 354 | } |
410 | 355 | ||
411 | int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) | 356 | int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) |
412 | { | 357 | { |
413 | unsigned long irq_flags; | ||
414 | uint64_t index; | 358 | uint64_t index; |
415 | int r; | 359 | int r; |
416 | 360 | ||
417 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
418 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); | 361 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); |
419 | if (rdev->wb.use_event) { | 362 | if (rdev->wb.use_event) { |
420 | rdev->fence_drv[ring].scratch_reg = 0; | 363 | rdev->fence_drv[ring].scratch_reg = 0; |
@@ -423,7 +366,6 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) | |||
423 | r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); | 366 | r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); |
424 | if (r) { | 367 | if (r) { |
425 | dev_err(rdev->dev, "fence failed to get scratch register\n"); | 368 | dev_err(rdev->dev, "fence failed to get scratch register\n"); |
426 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
427 | return r; | 369 | return r; |
428 | } | 370 | } |
429 | index = RADEON_WB_SCRATCH_OFFSET + | 371 | index = RADEON_WB_SCRATCH_OFFSET + |
@@ -434,9 +376,8 @@ int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) | |||
434 | rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; | 376 | rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; |
435 | radeon_fence_write(rdev, rdev->fence_drv[ring].seq, ring); | 377 | radeon_fence_write(rdev, rdev->fence_drv[ring].seq, ring); |
436 | rdev->fence_drv[ring].initialized = true; | 378 | rdev->fence_drv[ring].initialized = true; |
437 | DRM_INFO("fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", | 379 | dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n", |
438 | ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); | 380 | ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); |
439 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
440 | return 0; | 381 | return 0; |
441 | } | 382 | } |
442 | 383 | ||
@@ -447,22 +388,18 @@ static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) | |||
447 | rdev->fence_drv[ring].gpu_addr = 0; | 388 | rdev->fence_drv[ring].gpu_addr = 0; |
448 | rdev->fence_drv[ring].seq = 0; | 389 | rdev->fence_drv[ring].seq = 0; |
449 | atomic64_set(&rdev->fence_drv[ring].last_seq, 0); | 390 | atomic64_set(&rdev->fence_drv[ring].last_seq, 0); |
450 | INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted); | 391 | rdev->fence_drv[ring].last_activity = jiffies; |
451 | INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled); | ||
452 | init_waitqueue_head(&rdev->fence_drv[ring].queue); | 392 | init_waitqueue_head(&rdev->fence_drv[ring].queue); |
453 | rdev->fence_drv[ring].initialized = false; | 393 | rdev->fence_drv[ring].initialized = false; |
454 | } | 394 | } |
455 | 395 | ||
456 | int radeon_fence_driver_init(struct radeon_device *rdev) | 396 | int radeon_fence_driver_init(struct radeon_device *rdev) |
457 | { | 397 | { |
458 | unsigned long irq_flags; | ||
459 | int ring; | 398 | int ring; |
460 | 399 | ||
461 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
462 | for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { | 400 | for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { |
463 | radeon_fence_driver_init_ring(rdev, ring); | 401 | radeon_fence_driver_init_ring(rdev, ring); |
464 | } | 402 | } |
465 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
466 | if (radeon_debugfs_fence_init(rdev)) { | 403 | if (radeon_debugfs_fence_init(rdev)) { |
467 | dev_err(rdev->dev, "fence debugfs file creation failed\n"); | 404 | dev_err(rdev->dev, "fence debugfs file creation failed\n"); |
468 | } | 405 | } |
@@ -471,7 +408,6 @@ int radeon_fence_driver_init(struct radeon_device *rdev) | |||
471 | 408 | ||
472 | void radeon_fence_driver_fini(struct radeon_device *rdev) | 409 | void radeon_fence_driver_fini(struct radeon_device *rdev) |
473 | { | 410 | { |
474 | unsigned long irq_flags; | ||
475 | int ring; | 411 | int ring; |
476 | 412 | ||
477 | for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { | 413 | for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { |
@@ -479,9 +415,7 @@ void radeon_fence_driver_fini(struct radeon_device *rdev) | |||
479 | continue; | 415 | continue; |
480 | radeon_fence_wait_empty(rdev, ring); | 416 | radeon_fence_wait_empty(rdev, ring); |
481 | wake_up_all(&rdev->fence_drv[ring].queue); | 417 | wake_up_all(&rdev->fence_drv[ring].queue); |
482 | write_lock_irqsave(&rdev->fence_lock, irq_flags); | ||
483 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); | 418 | radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); |
484 | write_unlock_irqrestore(&rdev->fence_lock, irq_flags); | ||
485 | rdev->fence_drv[ring].initialized = false; | 419 | rdev->fence_drv[ring].initialized = false; |
486 | } | 420 | } |
487 | } | 421 | } |
@@ -496,7 +430,6 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) | |||
496 | struct drm_info_node *node = (struct drm_info_node *)m->private; | 430 | struct drm_info_node *node = (struct drm_info_node *)m->private; |
497 | struct drm_device *dev = node->minor->dev; | 431 | struct drm_device *dev = node->minor->dev; |
498 | struct radeon_device *rdev = dev->dev_private; | 432 | struct radeon_device *rdev = dev->dev_private; |
499 | struct radeon_fence *fence; | ||
500 | int i; | 433 | int i; |
501 | 434 | ||
502 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { | 435 | for (i = 0; i < RADEON_NUM_RINGS; ++i) { |
@@ -506,12 +439,8 @@ static int radeon_debugfs_fence_info(struct seq_file *m, void *data) | |||
506 | seq_printf(m, "--- ring %d ---\n", i); | 439 | seq_printf(m, "--- ring %d ---\n", i); |
507 | seq_printf(m, "Last signaled fence 0x%016lx\n", | 440 | seq_printf(m, "Last signaled fence 0x%016lx\n", |
508 | atomic64_read(&rdev->fence_drv[i].last_seq)); | 441 | atomic64_read(&rdev->fence_drv[i].last_seq)); |
509 | if (!list_empty(&rdev->fence_drv[i].emitted)) { | 442 | seq_printf(m, "Last emitted 0x%016llx\n", |
510 | fence = list_entry(rdev->fence_drv[i].emitted.prev, | 443 | rdev->fence_drv[i].seq); |
511 | struct radeon_fence, list); | ||
512 | seq_printf(m, "Last emitted fence %p with 0x%016llx\n", | ||
513 | fence, fence->seq); | ||
514 | } | ||
515 | } | 444 | } |
516 | return 0; | 445 | return 0; |
517 | } | 446 | } |