diff options
author | Debarshi Dutta <ddutta@nvidia.com> | 2017-08-03 06:04:44 -0400 |
---|---|---|
committer | mobile promotions <svcmobile_promotions@nvidia.com> | 2017-08-17 17:26:47 -0400 |
commit | 98186ec2c2127c2af65a34f9e697e04f518a79ab (patch) | |
tree | 08ad87f3bf8c739e96b36f01728a8f7a30749a0e /drivers/gpu/nvgpu | |
parent | 49dc335cfe588179cbb42d8bab53bc76ba88b28f (diff) |
gpu: nvgpu: Add wrapper over atomic_t and atomic64_t
- added wrapper structs nvgpu_atomic_t and nvgpu_atomic64_t over
atomic_t and atomic64_t
- added nvgpu_atomic_* and nvgpu_atomic64_* APIs to access the above
wrappers.
JIRA NVGPU-121
Change-Id: I61667bb0a84c2fc475365abb79bffb42b8b4786a
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1533044
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
GVS: Gerrit_Virtual_Submit
Diffstat (limited to 'drivers/gpu/nvgpu')
24 files changed, 355 insertions, 120 deletions
diff --git a/drivers/gpu/nvgpu/clk/clk_arb.c b/drivers/gpu/nvgpu/clk/clk_arb.c index 3caa5409..b00ecd31 100644 --- a/drivers/gpu/nvgpu/clk/clk_arb.c +++ b/drivers/gpu/nvgpu/clk/clk_arb.c | |||
@@ -108,8 +108,8 @@ struct nvgpu_clk_notification { | |||
108 | 108 | ||
109 | struct nvgpu_clk_notification_queue { | 109 | struct nvgpu_clk_notification_queue { |
110 | u32 size; | 110 | u32 size; |
111 | atomic_t head; | 111 | nvgpu_atomic_t head; |
112 | atomic_t tail; | 112 | nvgpu_atomic_t tail; |
113 | struct nvgpu_clk_notification *notifications; | 113 | struct nvgpu_clk_notification *notifications; |
114 | }; | 114 | }; |
115 | 115 | ||
@@ -183,13 +183,13 @@ struct nvgpu_clk_arb { | |||
183 | u32 vf_table_index; | 183 | u32 vf_table_index; |
184 | 184 | ||
185 | u16 *mclk_f_points; | 185 | u16 *mclk_f_points; |
186 | atomic_t req_nr; | 186 | nvgpu_atomic_t req_nr; |
187 | 187 | ||
188 | u32 mclk_f_numpoints; | 188 | u32 mclk_f_numpoints; |
189 | u16 *gpc2clk_f_points; | 189 | u16 *gpc2clk_f_points; |
190 | u32 gpc2clk_f_numpoints; | 190 | u32 gpc2clk_f_numpoints; |
191 | 191 | ||
192 | atomic64_t alarm_mask; | 192 | nvgpu_atomic64_t alarm_mask; |
193 | struct nvgpu_clk_notification_queue notification_queue; | 193 | struct nvgpu_clk_notification_queue notification_queue; |
194 | 194 | ||
195 | #ifdef CONFIG_DEBUG_FS | 195 | #ifdef CONFIG_DEBUG_FS |
@@ -206,11 +206,11 @@ struct nvgpu_clk_dev { | |||
206 | struct llist_node node; | 206 | struct llist_node node; |
207 | }; | 207 | }; |
208 | wait_queue_head_t readout_wq; | 208 | wait_queue_head_t readout_wq; |
209 | atomic_t poll_mask; | 209 | nvgpu_atomic_t poll_mask; |
210 | u16 gpc2clk_target_mhz; | 210 | u16 gpc2clk_target_mhz; |
211 | u16 mclk_target_mhz; | 211 | u16 mclk_target_mhz; |
212 | u32 alarms_reported; | 212 | u32 alarms_reported; |
213 | atomic_t enabled_mask; | 213 | nvgpu_atomic_t enabled_mask; |
214 | struct nvgpu_clk_notification_queue queue; | 214 | struct nvgpu_clk_notification_queue queue; |
215 | u32 arb_queue_head; | 215 | u32 arb_queue_head; |
216 | struct kref refcount; | 216 | struct kref refcount; |
@@ -253,8 +253,8 @@ static int nvgpu_clk_notification_queue_alloc(struct gk20a *g, | |||
253 | return -ENOMEM; | 253 | return -ENOMEM; |
254 | queue->size = events_number; | 254 | queue->size = events_number; |
255 | 255 | ||
256 | atomic_set(&queue->head, 0); | 256 | nvgpu_atomic_set(&queue->head, 0); |
257 | atomic_set(&queue->tail, 0); | 257 | nvgpu_atomic_set(&queue->tail, 0); |
258 | 258 | ||
259 | return 0; | 259 | return 0; |
260 | } | 260 | } |
@@ -263,8 +263,8 @@ static void nvgpu_clk_notification_queue_free(struct gk20a *g, | |||
263 | struct nvgpu_clk_notification_queue *queue) { | 263 | struct nvgpu_clk_notification_queue *queue) { |
264 | nvgpu_kfree(g, queue->notifications); | 264 | nvgpu_kfree(g, queue->notifications); |
265 | queue->size = 0; | 265 | queue->size = 0; |
266 | atomic_set(&queue->head, 0); | 266 | nvgpu_atomic_set(&queue->head, 0); |
267 | atomic_set(&queue->tail, 0); | 267 | nvgpu_atomic_set(&queue->tail, 0); |
268 | } | 268 | } |
269 | 269 | ||
270 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | 270 | int nvgpu_clk_arb_init_arbiter(struct gk20a *g) |
@@ -346,9 +346,9 @@ int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | |||
346 | 346 | ||
347 | arb->actual = &arb->actual_pool[0]; | 347 | arb->actual = &arb->actual_pool[0]; |
348 | 348 | ||
349 | atomic_set(&arb->req_nr, 0); | 349 | nvgpu_atomic_set(&arb->req_nr, 0); |
350 | 350 | ||
351 | atomic64_set(&arb->alarm_mask, 0); | 351 | nvgpu_atomic64_set(&arb->alarm_mask, 0); |
352 | err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue, | 352 | err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue, |
353 | DEFAULT_EVENT_NUMBER); | 353 | DEFAULT_EVENT_NUMBER); |
354 | if (err < 0) | 354 | if (err < 0) |
@@ -388,8 +388,8 @@ int nvgpu_clk_arb_init_arbiter(struct gk20a *g) | |||
388 | /* Check that first run is completed */ | 388 | /* Check that first run is completed */ |
389 | smp_mb(); | 389 | smp_mb(); |
390 | wait_event_interruptible(arb->request_wq, | 390 | wait_event_interruptible(arb->request_wq, |
391 | atomic_read(&arb->req_nr)); | 391 | nvgpu_atomic_read(&arb->req_nr)); |
392 | } while (!atomic_read(&arb->req_nr)); | 392 | } while (!nvgpu_atomic_read(&arb->req_nr)); |
393 | 393 | ||
394 | 394 | ||
395 | return arb->status; | 395 | return arb->status; |
@@ -430,7 +430,7 @@ static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) | |||
430 | u64 new_mask; | 430 | u64 new_mask; |
431 | 431 | ||
432 | do { | 432 | do { |
433 | current_mask = atomic64_read(&arb->alarm_mask); | 433 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); |
434 | /* atomic operations are strong so they do not need masks */ | 434 | /* atomic operations are strong so they do not need masks */ |
435 | 435 | ||
436 | refcnt = ((u32) (current_mask >> 32)) + 1; | 436 | refcnt = ((u32) (current_mask >> 32)) + 1; |
@@ -438,7 +438,7 @@ static void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm) | |||
438 | new_mask = ((u64) refcnt << 32) | alarm_mask; | 438 | new_mask = ((u64) refcnt << 32) | alarm_mask; |
439 | 439 | ||
440 | } while (unlikely(current_mask != | 440 | } while (unlikely(current_mask != |
441 | (u64)atomic64_cmpxchg(&arb->alarm_mask, | 441 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, |
442 | current_mask, new_mask))); | 442 | current_mask, new_mask))); |
443 | } | 443 | } |
444 | 444 | ||
@@ -452,7 +452,7 @@ static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) | |||
452 | u64 new_mask; | 452 | u64 new_mask; |
453 | 453 | ||
454 | do { | 454 | do { |
455 | current_mask = atomic64_read(&arb->alarm_mask); | 455 | current_mask = nvgpu_atomic64_read(&arb->alarm_mask); |
456 | /* atomic operations are strong so they do not need masks */ | 456 | /* atomic operations are strong so they do not need masks */ |
457 | 457 | ||
458 | refcnt = ((u32) (current_mask >> 32)) + 1; | 458 | refcnt = ((u32) (current_mask >> 32)) + 1; |
@@ -460,7 +460,7 @@ static void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm) | |||
460 | new_mask = ((u64) refcnt << 32) | alarm_mask; | 460 | new_mask = ((u64) refcnt << 32) | alarm_mask; |
461 | 461 | ||
462 | } while (unlikely(current_mask != | 462 | } while (unlikely(current_mask != |
463 | (u64)atomic64_cmpxchg(&arb->alarm_mask, | 463 | (u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask, |
464 | current_mask, new_mask))); | 464 | current_mask, new_mask))); |
465 | 465 | ||
466 | nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); | 466 | nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm); |
@@ -537,7 +537,7 @@ static int nvgpu_clk_arb_install_fd(struct gk20a *g, | |||
537 | 537 | ||
538 | init_waitqueue_head(&dev->readout_wq); | 538 | init_waitqueue_head(&dev->readout_wq); |
539 | 539 | ||
540 | atomic_set(&dev->poll_mask, 0); | 540 | nvgpu_atomic_set(&dev->poll_mask, 0); |
541 | 541 | ||
542 | dev->session = session; | 542 | dev->session = session; |
543 | kref_init(&dev->refcount); | 543 | kref_init(&dev->refcount); |
@@ -657,11 +657,11 @@ int nvgpu_clk_arb_install_event_fd(struct gk20a *g, | |||
657 | * updated | 657 | * updated |
658 | */ | 658 | */ |
659 | if (alarm_mask) | 659 | if (alarm_mask) |
660 | atomic_set(&dev->enabled_mask, alarm_mask); | 660 | nvgpu_atomic_set(&dev->enabled_mask, alarm_mask); |
661 | else | 661 | else |
662 | atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE)); | 662 | nvgpu_atomic_set(&dev->enabled_mask, EVENT(VF_UPDATE)); |
663 | 663 | ||
664 | dev->arb_queue_head = atomic_read(&arb->notification_queue.head); | 664 | dev->arb_queue_head = nvgpu_atomic_read(&arb->notification_queue.head); |
665 | 665 | ||
666 | nvgpu_spinlock_acquire(&arb->users_lock); | 666 | nvgpu_spinlock_acquire(&arb->users_lock); |
667 | list_add_tail_rcu(&dev->link, &arb->users); | 667 | list_add_tail_rcu(&dev->link, &arb->users); |
@@ -1056,7 +1056,7 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
1056 | gk20a_dbg_fn(""); | 1056 | gk20a_dbg_fn(""); |
1057 | 1057 | ||
1058 | /* bail out if gpu is down */ | 1058 | /* bail out if gpu is down */ |
1059 | if (atomic_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) | 1059 | if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) |
1060 | goto exit_arb; | 1060 | goto exit_arb; |
1061 | 1061 | ||
1062 | #ifdef CONFIG_DEBUG_FS | 1062 | #ifdef CONFIG_DEBUG_FS |
@@ -1247,7 +1247,7 @@ static void nvgpu_clk_arb_run_arbiter_cb(struct work_struct *work) | |||
1247 | 1247 | ||
1248 | /* status must be visible before atomic inc */ | 1248 | /* status must be visible before atomic inc */ |
1249 | smp_wmb(); | 1249 | smp_wmb(); |
1250 | atomic_inc(&arb->req_nr); | 1250 | nvgpu_atomic_inc(&arb->req_nr); |
1251 | 1251 | ||
1252 | /* Unlock pstate change for PG */ | 1252 | /* Unlock pstate change for PG */ |
1253 | nvgpu_mutex_release(&arb->pstate_lock); | 1253 | nvgpu_mutex_release(&arb->pstate_lock); |
@@ -1298,17 +1298,17 @@ exit_arb: | |||
1298 | EVENT(ALARM_CLOCK_ARBITER_FAILED)); | 1298 | EVENT(ALARM_CLOCK_ARBITER_FAILED)); |
1299 | } | 1299 | } |
1300 | 1300 | ||
1301 | current_alarm = (u32) atomic64_read(&arb->alarm_mask); | 1301 | current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask); |
1302 | /* notify completion for all requests */ | 1302 | /* notify completion for all requests */ |
1303 | head = llist_del_all(&arb->requests); | 1303 | head = llist_del_all(&arb->requests); |
1304 | llist_for_each_entry_safe(dev, tmp, head, node) { | 1304 | llist_for_each_entry_safe(dev, tmp, head, node) { |
1305 | atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM); | 1305 | nvgpu_atomic_set(&dev->poll_mask, POLLIN | POLLRDNORM); |
1306 | wake_up_interruptible(&dev->readout_wq); | 1306 | wake_up_interruptible(&dev->readout_wq); |
1307 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); | 1307 | kref_put(&dev->refcount, nvgpu_clk_arb_free_fd); |
1308 | } | 1308 | } |
1309 | 1309 | ||
1310 | atomic_set(&arb->notification_queue.head, | 1310 | nvgpu_atomic_set(&arb->notification_queue.head, |
1311 | atomic_read(&arb->notification_queue.tail)); | 1311 | nvgpu_atomic_read(&arb->notification_queue.tail)); |
1312 | /* notify event for all users */ | 1312 | /* notify event for all users */ |
1313 | rcu_read_lock(); | 1313 | rcu_read_lock(); |
1314 | list_for_each_entry_rcu(dev, &arb->users, link) { | 1314 | list_for_each_entry_rcu(dev, &arb->users, link) { |
@@ -1329,7 +1329,7 @@ static void nvgpu_clk_arb_queue_notification(struct gk20a *g, | |||
1329 | u32 queue_index; | 1329 | u32 queue_index; |
1330 | u64 timestamp; | 1330 | u64 timestamp; |
1331 | 1331 | ||
1332 | queue_index = (atomic_inc_return(&queue->tail)) % queue->size; | 1332 | queue_index = (nvgpu_atomic_inc_return(&queue->tail)) % queue->size; |
1333 | /* get current timestamp */ | 1333 | /* get current timestamp */ |
1334 | timestamp = (u64) sched_clock(); | 1334 | timestamp = (u64) sched_clock(); |
1335 | 1335 | ||
@@ -1355,14 +1355,14 @@ static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | |||
1355 | size_t size; | 1355 | size_t size; |
1356 | int index; | 1356 | int index; |
1357 | 1357 | ||
1358 | enabled_mask = atomic_read(&dev->enabled_mask); | 1358 | enabled_mask = nvgpu_atomic_read(&dev->enabled_mask); |
1359 | size = arb->notification_queue.size; | 1359 | size = arb->notification_queue.size; |
1360 | 1360 | ||
1361 | /* queue global arbiter notifications in buffer */ | 1361 | /* queue global arbiter notifications in buffer */ |
1362 | do { | 1362 | do { |
1363 | tail = atomic_read(&arb->notification_queue.tail); | 1363 | tail = nvgpu_atomic_read(&arb->notification_queue.tail); |
1364 | /* copy items to the queue */ | 1364 | /* copy items to the queue */ |
1365 | queue_index = atomic_read(&dev->queue.tail); | 1365 | queue_index = nvgpu_atomic_read(&dev->queue.tail); |
1366 | head = dev->arb_queue_head; | 1366 | head = dev->arb_queue_head; |
1367 | head = (tail - head) < arb->notification_queue.size ? | 1367 | head = (tail - head) < arb->notification_queue.size ? |
1368 | head : tail - arb->notification_queue.size; | 1368 | head : tail - arb->notification_queue.size; |
@@ -1389,10 +1389,10 @@ static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | |||
1389 | 1389 | ||
1390 | queue_alarm_mask |= alarm_detected; | 1390 | queue_alarm_mask |= alarm_detected; |
1391 | } | 1391 | } |
1392 | } while (unlikely(atomic_read(&arb->notification_queue.tail) != | 1392 | } while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) != |
1393 | (int)tail)); | 1393 | (int)tail)); |
1394 | 1394 | ||
1395 | atomic_set(&dev->queue.tail, queue_index); | 1395 | nvgpu_atomic_set(&dev->queue.tail, queue_index); |
1396 | /* update the last notification we processed from global queue */ | 1396 | /* update the last notification we processed from global queue */ |
1397 | 1397 | ||
1398 | dev->arb_queue_head = tail; | 1398 | dev->arb_queue_head = tail; |
@@ -1429,7 +1429,7 @@ static u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, | |||
1429 | } | 1429 | } |
1430 | 1430 | ||
1431 | if (poll_mask) { | 1431 | if (poll_mask) { |
1432 | atomic_set(&dev->poll_mask, poll_mask); | 1432 | nvgpu_atomic_set(&dev->poll_mask, poll_mask); |
1433 | wake_up_interruptible_all(&dev->readout_wq); | 1433 | wake_up_interruptible_all(&dev->readout_wq); |
1434 | } | 1434 | } |
1435 | 1435 | ||
@@ -1454,7 +1454,7 @@ static int nvgpu_clk_arb_set_event_filter(struct nvgpu_clk_dev *dev, | |||
1454 | return -EFAULT; | 1454 | return -EFAULT; |
1455 | 1455 | ||
1456 | /* update alarm mask */ | 1456 | /* update alarm mask */ |
1457 | atomic_set(&dev->enabled_mask, mask); | 1457 | nvgpu_atomic_set(&dev->enabled_mask, mask); |
1458 | 1458 | ||
1459 | return 0; | 1459 | return 0; |
1460 | } | 1460 | } |
@@ -1539,8 +1539,8 @@ static inline u32 __pending_event(struct nvgpu_clk_dev *dev, | |||
1539 | u32 events = 0; | 1539 | u32 events = 0; |
1540 | struct nvgpu_clk_notification *p_notif; | 1540 | struct nvgpu_clk_notification *p_notif; |
1541 | 1541 | ||
1542 | tail = atomic_read(&dev->queue.tail); | 1542 | tail = nvgpu_atomic_read(&dev->queue.tail); |
1543 | head = atomic_read(&dev->queue.head); | 1543 | head = nvgpu_atomic_read(&dev->queue.head); |
1544 | 1544 | ||
1545 | head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size; | 1545 | head = (tail - head) < dev->queue.size ? head : tail - dev->queue.size; |
1546 | 1546 | ||
@@ -1550,7 +1550,7 @@ static inline u32 __pending_event(struct nvgpu_clk_dev *dev, | |||
1550 | events |= p_notif->notification; | 1550 | events |= p_notif->notification; |
1551 | info->event_id = ffs(events) - 1; | 1551 | info->event_id = ffs(events) - 1; |
1552 | info->timestamp = p_notif->timestamp; | 1552 | info->timestamp = p_notif->timestamp; |
1553 | atomic_set(&dev->queue.head, head); | 1553 | nvgpu_atomic_set(&dev->queue.head, head); |
1554 | } | 1554 | } |
1555 | 1555 | ||
1556 | return events; | 1556 | return events; |
@@ -1594,7 +1594,7 @@ static unsigned int nvgpu_clk_arb_poll_dev(struct file *filp, poll_table *wait) | |||
1594 | gk20a_dbg_fn(""); | 1594 | gk20a_dbg_fn(""); |
1595 | 1595 | ||
1596 | poll_wait(filp, &dev->readout_wq, wait); | 1596 | poll_wait(filp, &dev->readout_wq, wait); |
1597 | return atomic_xchg(&dev->poll_mask, 0); | 1597 | return nvgpu_atomic_xchg(&dev->poll_mask, 0); |
1598 | } | 1598 | } |
1599 | 1599 | ||
1600 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, | 1600 | static int nvgpu_clk_arb_release_completion_dev(struct inode *inode, |
diff --git a/drivers/gpu/nvgpu/common/linux/debug_fifo.c b/drivers/gpu/nvgpu/common/linux/debug_fifo.c index a240a138..1763eb7e 100644 --- a/drivers/gpu/nvgpu/common/linux/debug_fifo.c +++ b/drivers/gpu/nvgpu/common/linux/debug_fifo.c | |||
@@ -167,7 +167,7 @@ static int gk20a_fifo_profile_enable(void *data, u64 val) | |||
167 | } | 167 | } |
168 | kref_init(&f->profile.ref); | 168 | kref_init(&f->profile.ref); |
169 | } | 169 | } |
170 | atomic_set(&f->profile.get, 0); | 170 | atomic_set(&f->profile.get.atomic_var, 0); |
171 | f->profile.enabled = true; | 171 | f->profile.enabled = true; |
172 | } | 172 | } |
173 | } | 173 | } |
@@ -246,7 +246,7 @@ static int gk20a_fifo_profile_stats(struct seq_file *s, void *unused) | |||
246 | return 0; | 246 | return 0; |
247 | } | 247 | } |
248 | 248 | ||
249 | get = atomic_read(&g->fifo.profile.get); | 249 | get = atomic_read(&g->fifo.profile.get.atomic_var); |
250 | 250 | ||
251 | __gk20a_fifo_create_stats(g, percentiles_ioctl, | 251 | __gk20a_fifo_create_stats(g, percentiles_ioctl, |
252 | PROFILE_IOCTL_EXIT, PROFILE_IOCTL_ENTRY); | 252 | PROFILE_IOCTL_EXIT, PROFILE_IOCTL_ENTRY); |
@@ -311,7 +311,7 @@ void gk20a_fifo_debugfs_init(struct gk20a *g) | |||
311 | 311 | ||
312 | nvgpu_mutex_init(&g->fifo.profile.lock); | 312 | nvgpu_mutex_init(&g->fifo.profile.lock); |
313 | g->fifo.profile.enabled = false; | 313 | g->fifo.profile.enabled = false; |
314 | atomic_set(&g->fifo.profile.get, 0); | 314 | atomic_set(&g->fifo.profile.get.atomic_var, 0); |
315 | atomic_set(&g->fifo.profile.ref.refcount, 0); | 315 | atomic_set(&g->fifo.profile.ref.refcount, 0); |
316 | 316 | ||
317 | debugfs_create_file("enable", 0600, profile_root, g, | 317 | debugfs_create_file("enable", 0600, profile_root, g, |
@@ -342,7 +342,7 @@ struct fifo_profile_gk20a *gk20a_fifo_profile_acquire(struct gk20a *g) | |||
342 | /* If kref is zero, profiling is not enabled */ | 342 | /* If kref is zero, profiling is not enabled */ |
343 | if (!kref_get_unless_zero(&f->profile.ref)) | 343 | if (!kref_get_unless_zero(&f->profile.ref)) |
344 | return NULL; | 344 | return NULL; |
345 | index = atomic_inc_return(&f->profile.get); | 345 | index = atomic_inc_return(&f->profile.get.atomic_var); |
346 | profile = &f->profile.data[index % FIFO_PROFILING_ENTRIES]; | 346 | profile = &f->profile.data[index % FIFO_PROFILING_ENTRIES]; |
347 | 347 | ||
348 | return profile; | 348 | return profile; |
diff --git a/drivers/gpu/nvgpu/common/linux/dma.c b/drivers/gpu/nvgpu/common/linux/dma.c index ea5b2837..2116053d 100644 --- a/drivers/gpu/nvgpu/common/linux/dma.c +++ b/drivers/gpu/nvgpu/common/linux/dma.c | |||
@@ -197,7 +197,7 @@ int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, | |||
197 | WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); | 197 | WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); |
198 | 198 | ||
199 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | 199 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); |
200 | before_pending = atomic64_read(&g->mm.vidmem.bytes_pending); | 200 | before_pending = atomic64_read(&g->mm.vidmem.bytes_pending.atomic_var); |
201 | addr = __nvgpu_dma_alloc(vidmem_alloc, at, size); | 201 | addr = __nvgpu_dma_alloc(vidmem_alloc, at, size); |
202 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | 202 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); |
203 | if (!addr) { | 203 | if (!addr) { |
@@ -394,7 +394,7 @@ static void nvgpu_dma_free_vid(struct gk20a *g, struct nvgpu_mem *mem) | |||
394 | was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head); | 394 | was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head); |
395 | nvgpu_list_add_tail(&mem->clear_list_entry, | 395 | nvgpu_list_add_tail(&mem->clear_list_entry, |
396 | &g->mm.vidmem.clear_list_head); | 396 | &g->mm.vidmem.clear_list_head); |
397 | atomic64_add(mem->size, &g->mm.vidmem.bytes_pending); | 397 | atomic64_add(mem->size, &g->mm.vidmem.bytes_pending.atomic_var); |
398 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | 398 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); |
399 | 399 | ||
400 | if (was_empty) { | 400 | if (was_empty) { |
diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c index bfbe7a58..f5c6ca1f 100644 --- a/drivers/gpu/nvgpu/common/linux/module.c +++ b/drivers/gpu/nvgpu/common/linux/module.c | |||
@@ -68,13 +68,13 @@ int gk20a_busy(struct gk20a *g) | |||
68 | if (!g) | 68 | if (!g) |
69 | return -ENODEV; | 69 | return -ENODEV; |
70 | 70 | ||
71 | atomic_inc(&g->usage_count); | 71 | atomic_inc(&g->usage_count.atomic_var); |
72 | 72 | ||
73 | down_read(&g->busy_lock); | 73 | down_read(&g->busy_lock); |
74 | 74 | ||
75 | if (!gk20a_can_busy(g)) { | 75 | if (!gk20a_can_busy(g)) { |
76 | ret = -ENODEV; | 76 | ret = -ENODEV; |
77 | atomic_dec(&g->usage_count); | 77 | atomic_dec(&g->usage_count.atomic_var); |
78 | goto fail; | 78 | goto fail; |
79 | } | 79 | } |
80 | 80 | ||
@@ -87,7 +87,7 @@ int gk20a_busy(struct gk20a *g) | |||
87 | /* Mark suspended so runtime pm will retry later */ | 87 | /* Mark suspended so runtime pm will retry later */ |
88 | pm_runtime_set_suspended(dev); | 88 | pm_runtime_set_suspended(dev); |
89 | pm_runtime_put_noidle(dev); | 89 | pm_runtime_put_noidle(dev); |
90 | atomic_dec(&g->usage_count); | 90 | atomic_dec(&g->usage_count.atomic_var); |
91 | goto fail; | 91 | goto fail; |
92 | } | 92 | } |
93 | } else { | 93 | } else { |
@@ -97,7 +97,7 @@ int gk20a_busy(struct gk20a *g) | |||
97 | vgpu_pm_finalize_poweron(dev) | 97 | vgpu_pm_finalize_poweron(dev) |
98 | : gk20a_pm_finalize_poweron(dev); | 98 | : gk20a_pm_finalize_poweron(dev); |
99 | if (ret) { | 99 | if (ret) { |
100 | atomic_dec(&g->usage_count); | 100 | atomic_dec(&g->usage_count.atomic_var); |
101 | nvgpu_mutex_release(&g->poweron_lock); | 101 | nvgpu_mutex_release(&g->poweron_lock); |
102 | goto fail; | 102 | goto fail; |
103 | } | 103 | } |
@@ -120,7 +120,7 @@ void gk20a_idle(struct gk20a *g) | |||
120 | { | 120 | { |
121 | struct device *dev; | 121 | struct device *dev; |
122 | 122 | ||
123 | atomic_dec(&g->usage_count); | 123 | atomic_dec(&g->usage_count.atomic_var); |
124 | 124 | ||
125 | dev = dev_from_gk20a(g); | 125 | dev = dev_from_gk20a(g); |
126 | 126 | ||
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c index 2a569efd..eeb86095 100644 --- a/drivers/gpu/nvgpu/common/mm/lockless_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c | |||
@@ -65,7 +65,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) | |||
65 | ret = cmpxchg(&pa->head, head, new_head); | 65 | ret = cmpxchg(&pa->head, head, new_head); |
66 | if (ret == head) { | 66 | if (ret == head) { |
67 | addr = pa->base + head * pa->blk_size; | 67 | addr = pa->base + head * pa->blk_size; |
68 | atomic_inc(&pa->nr_allocs); | 68 | nvgpu_atomic_inc(&pa->nr_allocs); |
69 | alloc_dbg(a, "Alloc node # %d @ addr 0x%llx\n", head, | ||
70 | addr); | ||
69 | break; | 71 | break; |
70 | } | 72 | } |
71 | head = ACCESS_ONCE(pa->head); | 73 | head = ACCESS_ONCE(pa->head); |
@@ -94,7 +96,8 @@ static void nvgpu_lockless_free(struct nvgpu_allocator *a, u64 addr) | |||
94 | ACCESS_ONCE(pa->next[cur_idx]) = head; | 96 | ACCESS_ONCE(pa->next[cur_idx]) = head; |
95 | ret = cmpxchg(&pa->head, head, cur_idx); | 97 | ret = cmpxchg(&pa->head, head, cur_idx); |
96 | if (ret == head) { | 98 | if (ret == head) { |
97 | atomic_dec(&pa->nr_allocs); | 99 | nvgpu_atomic_dec(&pa->nr_allocs); |
100 | alloc_dbg(a, "Free node # %llu\n", cur_idx); | ||
98 | break; | 101 | break; |
99 | } | 102 | } |
100 | } | 103 | } |
@@ -125,9 +128,9 @@ static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a, | |||
125 | /* Actual stats. */ | 128 | /* Actual stats. */ |
126 | __alloc_pstat(s, a, "Stats:\n"); | 129 | __alloc_pstat(s, a, "Stats:\n"); |
127 | __alloc_pstat(s, a, " Number allocs = %d\n", | 130 | __alloc_pstat(s, a, " Number allocs = %d\n", |
128 | atomic_read(&pa->nr_allocs)); | 131 | nvgpu_atomic_read(&pa->nr_allocs)); |
129 | __alloc_pstat(s, a, " Number free = %d\n", | 132 | __alloc_pstat(s, a, " Number free = %d\n", |
130 | pa->nr_nodes - atomic_read(&pa->nr_allocs)); | 133 | pa->nr_nodes - nvgpu_atomic_read(&pa->nr_allocs)); |
131 | } | 134 | } |
132 | #endif | 135 | #endif |
133 | 136 | ||
@@ -193,7 +196,7 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, | |||
193 | a->blk_size = blk_size; | 196 | a->blk_size = blk_size; |
194 | a->nr_nodes = nr_nodes; | 197 | a->nr_nodes = nr_nodes; |
195 | a->flags = flags; | 198 | a->flags = flags; |
196 | atomic_set(&a->nr_allocs, 0); | 199 | nvgpu_atomic_set(&a->nr_allocs, 0); |
197 | 200 | ||
198 | wmb(); | 201 | wmb(); |
199 | a->inited = true; | 202 | a->inited = true; |
diff --git a/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h index 32421ac1..c527bff9 100644 --- a/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -109,7 +109,7 @@ struct nvgpu_lockless_allocator { | |||
109 | bool inited; | 109 | bool inited; |
110 | 110 | ||
111 | /* Statistics */ | 111 | /* Statistics */ |
112 | atomic_t nr_allocs; | 112 | nvgpu_atomic_t nr_allocs; |
113 | }; | 113 | }; |
114 | 114 | ||
115 | static inline struct nvgpu_lockless_allocator *lockless_allocator( | 115 | static inline struct nvgpu_lockless_allocator *lockless_allocator( |
diff --git a/drivers/gpu/nvgpu/common/semaphore.c b/drivers/gpu/nvgpu/common/semaphore.c index 3e916b9d..ac45aaaa 100644 --- a/drivers/gpu/nvgpu/common/semaphore.c +++ b/drivers/gpu/nvgpu/common/semaphore.c | |||
@@ -364,7 +364,7 @@ static int __nvgpu_init_hw_sema(struct channel_gk20a *ch) | |||
364 | hw_sema->p = p; | 364 | hw_sema->p = p; |
365 | hw_sema->idx = hw_sema_idx; | 365 | hw_sema->idx = hw_sema_idx; |
366 | hw_sema->offset = SEMAPHORE_SIZE * hw_sema_idx; | 366 | hw_sema->offset = SEMAPHORE_SIZE * hw_sema_idx; |
367 | atomic_set(&hw_sema->next_value, 0); | 367 | nvgpu_atomic_set(&hw_sema->next_value, 0); |
368 | nvgpu_init_list_node(&hw_sema->hw_sema_list); | 368 | nvgpu_init_list_node(&hw_sema->hw_sema_list); |
369 | nvgpu_mem_wr(ch->g, &p->rw_mem, hw_sema->offset, 0); | 369 | nvgpu_mem_wr(ch->g, &p->rw_mem, hw_sema->offset, 0); |
370 | 370 | ||
@@ -425,7 +425,7 @@ struct nvgpu_semaphore *nvgpu_semaphore_alloc(struct channel_gk20a *ch) | |||
425 | 425 | ||
426 | kref_init(&s->ref); | 426 | kref_init(&s->ref); |
427 | s->hw_sema = ch->hw_sema; | 427 | s->hw_sema = ch->hw_sema; |
428 | atomic_set(&s->value, 0); | 428 | nvgpu_atomic_set(&s->value, 0); |
429 | 429 | ||
430 | /* | 430 | /* |
431 | * Take a ref on the pool so that we can keep this pool alive for | 431 | * Take a ref on the pool so that we can keep this pool alive for |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index 62b312b2..d96872f3 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c | |||
@@ -100,7 +100,7 @@ static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f) | |||
100 | ch = nvgpu_list_first_entry(&f->free_chs, channel_gk20a, | 100 | ch = nvgpu_list_first_entry(&f->free_chs, channel_gk20a, |
101 | free_chs); | 101 | free_chs); |
102 | nvgpu_list_del(&ch->free_chs); | 102 | nvgpu_list_del(&ch->free_chs); |
103 | WARN_ON(atomic_read(&ch->ref_count)); | 103 | WARN_ON(nvgpu_atomic_read(&ch->ref_count)); |
104 | WARN_ON(ch->referenceable); | 104 | WARN_ON(ch->referenceable); |
105 | f->used_channels++; | 105 | f->used_channels++; |
106 | } | 106 | } |
@@ -394,20 +394,20 @@ void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error) | |||
394 | } | 394 | } |
395 | 395 | ||
396 | static void gk20a_wait_until_counter_is_N( | 396 | static void gk20a_wait_until_counter_is_N( |
397 | struct channel_gk20a *ch, atomic_t *counter, int wait_value, | 397 | struct channel_gk20a *ch, nvgpu_atomic_t *counter, int wait_value, |
398 | struct nvgpu_cond *c, const char *caller, const char *counter_name) | 398 | struct nvgpu_cond *c, const char *caller, const char *counter_name) |
399 | { | 399 | { |
400 | while (true) { | 400 | while (true) { |
401 | if (NVGPU_COND_WAIT( | 401 | if (NVGPU_COND_WAIT( |
402 | c, | 402 | c, |
403 | atomic_read(counter) == wait_value, | 403 | nvgpu_atomic_read(counter) == wait_value, |
404 | 5000) == 0) | 404 | 5000) == 0) |
405 | break; | 405 | break; |
406 | 406 | ||
407 | nvgpu_warn(ch->g, | 407 | nvgpu_warn(ch->g, |
408 | "%s: channel %d, still waiting, %s left: %d, waiting for: %d", | 408 | "%s: channel %d, still waiting, %s left: %d, waiting for: %d", |
409 | caller, ch->chid, counter_name, | 409 | caller, ch->chid, counter_name, |
410 | atomic_read(counter), wait_value); | 410 | nvgpu_atomic_read(counter), wait_value); |
411 | 411 | ||
412 | gk20a_channel_dump_ref_actions(ch); | 412 | gk20a_channel_dump_ref_actions(ch); |
413 | } | 413 | } |
@@ -491,7 +491,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) | |||
491 | nvgpu_spinlock_release(&ch->ref_obtain_lock); | 491 | nvgpu_spinlock_release(&ch->ref_obtain_lock); |
492 | 492 | ||
493 | /* matches with the initial reference in gk20a_open_new_channel() */ | 493 | /* matches with the initial reference in gk20a_open_new_channel() */ |
494 | atomic_dec(&ch->ref_count); | 494 | nvgpu_atomic_dec(&ch->ref_count); |
495 | 495 | ||
496 | /* wait until no more refs to the channel */ | 496 | /* wait until no more refs to the channel */ |
497 | if (!force) | 497 | if (!force) |
@@ -635,7 +635,7 @@ static void gk20a_channel_dump_ref_actions(struct channel_gk20a *ch) | |||
635 | nvgpu_spinlock_acquire(&ch->ref_actions_lock); | 635 | nvgpu_spinlock_acquire(&ch->ref_actions_lock); |
636 | 636 | ||
637 | dev_info(dev, "ch %d: refs %d. Actions, most recent last:\n", | 637 | dev_info(dev, "ch %d: refs %d. Actions, most recent last:\n", |
638 | ch->chid, atomic_read(&ch->ref_count)); | 638 | ch->chid, nvgpu_atomic_read(&ch->ref_count)); |
639 | 639 | ||
640 | /* start at the oldest possible entry. put is next insertion point */ | 640 | /* start at the oldest possible entry. put is next insertion point */ |
641 | get = ch->ref_actions_put; | 641 | get = ch->ref_actions_put; |
@@ -709,7 +709,7 @@ struct channel_gk20a *_gk20a_channel_get(struct channel_gk20a *ch, | |||
709 | 709 | ||
710 | if (likely(ch->referenceable)) { | 710 | if (likely(ch->referenceable)) { |
711 | gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_get); | 711 | gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_get); |
712 | atomic_inc(&ch->ref_count); | 712 | nvgpu_atomic_inc(&ch->ref_count); |
713 | ret = ch; | 713 | ret = ch; |
714 | } else | 714 | } else |
715 | ret = NULL; | 715 | ret = NULL; |
@@ -726,17 +726,17 @@ void _gk20a_channel_put(struct channel_gk20a *ch, const char *caller) | |||
726 | { | 726 | { |
727 | gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_put); | 727 | gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_put); |
728 | trace_gk20a_channel_put(ch->chid, caller); | 728 | trace_gk20a_channel_put(ch->chid, caller); |
729 | atomic_dec(&ch->ref_count); | 729 | nvgpu_atomic_dec(&ch->ref_count); |
730 | nvgpu_cond_broadcast(&ch->ref_count_dec_wq); | 730 | nvgpu_cond_broadcast(&ch->ref_count_dec_wq); |
731 | 731 | ||
732 | /* More puts than gets. Channel is probably going to get | 732 | /* More puts than gets. Channel is probably going to get |
733 | * stuck. */ | 733 | * stuck. */ |
734 | WARN_ON(atomic_read(&ch->ref_count) < 0); | 734 | WARN_ON(nvgpu_atomic_read(&ch->ref_count) < 0); |
735 | 735 | ||
736 | /* Also, more puts than gets. ref_count can go to 0 only if | 736 | /* Also, more puts than gets. ref_count can go to 0 only if |
737 | * the channel is closing. Channel is probably going to get | 737 | * the channel is closing. Channel is probably going to get |
738 | * stuck. */ | 738 | * stuck. */ |
739 | WARN_ON(atomic_read(&ch->ref_count) == 0 && ch->referenceable); | 739 | WARN_ON(nvgpu_atomic_read(&ch->ref_count) == 0 && ch->referenceable); |
740 | } | 740 | } |
741 | 741 | ||
742 | void gk20a_channel_close(struct channel_gk20a *ch) | 742 | void gk20a_channel_close(struct channel_gk20a *ch) |
@@ -879,7 +879,7 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g, | |||
879 | * references. The initial reference will be decreased in | 879 | * references. The initial reference will be decreased in |
880 | * gk20a_free_channel() */ | 880 | * gk20a_free_channel() */ |
881 | ch->referenceable = true; | 881 | ch->referenceable = true; |
882 | atomic_set(&ch->ref_count, 1); | 882 | nvgpu_atomic_set(&ch->ref_count, 1); |
883 | wmb(); | 883 | wmb(); |
884 | 884 | ||
885 | return ch; | 885 | return ch; |
@@ -1745,7 +1745,7 @@ static int __gk20a_channel_worker_wakeup(struct gk20a *g) | |||
1745 | * pair. | 1745 | * pair. |
1746 | */ | 1746 | */ |
1747 | 1747 | ||
1748 | put = atomic_inc_return(&g->channel_worker.put); | 1748 | put = nvgpu_atomic_inc_return(&g->channel_worker.put); |
1749 | nvgpu_cond_signal(&g->channel_worker.wq); | 1749 | nvgpu_cond_signal(&g->channel_worker.wq); |
1750 | 1750 | ||
1751 | return put; | 1751 | return put; |
@@ -1761,7 +1761,7 @@ static int __gk20a_channel_worker_wakeup(struct gk20a *g) | |||
1761 | */ | 1761 | */ |
1762 | static bool __gk20a_channel_worker_pending(struct gk20a *g, int get) | 1762 | static bool __gk20a_channel_worker_pending(struct gk20a *g, int get) |
1763 | { | 1763 | { |
1764 | bool pending = atomic_read(&g->channel_worker.put) != get; | 1764 | bool pending = nvgpu_atomic_read(&g->channel_worker.put) != get; |
1765 | 1765 | ||
1766 | /* | 1766 | /* |
1767 | * This would be the place for a rmb() pairing a wmb() for a wakeup | 1767 | * This would be the place for a rmb() pairing a wmb() for a wakeup |
@@ -1864,7 +1864,7 @@ int nvgpu_channel_worker_init(struct gk20a *g) | |||
1864 | int err; | 1864 | int err; |
1865 | char thread_name[64]; | 1865 | char thread_name[64]; |
1866 | 1866 | ||
1867 | atomic_set(&g->channel_worker.put, 0); | 1867 | nvgpu_atomic_set(&g->channel_worker.put, 0); |
1868 | nvgpu_cond_init(&g->channel_worker.wq); | 1868 | nvgpu_cond_init(&g->channel_worker.wq); |
1869 | nvgpu_init_list_node(&g->channel_worker.items); | 1869 | nvgpu_init_list_node(&g->channel_worker.items); |
1870 | nvgpu_spinlock_init(&g->channel_worker.items_lock); | 1870 | nvgpu_spinlock_init(&g->channel_worker.items_lock); |
@@ -2086,7 +2086,8 @@ static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c, | |||
2086 | 2086 | ||
2087 | if (g->aggressive_sync_destroy_thresh) { | 2087 | if (g->aggressive_sync_destroy_thresh) { |
2088 | nvgpu_mutex_acquire(&c->sync_lock); | 2088 | nvgpu_mutex_acquire(&c->sync_lock); |
2089 | if (atomic_dec_and_test(&c->sync->refcount) && | 2089 | if (nvgpu_atomic_dec_and_test( |
2090 | &c->sync->refcount) && | ||
2090 | g->aggressive_sync_destroy) { | 2091 | g->aggressive_sync_destroy) { |
2091 | gk20a_channel_sync_destroy(c->sync); | 2092 | gk20a_channel_sync_destroy(c->sync); |
2092 | c->sync = NULL; | 2093 | c->sync = NULL; |
@@ -2321,7 +2322,7 @@ static int gk20a_submit_prepare_syncs(struct channel_gk20a *c, | |||
2321 | } | 2322 | } |
2322 | new_sync_created = true; | 2323 | new_sync_created = true; |
2323 | } | 2324 | } |
2324 | atomic_inc(&c->sync->refcount); | 2325 | nvgpu_atomic_inc(&c->sync->refcount); |
2325 | nvgpu_mutex_release(&c->sync_lock); | 2326 | nvgpu_mutex_release(&c->sync_lock); |
2326 | } | 2327 | } |
2327 | 2328 | ||
@@ -2774,9 +2775,9 @@ int gk20a_init_channel_support(struct gk20a *g, u32 chid) | |||
2774 | 2775 | ||
2775 | c->g = NULL; | 2776 | c->g = NULL; |
2776 | c->chid = chid; | 2777 | c->chid = chid; |
2777 | atomic_set(&c->bound, false); | 2778 | nvgpu_atomic_set(&c->bound, false); |
2778 | nvgpu_spinlock_init(&c->ref_obtain_lock); | 2779 | nvgpu_spinlock_init(&c->ref_obtain_lock); |
2779 | atomic_set(&c->ref_count, 0); | 2780 | nvgpu_atomic_set(&c->ref_count, 0); |
2780 | c->referenceable = false; | 2781 | c->referenceable = false; |
2781 | nvgpu_cond_init(&c->ref_count_dec_wq); | 2782 | nvgpu_cond_init(&c->ref_count_dec_wq); |
2782 | 2783 | ||
@@ -2935,7 +2936,7 @@ void gk20a_channel_semaphore_wakeup(struct gk20a *g, bool post_events) | |||
2935 | for (chid = 0; chid < f->num_channels; chid++) { | 2936 | for (chid = 0; chid < f->num_channels; chid++) { |
2936 | struct channel_gk20a *c = g->fifo.channel+chid; | 2937 | struct channel_gk20a *c = g->fifo.channel+chid; |
2937 | if (gk20a_channel_get(c)) { | 2938 | if (gk20a_channel_get(c)) { |
2938 | if (atomic_read(&c->bound)) { | 2939 | if (nvgpu_atomic_read(&c->bound)) { |
2939 | nvgpu_cond_broadcast_interruptible( | 2940 | nvgpu_cond_broadcast_interruptible( |
2940 | &c->semaphore_wq); | 2941 | &c->semaphore_wq); |
2941 | if (post_events) { | 2942 | if (post_events) { |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h index a9ccd93f..f022e630 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <nvgpu/lock.h> | 24 | #include <nvgpu/lock.h> |
25 | #include <nvgpu/timers.h> | 25 | #include <nvgpu/timers.h> |
26 | #include <nvgpu/cond.h> | 26 | #include <nvgpu/cond.h> |
27 | #include <nvgpu/atomic.h> | ||
27 | 28 | ||
28 | struct gk20a; | 29 | struct gk20a; |
29 | struct gr_gk20a; | 30 | struct gr_gk20a; |
@@ -173,7 +174,7 @@ struct channel_gk20a { | |||
173 | 174 | ||
174 | struct nvgpu_spinlock ref_obtain_lock; | 175 | struct nvgpu_spinlock ref_obtain_lock; |
175 | bool referenceable; | 176 | bool referenceable; |
176 | atomic_t ref_count; | 177 | nvgpu_atomic_t ref_count; |
177 | struct nvgpu_cond ref_count_dec_wq; | 178 | struct nvgpu_cond ref_count_dec_wq; |
178 | #if GK20A_CHANNEL_REFCOUNT_TRACKING | 179 | #if GK20A_CHANNEL_REFCOUNT_TRACKING |
179 | /* | 180 | /* |
@@ -191,7 +192,7 @@ struct channel_gk20a { | |||
191 | 192 | ||
192 | int chid; | 193 | int chid; |
193 | bool wdt_enabled; | 194 | bool wdt_enabled; |
194 | atomic_t bound; | 195 | nvgpu_atomic_t bound; |
195 | bool first_init; | 196 | bool first_init; |
196 | bool vpr; | 197 | bool vpr; |
197 | bool deterministic; | 198 | bool deterministic; |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c index c9c03d37..aa340ba6 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c | |||
@@ -346,7 +346,7 @@ gk20a_channel_syncpt_create(struct channel_gk20a *c) | |||
346 | 346 | ||
347 | nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); | 347 | nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id); |
348 | 348 | ||
349 | atomic_set(&sp->ops.refcount, 0); | 349 | nvgpu_atomic_set(&sp->ops.refcount, 0); |
350 | sp->ops.wait_syncpt = gk20a_channel_syncpt_wait_syncpt; | 350 | sp->ops.wait_syncpt = gk20a_channel_syncpt_wait_syncpt; |
351 | sp->ops.wait_fd = gk20a_channel_syncpt_wait_fd; | 351 | sp->ops.wait_fd = gk20a_channel_syncpt_wait_fd; |
352 | sp->ops.incr = gk20a_channel_syncpt_incr; | 352 | sp->ops.incr = gk20a_channel_syncpt_incr; |
@@ -619,7 +619,7 @@ static int __semaphore_wait_fd_fast_path(struct channel_gk20a *c, | |||
619 | return err; | 619 | return err; |
620 | 620 | ||
621 | nvgpu_semaphore_get(sema); | 621 | nvgpu_semaphore_get(sema); |
622 | BUG_ON(!atomic_read(&sema->value)); | 622 | BUG_ON(!nvgpu_atomic_read(&sema->value)); |
623 | add_sema_cmd(c->g, c, sema, wait_cmd, 8, true, false); | 623 | add_sema_cmd(c->g, c, sema, wait_cmd, 8, true, false); |
624 | 624 | ||
625 | /* | 625 | /* |
@@ -922,7 +922,7 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c) | |||
922 | return NULL; | 922 | return NULL; |
923 | } | 923 | } |
924 | #endif | 924 | #endif |
925 | atomic_set(&sema->ops.refcount, 0); | 925 | nvgpu_atomic_set(&sema->ops.refcount, 0); |
926 | sema->ops.wait_syncpt = gk20a_channel_semaphore_wait_syncpt; | 926 | sema->ops.wait_syncpt = gk20a_channel_semaphore_wait_syncpt; |
927 | sema->ops.wait_fd = gk20a_channel_semaphore_wait_fd; | 927 | sema->ops.wait_fd = gk20a_channel_semaphore_wait_fd; |
928 | sema->ops.incr = gk20a_channel_semaphore_incr; | 928 | sema->ops.incr = gk20a_channel_semaphore_incr; |
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h index 4efd1b76..9bdc5d12 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h | |||
@@ -25,7 +25,7 @@ struct gk20a_fence; | |||
25 | struct gk20a; | 25 | struct gk20a; |
26 | 26 | ||
27 | struct gk20a_channel_sync { | 27 | struct gk20a_channel_sync { |
28 | atomic_t refcount; | 28 | nvgpu_atomic_t refcount; |
29 | 29 | ||
30 | /* Generate a gpu wait cmdbuf from syncpoint. | 30 | /* Generate a gpu wait cmdbuf from syncpoint. |
31 | * Returns | 31 | * Returns |
diff --git a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c index cc05ceff..546917f1 100644 --- a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c | |||
@@ -47,7 +47,7 @@ struct gk20a_ctxsw_dev { | |||
47 | size_t size; | 47 | size_t size; |
48 | u32 num_ents; | 48 | u32 num_ents; |
49 | 49 | ||
50 | atomic_t vma_ref; | 50 | nvgpu_atomic_t vma_ref; |
51 | 51 | ||
52 | struct nvgpu_mutex write_lock; | 52 | struct nvgpu_mutex write_lock; |
53 | }; | 53 | }; |
@@ -152,7 +152,7 @@ static int gk20a_ctxsw_dev_alloc_buffer(struct gk20a_ctxsw_dev *dev, | |||
152 | void *buf; | 152 | void *buf; |
153 | int err; | 153 | int err; |
154 | 154 | ||
155 | if ((dev->write_enabled) || (atomic_read(&dev->vma_ref))) | 155 | if ((dev->write_enabled) || (nvgpu_atomic_read(&dev->vma_ref))) |
156 | return -EBUSY; | 156 | return -EBUSY; |
157 | 157 | ||
158 | err = g->ops.fecs_trace.alloc_user_buffer(g, &buf, &size); | 158 | err = g->ops.fecs_trace.alloc_user_buffer(g, &buf, &size); |
@@ -438,18 +438,18 @@ static void gk20a_ctxsw_dev_vma_open(struct vm_area_struct *vma) | |||
438 | { | 438 | { |
439 | struct gk20a_ctxsw_dev *dev = vma->vm_private_data; | 439 | struct gk20a_ctxsw_dev *dev = vma->vm_private_data; |
440 | 440 | ||
441 | atomic_inc(&dev->vma_ref); | 441 | nvgpu_atomic_inc(&dev->vma_ref); |
442 | gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d", | 442 | gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d", |
443 | atomic_read(&dev->vma_ref)); | 443 | nvgpu_atomic_read(&dev->vma_ref)); |
444 | } | 444 | } |
445 | 445 | ||
446 | static void gk20a_ctxsw_dev_vma_close(struct vm_area_struct *vma) | 446 | static void gk20a_ctxsw_dev_vma_close(struct vm_area_struct *vma) |
447 | { | 447 | { |
448 | struct gk20a_ctxsw_dev *dev = vma->vm_private_data; | 448 | struct gk20a_ctxsw_dev *dev = vma->vm_private_data; |
449 | 449 | ||
450 | atomic_dec(&dev->vma_ref); | 450 | nvgpu_atomic_dec(&dev->vma_ref); |
451 | gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d", | 451 | gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d", |
452 | atomic_read(&dev->vma_ref)); | 452 | nvgpu_atomic_read(&dev->vma_ref)); |
453 | } | 453 | } |
454 | 454 | ||
455 | static struct vm_operations_struct gk20a_ctxsw_dev_vma_ops = { | 455 | static struct vm_operations_struct gk20a_ctxsw_dev_vma_ops = { |
@@ -497,7 +497,7 @@ static int gk20a_ctxsw_init_devs(struct gk20a *g) | |||
497 | err = nvgpu_mutex_init(&dev->write_lock); | 497 | err = nvgpu_mutex_init(&dev->write_lock); |
498 | if (err) | 498 | if (err) |
499 | return err; | 499 | return err; |
500 | atomic_set(&dev->vma_ref, 0); | 500 | nvgpu_atomic_set(&dev->vma_ref, 0); |
501 | dev++; | 501 | dev++; |
502 | } | 502 | } |
503 | return 0; | 503 | return 0; |
diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c index 1572ff48..00050850 100644 --- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c | |||
@@ -26,6 +26,7 @@ | |||
26 | #include <nvgpu/kmem.h> | 26 | #include <nvgpu/kmem.h> |
27 | #include <nvgpu/log.h> | 27 | #include <nvgpu/log.h> |
28 | #include <nvgpu/vm.h> | 28 | #include <nvgpu/vm.h> |
29 | #include <nvgpu/atomic.h> | ||
29 | 30 | ||
30 | #include "gk20a.h" | 31 | #include "gk20a.h" |
31 | #include "gk20a/platform_gk20a.h" | 32 | #include "gk20a/platform_gk20a.h" |
@@ -74,10 +75,10 @@ nvgpu_dbg_gpu_get_session_channel(struct dbg_session_gk20a *dbg_s) | |||
74 | } | 75 | } |
75 | 76 | ||
76 | /* silly allocator - just increment id */ | 77 | /* silly allocator - just increment id */ |
77 | static atomic_t unique_id = ATOMIC_INIT(0); | 78 | static nvgpu_atomic_t unique_id = NVGPU_ATOMIC_INIT(0); |
78 | static int generate_unique_id(void) | 79 | static int generate_unique_id(void) |
79 | { | 80 | { |
80 | return atomic_add_return(1, &unique_id); | 81 | return nvgpu_atomic_add_return(1, &unique_id); |
81 | } | 82 | } |
82 | 83 | ||
83 | static int alloc_session(struct gk20a *g, struct dbg_session_gk20a **_dbg_s) | 84 | static int alloc_session(struct gk20a *g, struct dbg_session_gk20a **_dbg_s) |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index abd455d7..47e7d82e 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | |||
@@ -3439,7 +3439,7 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g, | |||
3439 | gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " | 3439 | gk20a_debug_output(o, "SEMA STATE: value: 0x%08x " |
3440 | "next_val: 0x%08x addr: 0x%010llx\n", | 3440 | "next_val: 0x%08x addr: 0x%010llx\n", |
3441 | __nvgpu_semaphore_read(hw_sema), | 3441 | __nvgpu_semaphore_read(hw_sema), |
3442 | atomic_read(&hw_sema->next_value), | 3442 | nvgpu_atomic_read(&hw_sema->next_value), |
3443 | nvgpu_hw_sema_addr(hw_sema)); | 3443 | nvgpu_hw_sema_addr(hw_sema)); |
3444 | 3444 | ||
3445 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | 3445 | #ifdef CONFIG_TEGRA_GK20A_NVHOST |
@@ -3489,7 +3489,7 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g, | |||
3489 | continue; | 3489 | continue; |
3490 | 3490 | ||
3491 | ch_state[chid]->pid = ch->pid; | 3491 | ch_state[chid]->pid = ch->pid; |
3492 | ch_state[chid]->refs = atomic_read(&ch->ref_count); | 3492 | ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count); |
3493 | ch_state[chid]->deterministic = ch->deterministic; | 3493 | ch_state[chid]->deterministic = ch->deterministic; |
3494 | nvgpu_mem_rd_n(g, &ch->inst_block, 0, | 3494 | nvgpu_mem_rd_n(g, &ch->inst_block, 0, |
3495 | &ch_state[chid]->inst_block[0], | 3495 | &ch_state[chid]->inst_block[0], |
@@ -3591,7 +3591,7 @@ void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a) | |||
3591 | 3591 | ||
3592 | gk20a_dbg_fn(""); | 3592 | gk20a_dbg_fn(""); |
3593 | 3593 | ||
3594 | if (atomic_cmpxchg(&ch_gk20a->bound, true, false)) { | 3594 | if (nvgpu_atomic_cmpxchg(&ch_gk20a->bound, true, false)) { |
3595 | gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->chid), | 3595 | gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->chid), |
3596 | ccsr_channel_inst_ptr_f(0) | | 3596 | ccsr_channel_inst_ptr_f(0) | |
3597 | ccsr_channel_inst_bind_false_f()); | 3597 | ccsr_channel_inst_bind_false_f()); |
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index b19a7b68..a6eae8ca 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | |||
@@ -156,7 +156,7 @@ struct fifo_gk20a { | |||
156 | #ifdef CONFIG_DEBUG_FS | 156 | #ifdef CONFIG_DEBUG_FS |
157 | struct { | 157 | struct { |
158 | struct fifo_profile_gk20a *data; | 158 | struct fifo_profile_gk20a *data; |
159 | atomic_t get; | 159 | nvgpu_atomic_t get; |
160 | bool enabled; | 160 | bool enabled; |
161 | u64 *sorted; | 161 | u64 *sorted; |
162 | struct kref ref; | 162 | struct kref ref; |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index c50d800f..550b22c0 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c | |||
@@ -373,13 +373,13 @@ int gk20a_wait_for_idle(struct gk20a *g) | |||
373 | if (g->user_railgate_disabled) | 373 | if (g->user_railgate_disabled) |
374 | target_usage_count = 1; | 374 | target_usage_count = 1; |
375 | 375 | ||
376 | while ((atomic_read(&g->usage_count) != target_usage_count) | 376 | while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count) |
377 | && (wait_length-- >= 0)) | 377 | && (wait_length-- >= 0)) |
378 | nvgpu_msleep(20); | 378 | nvgpu_msleep(20); |
379 | 379 | ||
380 | if (wait_length < 0) { | 380 | if (wait_length < 0) { |
381 | pr_warn("%s: Timed out waiting for idle (%d)!\n", | 381 | pr_warn("%s: Timed out waiting for idle (%d)!\n", |
382 | __func__, atomic_read(&g->usage_count)); | 382 | __func__, nvgpu_atomic_read(&g->usage_count)); |
383 | return -ETIMEDOUT; | 383 | return -ETIMEDOUT; |
384 | } | 384 | } |
385 | 385 | ||
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 4878fdd6..47fd3aef 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -48,6 +48,7 @@ struct nvgpu_cpu_time_correlation_sample; | |||
48 | #include <nvgpu/kref.h> | 48 | #include <nvgpu/kref.h> |
49 | #include <nvgpu/falcon.h> | 49 | #include <nvgpu/falcon.h> |
50 | #include <nvgpu/pmu.h> | 50 | #include <nvgpu/pmu.h> |
51 | #include <nvgpu/atomic.h> | ||
51 | 52 | ||
52 | #include "clk_gk20a.h" | 53 | #include "clk_gk20a.h" |
53 | #include "ce2_gk20a.h" | 54 | #include "ce2_gk20a.h" |
@@ -1038,7 +1039,7 @@ struct gk20a { | |||
1038 | */ | 1039 | */ |
1039 | unsigned long *enabled_flags; | 1040 | unsigned long *enabled_flags; |
1040 | 1041 | ||
1041 | atomic_t usage_count; | 1042 | nvgpu_atomic_t usage_count; |
1042 | 1043 | ||
1043 | struct kref refcount; | 1044 | struct kref refcount; |
1044 | 1045 | ||
@@ -1205,7 +1206,7 @@ struct gk20a { | |||
1205 | 1206 | ||
1206 | struct gk20a_channel_worker { | 1207 | struct gk20a_channel_worker { |
1207 | struct nvgpu_thread poll_task; | 1208 | struct nvgpu_thread poll_task; |
1208 | atomic_t put; | 1209 | nvgpu_atomic_t put; |
1209 | struct nvgpu_cond wq; | 1210 | struct nvgpu_cond wq; |
1210 | struct nvgpu_list_node items; | 1211 | struct nvgpu_list_node items; |
1211 | struct nvgpu_spinlock items_lock; | 1212 | struct nvgpu_spinlock items_lock; |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 16fe7149..e21be1e5 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c | |||
@@ -617,7 +617,7 @@ static int gk20a_init_vidmem(struct mm_gk20a *mm) | |||
617 | nvgpu_mutex_init(&mm->vidmem.first_clear_mutex); | 617 | nvgpu_mutex_init(&mm->vidmem.first_clear_mutex); |
618 | 618 | ||
619 | INIT_WORK(&mm->vidmem.clear_mem_worker, gk20a_vidmem_clear_mem_worker); | 619 | INIT_WORK(&mm->vidmem.clear_mem_worker, gk20a_vidmem_clear_mem_worker); |
620 | atomic64_set(&mm->vidmem.bytes_pending, 0); | 620 | nvgpu_atomic64_set(&mm->vidmem.bytes_pending, 0); |
621 | nvgpu_init_list_node(&mm->vidmem.clear_list_head); | 621 | nvgpu_init_list_node(&mm->vidmem.clear_list_head); |
622 | nvgpu_mutex_init(&mm->vidmem.clear_list_mutex); | 622 | nvgpu_mutex_init(&mm->vidmem.clear_list_mutex); |
623 | 623 | ||
@@ -1165,7 +1165,7 @@ int gk20a_vidmem_get_space(struct gk20a *g, u64 *space) | |||
1165 | 1165 | ||
1166 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | 1166 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); |
1167 | *space = nvgpu_alloc_space(allocator) + | 1167 | *space = nvgpu_alloc_space(allocator) + |
1168 | atomic64_read(&g->mm.vidmem.bytes_pending); | 1168 | nvgpu_atomic64_read(&g->mm.vidmem.bytes_pending); |
1169 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | 1169 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); |
1170 | return 0; | 1170 | return 0; |
1171 | #else | 1171 | #else |
@@ -1483,7 +1483,7 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work) | |||
1483 | (u64)get_vidmem_page_alloc(mem->priv.sgt->sgl)); | 1483 | (u64)get_vidmem_page_alloc(mem->priv.sgt->sgl)); |
1484 | nvgpu_free_sgtable(g, &mem->priv.sgt); | 1484 | nvgpu_free_sgtable(g, &mem->priv.sgt); |
1485 | 1485 | ||
1486 | WARN_ON(atomic64_sub_return(mem->size, | 1486 | WARN_ON(nvgpu_atomic64_sub_return(mem->size, |
1487 | &g->mm.vidmem.bytes_pending) < 0); | 1487 | &g->mm.vidmem.bytes_pending) < 0); |
1488 | mem->size = 0; | 1488 | mem->size = 0; |
1489 | mem->aperture = APERTURE_INVALID; | 1489 | mem->aperture = APERTURE_INVALID; |
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index 7b2c0dfc..af176a73 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h | |||
@@ -30,6 +30,7 @@ | |||
30 | #include <nvgpu/list.h> | 30 | #include <nvgpu/list.h> |
31 | #include <nvgpu/rbtree.h> | 31 | #include <nvgpu/rbtree.h> |
32 | #include <nvgpu/kref.h> | 32 | #include <nvgpu/kref.h> |
33 | #include <nvgpu/atomic.h> | ||
33 | 34 | ||
34 | struct nvgpu_pd_cache; | 35 | struct nvgpu_pd_cache; |
35 | 36 | ||
@@ -283,7 +284,7 @@ struct mm_gk20a { | |||
283 | struct nvgpu_mutex clear_list_mutex; | 284 | struct nvgpu_mutex clear_list_mutex; |
284 | 285 | ||
285 | struct work_struct clear_mem_worker; | 286 | struct work_struct clear_mem_worker; |
286 | atomic64_t bytes_pending; | 287 | nvgpu_atomic64_t bytes_pending; |
287 | } vidmem; | 288 | } vidmem; |
288 | }; | 289 | }; |
289 | 290 | ||
diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c index afdfba91..e688c863 100644 --- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c | |||
@@ -51,7 +51,7 @@ void channel_gm20b_bind(struct channel_gk20a *c) | |||
51 | ~ccsr_channel_enable_set_f(~0)) | | 51 | ~ccsr_channel_enable_set_f(~0)) | |
52 | ccsr_channel_enable_set_true_f()); | 52 | ccsr_channel_enable_set_true_f()); |
53 | wmb(); | 53 | wmb(); |
54 | atomic_set(&c->bound, true); | 54 | nvgpu_atomic_set(&c->bound, true); |
55 | } | 55 | } |
56 | 56 | ||
57 | static inline u32 gm20b_engine_id_to_mmu_id(struct gk20a *g, u32 engine_id) | 57 | static inline u32 gm20b_engine_id_to_mmu_id(struct gk20a *g, u32 engine_id) |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/atomic.h b/drivers/gpu/nvgpu/include/nvgpu/atomic.h index 700e29fe..c7a5fcd9 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/atomic.h +++ b/drivers/gpu/nvgpu/include/nvgpu/atomic.h | |||
@@ -9,12 +9,102 @@ | |||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
11 | * more details. | 11 | * more details. |
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
12 | */ | 15 | */ |
13 | #ifndef __NVGPU_ATOMIC_H__ | 16 | #ifndef __NVGPU_ATOMIC_H__ |
14 | #define __NVGPU_ATOMIC_H__ | 17 | #define __NVGPU_ATOMIC_H__ |
15 | 18 | ||
16 | #ifdef __KERNEL__ | 19 | #include <nvgpu/linux/atomic.h> |
17 | #include <linux/atomic.h> | 20 | |
18 | #endif | 21 | #define NVGPU_ATOMIC_INIT(i) __nvgpu_atomic_init(i) |
22 | #define NVGPU_ATOMIC64_INIT(i) __nvgpu_atomic64_init(i) | ||
23 | |||
24 | static inline void nvgpu_atomic_set(nvgpu_atomic_t *v, int i) | ||
25 | { | ||
26 | __nvgpu_atomic_set(v, i); | ||
27 | } | ||
28 | static inline int nvgpu_atomic_read(nvgpu_atomic_t *v) | ||
29 | { | ||
30 | return __nvgpu_atomic_read(v); | ||
31 | } | ||
32 | static inline void nvgpu_atomic_inc(nvgpu_atomic_t *v) | ||
33 | { | ||
34 | __nvgpu_atomic_inc(v); | ||
35 | } | ||
36 | static inline int nvgpu_atomic_inc_return(nvgpu_atomic_t *v) | ||
37 | { | ||
38 | return __nvgpu_atomic_inc_return(v); | ||
39 | } | ||
40 | static inline void nvgpu_atomic_dec(nvgpu_atomic_t *v) | ||
41 | { | ||
42 | __nvgpu_atomic_dec(v); | ||
43 | } | ||
44 | static inline int nvgpu_atomic_dec_return(nvgpu_atomic_t *v) | ||
45 | { | ||
46 | return __nvgpu_atomic_dec_return(v); | ||
47 | } | ||
48 | static inline int nvgpu_atomic_cmpxchg(nvgpu_atomic_t *v, int old, int new) | ||
49 | { | ||
50 | return __nvgpu_atomic_cmpxchg(v, old, new); | ||
51 | } | ||
52 | static inline int nvgpu_atomic_xchg(nvgpu_atomic_t *v, int new) | ||
53 | { | ||
54 | return __nvgpu_atomic_xchg(v, new); | ||
55 | } | ||
56 | static inline bool nvgpu_atomic_inc_and_test(nvgpu_atomic_t *v) | ||
57 | { | ||
58 | return __nvgpu_atomic_inc_and_test(v); | ||
59 | } | ||
60 | static inline bool nvgpu_atomic_dec_and_test(nvgpu_atomic_t *v) | ||
61 | { | ||
62 | return __nvgpu_atomic_dec_and_test(v); | ||
63 | } | ||
64 | static inline int nvgpu_atomic_add_return(int i, nvgpu_atomic_t *v) | ||
65 | { | ||
66 | return __nvgpu_atomic_add_return(i, v); | ||
67 | } | ||
68 | static inline void nvgpu_atomic64_set(nvgpu_atomic64_t *v, long i) | ||
69 | { | ||
70 | return __nvgpu_atomic64_set(v, i); | ||
71 | } | ||
72 | static inline long nvgpu_atomic64_read(nvgpu_atomic64_t *v) | ||
73 | { | ||
74 | return __nvgpu_atomic64_read(v); | ||
75 | } | ||
76 | static inline void nvgpu_atomic64_add(long x, nvgpu_atomic64_t *v) | ||
77 | { | ||
78 | __nvgpu_atomic64_add(x, v); | ||
79 | } | ||
80 | static inline void nvgpu_atomic64_inc(nvgpu_atomic64_t *v) | ||
81 | { | ||
82 | __nvgpu_atomic64_inc(v); | ||
83 | } | ||
84 | static inline long nvgpu_atomic64_inc_return(nvgpu_atomic64_t *v) | ||
85 | { | ||
86 | return __nvgpu_atomic64_inc_return(v); | ||
87 | } | ||
88 | static inline void nvgpu_atomic64_dec(nvgpu_atomic64_t *v) | ||
89 | { | ||
90 | __nvgpu_atomic64_dec(v); | ||
91 | } | ||
92 | static inline void nvgpu_atomic64_dec_return(nvgpu_atomic64_t *v) | ||
93 | { | ||
94 | __nvgpu_atomic64_dec_return(v); | ||
95 | } | ||
96 | static inline long nvgpu_atomic64_cmpxchg(nvgpu_atomic64_t *v, long old, | ||
97 | long new) | ||
98 | { | ||
99 | return __nvgpu_atomic64_cmpxchg(v, old, new); | ||
100 | } | ||
101 | static inline void nvgpu_atomic64_sub(long x, nvgpu_atomic64_t *v) | ||
102 | { | ||
103 | __nvgpu_atomic64_sub(x, v); | ||
104 | } | ||
105 | static inline long nvgpu_atomic64_sub_return(long x, nvgpu_atomic64_t *v) | ||
106 | { | ||
107 | return __nvgpu_atomic64_sub_return(x, v); | ||
108 | } | ||
19 | 109 | ||
20 | #endif | 110 | #endif /* __NVGPU_ATOMIC_H__ */ |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/linux/atomic.h b/drivers/gpu/nvgpu/include/nvgpu/linux/atomic.h new file mode 100644 index 00000000..c6dd4650 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/linux/atomic.h | |||
@@ -0,0 +1,137 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | #ifndef __NVGPU_ATOMIC_LINUX_H__ | ||
17 | #define __NVGPU_ATOMIC_LINUX_H__ | ||
18 | |||
19 | #include <linux/atomic.h> | ||
20 | |||
21 | typedef struct nvgpu_atomic { | ||
22 | atomic_t atomic_var; | ||
23 | } nvgpu_atomic_t; | ||
24 | |||
25 | typedef struct nvgpu_atomic64 { | ||
26 | atomic64_t atomic_var; | ||
27 | } nvgpu_atomic64_t; | ||
28 | |||
29 | #define __nvgpu_atomic_init(i) { ATOMIC_INIT(i) } | ||
30 | #define __nvgpu_atomic64_init(i) { ATOMIC64_INIT(i) } | ||
31 | |||
32 | static inline void __nvgpu_atomic_set(nvgpu_atomic_t *v, int i) | ||
33 | { | ||
34 | atomic_set(&v->atomic_var, i); | ||
35 | } | ||
36 | |||
37 | static inline int __nvgpu_atomic_read(nvgpu_atomic_t *v) | ||
38 | { | ||
39 | return atomic_read(&v->atomic_var); | ||
40 | } | ||
41 | |||
42 | static inline void __nvgpu_atomic_inc(nvgpu_atomic_t *v) | ||
43 | { | ||
44 | atomic_inc(&v->atomic_var); | ||
45 | } | ||
46 | |||
47 | static inline int __nvgpu_atomic_inc_return(nvgpu_atomic_t *v) | ||
48 | { | ||
49 | return atomic_inc_return(&v->atomic_var); | ||
50 | } | ||
51 | |||
52 | static inline void __nvgpu_atomic_dec(nvgpu_atomic_t *v) | ||
53 | { | ||
54 | atomic_dec(&v->atomic_var); | ||
55 | } | ||
56 | |||
57 | static inline int __nvgpu_atomic_dec_return(nvgpu_atomic_t *v) | ||
58 | { | ||
59 | return atomic_dec_return(&v->atomic_var); | ||
60 | } | ||
61 | |||
62 | static inline int __nvgpu_atomic_cmpxchg(nvgpu_atomic_t *v, int old, int new) | ||
63 | { | ||
64 | return atomic_cmpxchg(&v->atomic_var, old, new); | ||
65 | } | ||
66 | |||
67 | static inline int __nvgpu_atomic_xchg(nvgpu_atomic_t *v, int new) | ||
68 | { | ||
69 | return atomic_xchg(&v->atomic_var, new); | ||
70 | } | ||
71 | |||
72 | static inline bool __nvgpu_atomic_inc_and_test(nvgpu_atomic_t *v) | ||
73 | { | ||
74 | return atomic_inc_and_test(&v->atomic_var); | ||
75 | } | ||
76 | |||
77 | static inline bool __nvgpu_atomic_dec_and_test(nvgpu_atomic_t *v) | ||
78 | { | ||
79 | return atomic_dec_and_test(&v->atomic_var); | ||
80 | } | ||
81 | |||
82 | static inline int __nvgpu_atomic_add_return(int i, nvgpu_atomic_t *v) | ||
83 | { | ||
84 | return atomic_add_return(i, &v->atomic_var); | ||
85 | } | ||
86 | |||
87 | static inline void __nvgpu_atomic64_set(nvgpu_atomic64_t *v, long i) | ||
88 | { | ||
89 | atomic64_set(&v->atomic_var, i); | ||
90 | } | ||
91 | |||
92 | static inline long __nvgpu_atomic64_read(nvgpu_atomic64_t *v) | ||
93 | { | ||
94 | return atomic64_read(&v->atomic_var); | ||
95 | } | ||
96 | |||
97 | static inline void __nvgpu_atomic64_add(long x, nvgpu_atomic64_t *v) | ||
98 | { | ||
99 | atomic64_add(x, &v->atomic_var); | ||
100 | } | ||
101 | |||
102 | static inline void __nvgpu_atomic64_inc(nvgpu_atomic64_t *v) | ||
103 | { | ||
104 | atomic64_inc(&v->atomic_var); | ||
105 | } | ||
106 | |||
107 | static inline long __nvgpu_atomic64_inc_return(nvgpu_atomic64_t *v) | ||
108 | { | ||
109 | return atomic64_inc_return(&v->atomic_var); | ||
110 | } | ||
111 | |||
112 | static inline void __nvgpu_atomic64_dec(nvgpu_atomic64_t *v) | ||
113 | { | ||
114 | atomic64_dec(&v->atomic_var); | ||
115 | } | ||
116 | |||
117 | static inline void __nvgpu_atomic64_dec_return(nvgpu_atomic64_t *v) | ||
118 | { | ||
119 | atomic64_dec_return(&v->atomic_var); | ||
120 | } | ||
121 | |||
122 | static inline long __nvgpu_atomic64_cmpxchg(nvgpu_atomic64_t *v, | ||
123 | long old, long new) | ||
124 | { | ||
125 | return atomic64_cmpxchg(&v->atomic_var, old, new); | ||
126 | } | ||
127 | |||
128 | static inline void __nvgpu_atomic64_sub(long x, nvgpu_atomic64_t *v) | ||
129 | { | ||
130 | atomic64_sub(x, &v->atomic_var); | ||
131 | } | ||
132 | |||
133 | static inline long __nvgpu_atomic64_sub_return(long x, nvgpu_atomic64_t *v) | ||
134 | { | ||
135 | return atomic64_sub_return(x, &v->atomic_var); | ||
136 | } | ||
137 | #endif /*__NVGPU_ATOMIC_LINUX_H__ */ | ||
diff --git a/drivers/gpu/nvgpu/include/nvgpu/semaphore.h b/drivers/gpu/nvgpu/include/nvgpu/semaphore.h index faa8d945..90261d81 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/semaphore.h +++ b/drivers/gpu/nvgpu/include/nvgpu/semaphore.h | |||
@@ -48,7 +48,7 @@ struct nvgpu_semaphore_sea; | |||
48 | struct nvgpu_semaphore_int { | 48 | struct nvgpu_semaphore_int { |
49 | int idx; /* Semaphore index. */ | 49 | int idx; /* Semaphore index. */ |
50 | u32 offset; /* Offset into the pool. */ | 50 | u32 offset; /* Offset into the pool. */ |
51 | atomic_t next_value; /* Next available value. */ | 51 | nvgpu_atomic_t next_value; /* Next available value. */ |
52 | u32 nr_incrs; /* Number of increments programmed. */ | 52 | u32 nr_incrs; /* Number of increments programmed. */ |
53 | struct nvgpu_semaphore_pool *p; /* Pool that owns this sema. */ | 53 | struct nvgpu_semaphore_pool *p; /* Pool that owns this sema. */ |
54 | struct channel_gk20a *ch; /* Channel that owns this sema. */ | 54 | struct channel_gk20a *ch; /* Channel that owns this sema. */ |
@@ -70,7 +70,7 @@ nvgpu_semaphore_int_from_hw_sema_list(struct nvgpu_list_node *node) | |||
70 | struct nvgpu_semaphore { | 70 | struct nvgpu_semaphore { |
71 | struct nvgpu_semaphore_int *hw_sema; | 71 | struct nvgpu_semaphore_int *hw_sema; |
72 | 72 | ||
73 | atomic_t value; | 73 | nvgpu_atomic_t value; |
74 | int incremented; | 74 | int incremented; |
75 | 75 | ||
76 | struct kref ref; | 76 | struct kref ref; |
@@ -242,7 +242,7 @@ static inline bool nvgpu_semaphore_is_released(struct nvgpu_semaphore *s) | |||
242 | * the value of the semaphore then the semaphore has been signaled | 242 | * the value of the semaphore then the semaphore has been signaled |
243 | * (a.k.a. released). | 243 | * (a.k.a. released). |
244 | */ | 244 | */ |
245 | return (int)sema_val >= atomic_read(&s->value); | 245 | return (int)sema_val >= nvgpu_atomic_read(&s->value); |
246 | } | 246 | } |
247 | 247 | ||
248 | static inline bool nvgpu_semaphore_is_acquired(struct nvgpu_semaphore *s) | 248 | static inline bool nvgpu_semaphore_is_acquired(struct nvgpu_semaphore *s) |
@@ -252,12 +252,12 @@ static inline bool nvgpu_semaphore_is_acquired(struct nvgpu_semaphore *s) | |||
252 | 252 | ||
253 | static inline u32 nvgpu_semaphore_get_value(struct nvgpu_semaphore *s) | 253 | static inline u32 nvgpu_semaphore_get_value(struct nvgpu_semaphore *s) |
254 | { | 254 | { |
255 | return (u32)atomic_read(&s->value); | 255 | return (u32)nvgpu_atomic_read(&s->value); |
256 | } | 256 | } |
257 | 257 | ||
258 | static inline u32 nvgpu_semaphore_next_value(struct nvgpu_semaphore *s) | 258 | static inline u32 nvgpu_semaphore_next_value(struct nvgpu_semaphore *s) |
259 | { | 259 | { |
260 | return (u32)atomic_read(&s->hw_sema->next_value); | 260 | return (u32)nvgpu_atomic_read(&s->hw_sema->next_value); |
261 | } | 261 | } |
262 | 262 | ||
263 | /* | 263 | /* |
@@ -320,7 +320,7 @@ static inline void nvgpu_semaphore_incr(struct nvgpu_semaphore *s) | |||
320 | { | 320 | { |
321 | BUG_ON(s->incremented); | 321 | BUG_ON(s->incremented); |
322 | 322 | ||
323 | atomic_set(&s->value, atomic_add_return(1, &s->hw_sema->next_value)); | 323 | nvgpu_atomic_set(&s->value, nvgpu_atomic_add_return(1, &s->hw_sema->next_value)); |
324 | s->incremented = 1; | 324 | s->incremented = 1; |
325 | 325 | ||
326 | gpu_sema_verbose_dbg(s->hw_sema->p->sema_sea->gk20a, | 326 | gpu_sema_verbose_dbg(s->hw_sema->p->sema_sea->gk20a, |
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c index f1ae2f1f..c8519905 100644 --- a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | |||
@@ -43,7 +43,7 @@ static void vgpu_channel_bind(struct channel_gk20a *ch) | |||
43 | WARN_ON(err || msg.ret); | 43 | WARN_ON(err || msg.ret); |
44 | 44 | ||
45 | wmb(); | 45 | wmb(); |
46 | atomic_set(&ch->bound, true); | 46 | nvgpu_atomic_set(&ch->bound, true); |
47 | } | 47 | } |
48 | 48 | ||
49 | static void vgpu_channel_unbind(struct channel_gk20a *ch) | 49 | static void vgpu_channel_unbind(struct channel_gk20a *ch) |
@@ -51,7 +51,7 @@ static void vgpu_channel_unbind(struct channel_gk20a *ch) | |||
51 | 51 | ||
52 | gk20a_dbg_fn(""); | 52 | gk20a_dbg_fn(""); |
53 | 53 | ||
54 | if (atomic_cmpxchg(&ch->bound, true, false)) { | 54 | if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) { |
55 | struct tegra_vgpu_cmd_msg msg; | 55 | struct tegra_vgpu_cmd_msg msg; |
56 | struct tegra_vgpu_channel_config_params *p = | 56 | struct tegra_vgpu_channel_config_params *p = |
57 | &msg.params.channel_config; | 57 | &msg.params.channel_config; |
@@ -425,7 +425,7 @@ static int vgpu_fifo_preempt_channel(struct gk20a *g, u32 chid) | |||
425 | 425 | ||
426 | gk20a_dbg_fn(""); | 426 | gk20a_dbg_fn(""); |
427 | 427 | ||
428 | if (!atomic_read(&ch->bound)) | 428 | if (!nvgpu_atomic_read(&ch->bound)) |
429 | return 0; | 429 | return 0; |
430 | 430 | ||
431 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT; | 431 | msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT; |