gpu: nvgpu: Add wrapper over atomic_t and atomic64_t

- added wrapper structs nvgpu_atomic_t and nvgpu_atomic64_t over
  atomic_t and atomic64_t
- added nvgpu_atomic_* and nvgpu_atomic64_* APIs to access the above
  wrappers.

JIRA NVGPU-121

Change-Id: I61667bb0a84c2fc475365abb79bffb42b8b4786a
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1533044
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
GVS: Gerrit_Virtual_Submit

12 files changed, 51 insertions, 46 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index 62b312b2..d96872f3 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -100,7 +100,7 @@ static struct channel_gk20a *allocate_channel(struct fifo_gk20a *f)
                 ch = nvgpu_list_first_entry(&f->free_chs, channel_gk20a,
                                                           free_chs);
                 nvgpu_list_del(&ch->free_chs);
-                WARN_ON(atomic_read(&ch->ref_count));
+                WARN_ON(nvgpu_atomic_read(&ch->ref_count));
                 WARN_ON(ch->referenceable);
                 f->used_channels++;
         }
@@ -394,20 +394,20 @@ void gk20a_set_error_notifier(struct channel_gk20a *ch, __u32 error)
 }
 
 static void gk20a_wait_until_counter_is_N(
-        struct channel_gk20a *ch, atomic_t *counter, int wait_value,
+        struct channel_gk20a *ch, nvgpu_atomic_t *counter, int wait_value,
         struct nvgpu_cond *c, const char *caller, const char *counter_name)
 {
         while (true) {
                 if (NVGPU_COND_WAIT(
                             c,
-                            atomic_read(counter) == wait_value,
+                            nvgpu_atomic_read(counter) == wait_value,
                             5000) == 0)
                         break;
 
                 nvgpu_warn(ch->g,
                            "%s: channel %d, still waiting, %s left: %d, waiting for: %d",
                            caller, ch->chid, counter_name,
-                           atomic_read(counter), wait_value);
+                           nvgpu_atomic_read(counter), wait_value);
 
                 gk20a_channel_dump_ref_actions(ch);
         }
@@ -491,7 +491,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force)
         nvgpu_spinlock_release(&ch->ref_obtain_lock);
 
         /* matches with the initial reference in gk20a_open_new_channel() */
-        atomic_dec(&ch->ref_count);
+        nvgpu_atomic_dec(&ch->ref_count);
 
         /* wait until no more refs to the channel */
         if (!force)
@@ -635,7 +635,7 @@ static void gk20a_channel_dump_ref_actions(struct channel_gk20a *ch)
         nvgpu_spinlock_acquire(&ch->ref_actions_lock);
 
         dev_info(dev, "ch %d: refs %d. Actions, most recent last:\n",
-                        ch->chid, atomic_read(&ch->ref_count));
+                        ch->chid, nvgpu_atomic_read(&ch->ref_count));
 
         /* start at the oldest possible entry. put is next insertion point */
         get = ch->ref_actions_put;
@@ -709,7 +709,7 @@ struct channel_gk20a *_gk20a_channel_get(struct channel_gk20a *ch,
 
         if (likely(ch->referenceable)) {
                 gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_get);
-                atomic_inc(&ch->ref_count);
+                nvgpu_atomic_inc(&ch->ref_count);
                 ret = ch;
         } else
                 ret = NULL;
@@ -726,17 +726,17 @@ void _gk20a_channel_put(struct channel_gk20a *ch, const char *caller)
 {
         gk20a_channel_save_ref_source(ch, channel_gk20a_ref_action_put);
         trace_gk20a_channel_put(ch->chid, caller);
-        atomic_dec(&ch->ref_count);
+        nvgpu_atomic_dec(&ch->ref_count);
         nvgpu_cond_broadcast(&ch->ref_count_dec_wq);
 
         /* More puts than gets. Channel is probably going to get
          * stuck. */
-        WARN_ON(atomic_read(&ch->ref_count) < 0);
+        WARN_ON(nvgpu_atomic_read(&ch->ref_count) < 0);
 
         /* Also, more puts than gets. ref_count can go to 0 only if
          * the channel is closing. Channel is probably going to get
          * stuck. */
-        WARN_ON(atomic_read(&ch->ref_count) == 0 && ch->referenceable);
+        WARN_ON(nvgpu_atomic_read(&ch->ref_count) == 0 && ch->referenceable);
 }
 
 void gk20a_channel_close(struct channel_gk20a *ch)
@@ -879,7 +879,7 @@ struct channel_gk20a *gk20a_open_new_channel(struct gk20a *g,
          * references. The initial reference will be decreased in
          * gk20a_free_channel() */
         ch->referenceable = true;
-        atomic_set(&ch->ref_count, 1);
+        nvgpu_atomic_set(&ch->ref_count, 1);
         wmb();
 
         return ch;
@@ -1745,7 +1745,7 @@ static int __gk20a_channel_worker_wakeup(struct gk20a *g)
          * pair.
          */
 
-        put = atomic_inc_return(&g->channel_worker.put);
+        put = nvgpu_atomic_inc_return(&g->channel_worker.put);
         nvgpu_cond_signal(&g->channel_worker.wq);
 
         return put;
@@ -1761,7 +1761,7 @@ static int __gk20a_channel_worker_wakeup(struct gk20a *g)
  */
 static bool __gk20a_channel_worker_pending(struct gk20a *g, int get)
 {
-        bool pending = atomic_read(&g->channel_worker.put) != get;
+        bool pending = nvgpu_atomic_read(&g->channel_worker.put) != get;
 
         /*
          * This would be the place for a rmb() pairing a wmb() for a wakeup
@@ -1864,7 +1864,7 @@ int nvgpu_channel_worker_init(struct gk20a *g)
         int err;
         char thread_name[64];
 
-        atomic_set(&g->channel_worker.put, 0);
+        nvgpu_atomic_set(&g->channel_worker.put, 0);
         nvgpu_cond_init(&g->channel_worker.wq);
         nvgpu_init_list_node(&g->channel_worker.items);
         nvgpu_spinlock_init(&g->channel_worker.items_lock);
@@ -2086,7 +2086,8 @@ static void gk20a_channel_clean_up_jobs(struct channel_gk20a *c,
 
                         if (g->aggressive_sync_destroy_thresh) {
                                 nvgpu_mutex_acquire(&c->sync_lock);
-                                if (atomic_dec_and_test(&c->sync->refcount) &&
+                                if (nvgpu_atomic_dec_and_test(
+                                        &c->sync->refcount) &&
                                                 g->aggressive_sync_destroy) {
                                         gk20a_channel_sync_destroy(c->sync);
                                         c->sync = NULL;
@@ -2321,7 +2322,7 @@ static int gk20a_submit_prepare_syncs(struct channel_gk20a *c,
                         }
                         new_sync_created = true;
                 }
-                atomic_inc(&c->sync->refcount);
+                nvgpu_atomic_inc(&c->sync->refcount);
                 nvgpu_mutex_release(&c->sync_lock);
         }
 
@@ -2774,9 +2775,9 @@ int gk20a_init_channel_support(struct gk20a *g, u32 chid)
 
         c->g = NULL;
         c->chid = chid;
-        atomic_set(&c->bound, false);
+        nvgpu_atomic_set(&c->bound, false);
         nvgpu_spinlock_init(&c->ref_obtain_lock);
-        atomic_set(&c->ref_count, 0);
+        nvgpu_atomic_set(&c->ref_count, 0);
         c->referenceable = false;
         nvgpu_cond_init(&c->ref_count_dec_wq);
 
@@ -2935,7 +2936,7 @@ void gk20a_channel_semaphore_wakeup(struct gk20a *g, bool post_events)
         for (chid = 0; chid < f->num_channels; chid++) {
                 struct channel_gk20a *c = g->fifo.channel+chid;
                 if (gk20a_channel_get(c)) {
-                        if (atomic_read(&c->bound)) {
+                        if (nvgpu_atomic_read(&c->bound)) {
                                 nvgpu_cond_broadcast_interruptible(
                                                 &c->semaphore_wq);
                                 if (post_events) {
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
index a9ccd93f..f022e630 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h
@@ -24,6 +24,7 @@
 #include <nvgpu/lock.h>
 #include <nvgpu/timers.h>
 #include <nvgpu/cond.h>
+#include <nvgpu/atomic.h>
 
 struct gk20a;
 struct gr_gk20a;
@@ -173,7 +174,7 @@ struct channel_gk20a {
 
         struct nvgpu_spinlock ref_obtain_lock;
         bool referenceable;
-        atomic_t ref_count;
+        nvgpu_atomic_t ref_count;
         struct nvgpu_cond ref_count_dec_wq;
 #if GK20A_CHANNEL_REFCOUNT_TRACKING
         /*
@@ -191,7 +192,7 @@ struct channel_gk20a {
 
         int chid;
         bool wdt_enabled;
-        atomic_t bound;
+        nvgpu_atomic_t bound;
         bool first_init;
         bool vpr;
         bool deterministic;
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
index c9c03d37..aa340ba6 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.c
@@ -346,7 +346,7 @@ gk20a_channel_syncpt_create(struct channel_gk20a *c)
 
         nvgpu_nvhost_syncpt_set_min_eq_max_ext(sp->nvhost_dev, sp->id);
 
-        atomic_set(&sp->ops.refcount, 0);
+        nvgpu_atomic_set(&sp->ops.refcount, 0);
         sp->ops.wait_syncpt             = gk20a_channel_syncpt_wait_syncpt;
         sp->ops.wait_fd                 = gk20a_channel_syncpt_wait_fd;
         sp->ops.incr                    = gk20a_channel_syncpt_incr;
@@ -619,7 +619,7 @@ static int __semaphore_wait_fd_fast_path(struct channel_gk20a *c,
                 return err;
 
         nvgpu_semaphore_get(sema);
-        BUG_ON(!atomic_read(&sema->value));
+        BUG_ON(!nvgpu_atomic_read(&sema->value));
         add_sema_cmd(c->g, c, sema, wait_cmd, 8, true, false);
 
         /*
@@ -922,7 +922,7 @@ gk20a_channel_semaphore_create(struct channel_gk20a *c)
                 return NULL;
         }
 #endif
-        atomic_set(&sema->ops.refcount, 0);
+        nvgpu_atomic_set(&sema->ops.refcount, 0);
         sema->ops.wait_syncpt   = gk20a_channel_semaphore_wait_syncpt;
         sema->ops.wait_fd       = gk20a_channel_semaphore_wait_fd;
         sema->ops.incr          = gk20a_channel_semaphore_incr;
diff --git a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h
index 4efd1b76..9bdc5d12 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/channel_sync_gk20a.h
@@ -25,7 +25,7 @@ struct gk20a_fence;
 struct gk20a;
 
 struct gk20a_channel_sync {
-        atomic_t refcount;
+        nvgpu_atomic_t refcount;
 
         /* Generate a gpu wait cmdbuf from syncpoint.
          * Returns
diff --git a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
index cc05ceff..546917f1 100644
--- a/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ctxsw_trace_gk20a.c
@@ -47,7 +47,7 @@ struct gk20a_ctxsw_dev {
         size_t size;
         u32 num_ents;
 
-        atomic_t vma_ref;
+        nvgpu_atomic_t vma_ref;
 
         struct nvgpu_mutex write_lock;
 };
@@ -152,7 +152,7 @@ static int gk20a_ctxsw_dev_alloc_buffer(struct gk20a_ctxsw_dev *dev,
         void *buf;
         int err;
 
-        if ((dev->write_enabled) || (atomic_read(&dev->vma_ref)))
+        if ((dev->write_enabled) || (nvgpu_atomic_read(&dev->vma_ref)))
                 return -EBUSY;
 
         err = g->ops.fecs_trace.alloc_user_buffer(g, &buf, &size);
@@ -438,18 +438,18 @@ static void gk20a_ctxsw_dev_vma_open(struct vm_area_struct *vma)
 {
         struct gk20a_ctxsw_dev *dev = vma->vm_private_data;
 
-        atomic_inc(&dev->vma_ref);
+        nvgpu_atomic_inc(&dev->vma_ref);
         gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d",
-                atomic_read(&dev->vma_ref));
+                nvgpu_atomic_read(&dev->vma_ref));
 }
 
 static void gk20a_ctxsw_dev_vma_close(struct vm_area_struct *vma)
 {
         struct gk20a_ctxsw_dev *dev = vma->vm_private_data;
 
-        atomic_dec(&dev->vma_ref);
+        nvgpu_atomic_dec(&dev->vma_ref);
         gk20a_dbg(gpu_dbg_fn|gpu_dbg_ctxsw, "vma_ref=%d",
-                atomic_read(&dev->vma_ref));
+                nvgpu_atomic_read(&dev->vma_ref));
 }
 
 static struct vm_operations_struct gk20a_ctxsw_dev_vma_ops = {
@@ -497,7 +497,7 @@ static int gk20a_ctxsw_init_devs(struct gk20a *g)
                 err = nvgpu_mutex_init(&dev->write_lock);
                 if (err)
                         return err;
-                atomic_set(&dev->vma_ref, 0);
+                nvgpu_atomic_set(&dev->vma_ref, 0);
                 dev++;
         }
         return 0;
diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
index 1572ff48..00050850 100644
--- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c
@@ -26,6 +26,7 @@
 #include <nvgpu/kmem.h>
 #include <nvgpu/log.h>
 #include <nvgpu/vm.h>
+#include <nvgpu/atomic.h>
 
 #include "gk20a.h"
 #include "gk20a/platform_gk20a.h"
@@ -74,10 +75,10 @@ nvgpu_dbg_gpu_get_session_channel(struct dbg_session_gk20a *dbg_s)
 }
 
 /* silly allocator - just increment id */
-static atomic_t unique_id = ATOMIC_INIT(0);
+static nvgpu_atomic_t unique_id = NVGPU_ATOMIC_INIT(0);
 static int generate_unique_id(void)
 {
-        return atomic_add_return(1, &unique_id);
+        return nvgpu_atomic_add_return(1, &unique_id);
 }
 
 static int alloc_session(struct gk20a *g, struct dbg_session_gk20a **_dbg_s)
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index abd455d7..47e7d82e 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -3439,7 +3439,7 @@ void gk20a_dump_channel_status_ramfc(struct gk20a *g,
                 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x "
                                    "next_val: 0x%08x addr: 0x%010llx\n",
                                    __nvgpu_semaphore_read(hw_sema),
-                                   atomic_read(&hw_sema->next_value),
+                                   nvgpu_atomic_read(&hw_sema->next_value),
                                    nvgpu_hw_sema_addr(hw_sema));
 
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
@@ -3489,7 +3489,7 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
                         continue;
 
                 ch_state[chid]->pid = ch->pid;
-                ch_state[chid]->refs = atomic_read(&ch->ref_count);
+                ch_state[chid]->refs = nvgpu_atomic_read(&ch->ref_count);
                 ch_state[chid]->deterministic = ch->deterministic;
                 nvgpu_mem_rd_n(g, &ch->inst_block, 0,
                                 &ch_state[chid]->inst_block[0],
@@ -3591,7 +3591,7 @@ void gk20a_fifo_channel_unbind(struct channel_gk20a *ch_gk20a)
 
         gk20a_dbg_fn("");
 
-        if (atomic_cmpxchg(&ch_gk20a->bound, true, false)) {
+        if (nvgpu_atomic_cmpxchg(&ch_gk20a->bound, true, false)) {
                 gk20a_writel(g, ccsr_channel_inst_r(ch_gk20a->chid),
                         ccsr_channel_inst_ptr_f(0) |
                         ccsr_channel_inst_bind_false_f());
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
index b19a7b68..a6eae8ca 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h
@@ -156,7 +156,7 @@ struct fifo_gk20a {
 #ifdef CONFIG_DEBUG_FS
         struct {
                 struct fifo_profile_gk20a *data;
-                atomic_t get;
+                nvgpu_atomic_t get;
                 bool enabled;
                 u64 *sorted;
                 struct kref ref;
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c
index c50d800f..550b22c0 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.c
@@ -373,13 +373,13 @@ int gk20a_wait_for_idle(struct gk20a *g)
         if (g->user_railgate_disabled)
                 target_usage_count = 1;
 
-        while ((atomic_read(&g->usage_count) != target_usage_count)
+        while ((nvgpu_atomic_read(&g->usage_count) != target_usage_count)
                         && (wait_length-- >= 0))
                 nvgpu_msleep(20);
 
         if (wait_length < 0) {
                 pr_warn("%s: Timed out waiting for idle (%d)!\n",
-                        __func__, atomic_read(&g->usage_count));
+                        __func__, nvgpu_atomic_read(&g->usage_count));
                 return -ETIMEDOUT;
         }
 
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 4878fdd6..47fd3aef 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -48,6 +48,7 @@ struct nvgpu_cpu_time_correlation_sample;
 #include <nvgpu/kref.h>
 #include <nvgpu/falcon.h>
 #include <nvgpu/pmu.h>
+#include <nvgpu/atomic.h>
 
 #include "clk_gk20a.h"
 #include "ce2_gk20a.h"
@@ -1038,7 +1039,7 @@ struct gk20a {
          */
         unsigned long *enabled_flags;
 
-        atomic_t usage_count;
+        nvgpu_atomic_t usage_count;
 
         struct kref refcount;
 
@@ -1205,7 +1206,7 @@ struct gk20a {
 
         struct gk20a_channel_worker {
                 struct nvgpu_thread poll_task;
-                atomic_t put;
+                nvgpu_atomic_t put;
                 struct nvgpu_cond wq;
                 struct nvgpu_list_node items;
                 struct nvgpu_spinlock items_lock;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index 16fe7149..e21be1e5 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -617,7 +617,7 @@ static int gk20a_init_vidmem(struct mm_gk20a *mm)
         nvgpu_mutex_init(&mm->vidmem.first_clear_mutex);
 
         INIT_WORK(&mm->vidmem.clear_mem_worker, gk20a_vidmem_clear_mem_worker);
-        atomic64_set(&mm->vidmem.bytes_pending, 0);
+        nvgpu_atomic64_set(&mm->vidmem.bytes_pending, 0);
         nvgpu_init_list_node(&mm->vidmem.clear_list_head);
         nvgpu_mutex_init(&mm->vidmem.clear_list_mutex);
 
@@ -1165,7 +1165,7 @@ int gk20a_vidmem_get_space(struct gk20a *g, u64 *space)
 
         nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
         *space = nvgpu_alloc_space(allocator) +
-                atomic64_read(&g->mm.vidmem.bytes_pending);
+                nvgpu_atomic64_read(&g->mm.vidmem.bytes_pending);
         nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
         return 0;
 #else
@@ -1483,7 +1483,7 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work)
                            (u64)get_vidmem_page_alloc(mem->priv.sgt->sgl));
                 nvgpu_free_sgtable(g, &mem->priv.sgt);
 
-                WARN_ON(atomic64_sub_return(mem->size,
+                WARN_ON(nvgpu_atomic64_sub_return(mem->size,
                                         &g->mm.vidmem.bytes_pending) < 0);
                 mem->size = 0;
                 mem->aperture = APERTURE_INVALID;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index 7b2c0dfc..af176a73 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -30,6 +30,7 @@
 #include <nvgpu/list.h>
 #include <nvgpu/rbtree.h>
 #include <nvgpu/kref.h>
+#include <nvgpu/atomic.h>
 
 struct nvgpu_pd_cache;
 
@@ -283,7 +284,7 @@ struct mm_gk20a {
                 struct nvgpu_mutex clear_list_mutex;
 
                 struct work_struct clear_mem_worker;
-                atomic64_t bytes_pending;
+                nvgpu_atomic64_t bytes_pending;
         } vidmem;
 };