1 files changed, 80 insertions, 42 deletions

diff --git a/include/os/linux/ioctl_ctrl.c b/include/os/linux/ioctl_ctrl.c
index ee141ff..841d345 100644
--- a/include/os/linux/ioctl_ctrl.c
+++ b/include/os/linux/ioctl_ctrl.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011-2020, NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2011-2021, NVIDIA Corporation.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -60,7 +60,6 @@ struct gk20a_ctrl_priv {
         struct nvgpu_list_node list;
         struct {
                 struct vm_area_struct *vma;
-                unsigned long flags;
                 bool vma_mapped;
         } usermode_vma;
 };
@@ -488,27 +487,26 @@ static int gk20a_ctrl_alloc_as(
 
         snprintf(name, sizeof(name), "nvhost-%s-fd%d", g->name, fd);
 
-        file = anon_inode_getfile(name, l->as_dev.cdev.ops, NULL, O_RDWR);
-        if (IS_ERR(file)) {
-                err = PTR_ERR(file);
-                goto clean_up;
-        }
-
         err = gk20a_as_alloc_share(g, args->big_page_size,
                                    gk20a_as_translate_as_alloc_flags(g,
                                            args->flags),
                                    &as_share);
         if (err)
-                goto clean_up_file;
+                goto clean_up;
+
+        file = anon_inode_getfile(name, l->as_dev.cdev.ops, as_share, O_RDWR);
+        if (IS_ERR(file)) {
+                err = PTR_ERR(file);
+                goto clean_up_as;
+        }
 
         fd_install(fd, file);
-        file->private_data = as_share;
 
         args->as_fd = fd;
         return 0;
 
-clean_up_file:
-        fput(file);
+clean_up_as:
+        gk20a_as_release_share(as_share);
 clean_up:
         put_unused_fd(fd);
         return err;
@@ -692,12 +690,15 @@ static int nvgpu_gpu_ioctl_trigger_suspend(struct gk20a *g)
 
         err = gk20a_busy(g);
         if (err)
-            return err;
+                return err;
 
-        nvgpu_mutex_acquire(&g->dbg_sessions_lock);
-        err = gr_gk20a_elpg_protected_call(g,
+        if (g->ops.gr.trigger_suspend) {
+                nvgpu_mutex_acquire(&g->dbg_sessions_lock);
+                err = gr_gk20a_elpg_protected_call(g,
                         g->ops.gr.trigger_suspend(g));
-        nvgpu_mutex_release(&g->dbg_sessions_lock);
+                nvgpu_mutex_release(&g->dbg_sessions_lock);
+        } else
+                err = -EINVAL;
 
         gk20a_idle(g);
 
@@ -731,8 +732,13 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
                 goto out_free;
 
         nvgpu_mutex_acquire(&g->dbg_sessions_lock);
-        (void)gr_gk20a_elpg_protected_call(g,
+        if (g->ops.gr.wait_for_pause) {
+                (void)gr_gk20a_elpg_protected_call(g,
                         g->ops.gr.wait_for_pause(g, w_state));
+        } else {
+                err = -EINVAL;
+                goto out_idle;
+        }
 
         for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
                 ioctl_w_state[sm_id].valid_warps[0] =
@@ -755,6 +761,7 @@ static int nvgpu_gpu_ioctl_wait_for_pause(struct gk20a *g,
                 err = -EFAULT;
         }
 
+out_idle:
         nvgpu_mutex_release(&g->dbg_sessions_lock);
 
         gk20a_idle(g);
@@ -772,12 +779,15 @@ static int nvgpu_gpu_ioctl_resume_from_pause(struct gk20a *g)
 
         err = gk20a_busy(g);
         if (err)
-            return err;
+                return err;
 
-        nvgpu_mutex_acquire(&g->dbg_sessions_lock);
-        err = gr_gk20a_elpg_protected_call(g,
+        if (g->ops.gr.resume_from_pause) {
+                nvgpu_mutex_acquire(&g->dbg_sessions_lock);
+                err = gr_gk20a_elpg_protected_call(g,
                         g->ops.gr.resume_from_pause(g));
-        nvgpu_mutex_release(&g->dbg_sessions_lock);
+                nvgpu_mutex_release(&g->dbg_sessions_lock);
+        } else
+                err = -EINVAL;
 
         gk20a_idle(g);
 
@@ -792,8 +802,11 @@ static int nvgpu_gpu_ioctl_clear_sm_errors(struct gk20a *g)
         if (err)
                 return err;
 
-        err = gr_gk20a_elpg_protected_call(g,
+        if (g->ops.gr.clear_sm_errors) {
+                err = gr_gk20a_elpg_protected_call(g,
                         g->ops.gr.clear_sm_errors(g));
+        } else
+                err = -EINVAL;
 
         gk20a_idle(g);
 
@@ -806,9 +819,12 @@ static int nvgpu_gpu_ioctl_has_any_exception(
 {
         u32 tpc_exception_en;
 
-        nvgpu_mutex_acquire(&g->dbg_sessions_lock);
-        tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
-        nvgpu_mutex_release(&g->dbg_sessions_lock);
+        if (g->ops.gr.tpc_enabled_exceptions) {
+                nvgpu_mutex_acquire(&g->dbg_sessions_lock);
+                tpc_exception_en = g->ops.gr.tpc_enabled_exceptions(g);
+                nvgpu_mutex_release(&g->dbg_sessions_lock);
+        } else
+                return -EINVAL;
 
         args->tpc_exception_en_sm_mask = tpc_exception_en;
 
@@ -2023,7 +2039,6 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
                         vma->vm_end - vma->vm_start, vma->vm_page_prot);
         if (!err) {
                 priv->usermode_vma.vma = vma;
-                priv->usermode_vma.flags = vma->vm_flags;
                 vma->vm_private_data = priv;
                 priv->usermode_vma.vma_mapped = true;
         }
@@ -2034,7 +2049,7 @@ int gk20a_ctrl_dev_mmap(struct file *filp, struct vm_area_struct *vma)
         return err;
 }
 
-static void alter_usermode_mapping(struct gk20a *g,
+static int alter_usermode_mapping(struct gk20a *g,
                 struct gk20a_ctrl_priv *priv,
                 bool poweroff)
 {
@@ -2042,57 +2057,80 @@ static void alter_usermode_mapping(struct gk20a *g,
         struct vm_area_struct *vma = priv->usermode_vma.vma;
         bool vma_mapped = priv->usermode_vma.vma_mapped;
         u64 addr;
-        int err;
+        int err = 0;
 
         if (!vma) {
                 /* Nothing to do - no mmap called */
-                return;
+                return 0;
         }
 
         addr = l->regs_bus_addr + g->ops.fifo.usermode_base(g);
 
-        down_write(&vma->vm_mm->mmap_sem);
-
         /*
          * This is a no-op for the below cases
          * a) poweroff and !vma_mapped - > do nothing as no map exists
          * b) !poweroff and vmap_mapped -> do nothing as already mapped
          */
-        if (poweroff && vma_mapped) {
+        if (poweroff != vma_mapped) {
+                return 0;
+        }
+
+        /*
+         * We use trylock due to lock inversion: we need to acquire
+         * mmap_lock while holding ctrl_privs_lock. usermode_vma_close
+         * does it in reverse order. Trylock is a way to avoid deadlock.
+         */
+        if (!down_write_trylock(&vma->vm_mm->mmap_sem)) {
+                return -EBUSY;
+        }
+
+        if (poweroff) {
                 err = zap_vma_ptes(vma, vma->vm_start, SZ_4K);
                 if (err == 0) {
-                        vma->vm_flags = VM_NONE;
                         priv->usermode_vma.vma_mapped = false;
                 } else {
                         nvgpu_err(g, "can't remove usermode mapping");
                 }
-        } else if (!poweroff && !vma_mapped) {
-                vma->vm_flags = priv->usermode_vma.flags;
+        } else {
                 err = io_remap_pfn_range(vma, vma->vm_start,
                                 addr >> PAGE_SHIFT,
                                 SZ_4K, vma->vm_page_prot);
                 if (err != 0) {
                         nvgpu_err(g, "can't restore usermode mapping");
-                        vma->vm_flags = VM_NONE;
                 } else {
                         priv->usermode_vma.vma_mapped = true;
                 }
         }
 
         up_write(&vma->vm_mm->mmap_sem);
+
+        return err;
 }
 
 static void alter_usermode_mappings(struct gk20a *g, bool poweroff)
 {
         struct gk20a_ctrl_priv *priv;
         struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+        int err = 0;
 
-        nvgpu_mutex_acquire(&l->ctrl.privs_lock);
-        nvgpu_list_for_each_entry(priv, &l->ctrl.privs,
-                        gk20a_ctrl_priv, list) {
-                alter_usermode_mapping(g, priv, poweroff);
-        }
-        nvgpu_mutex_release(&l->ctrl.privs_lock);
+        do {
+                nvgpu_mutex_acquire(&l->ctrl.privs_lock);
+                nvgpu_list_for_each_entry(priv, &l->ctrl.privs,
+                                gk20a_ctrl_priv, list) {
+                        err = alter_usermode_mapping(g, priv, poweroff);
+                        if (err != 0) {
+                                break;
+                        }
+                }
+                nvgpu_mutex_release(&l->ctrl.privs_lock);
+
+                if (err == -EBUSY) {
+                        nvgpu_log_info(g, "ctrl_privs_lock lock contended. retry altering usermode mappings");
+                        nvgpu_udelay(10);
+                } else if (err != 0) {
+                        nvgpu_err(g, "can't alter usermode mapping. err = %d", err);
+                }
+        } while (err == -EBUSY);
 }
 
 void nvgpu_hide_usermode_for_poweroff(struct gk20a *g)