gpu: nvgpu: suspend/resume support for vGPU

- Added suspend/resume power management callbacks for vGPU
- Added suspend/resume commands for communication between vGPU and
  RM server
- Added suspend/resume message parameters for IVC messages between
  vGPU and RM server

JIRA EVLR-2305
JIRA EVLR-2306

Change-Id: I83a314b4e125a53117d16c5ea72dbc5d8ef96ef7
Signed-off-by: Deepak Bhosale <dbhosale@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1735153
Reviewed-by: Automatic_Commit_Validation_User
Reviewed-by: Richard Zhao <rizhao@nvidia.com>
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Nirav Patel <nipatel@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

7 files changed, 90 insertions, 8 deletions

diff --git a/drivers/gpu/nvgpu/include/nvgpu/vgpu/tegra_vgpu.h b/drivers/gpu/nvgpu/include/nvgpu/vgpu/tegra_vgpu.h
index 3e4372eb..6d05e44c 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/vgpu/tegra_vgpu.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/vgpu/tegra_vgpu.h
@@ -118,6 +118,8 @@ enum {
         TEGRA_VGPU_CMD_MAP_SYNCPT = 79,
         TEGRA_VGPU_CMD_TSG_BIND_CHANNEL_EX = 80,
         TEGRA_VGPU_CMD_UPDATE_PC_SAMPLING = 81,
+        TEGRA_VGPU_CMD_SUSPEND = 82,
+        TEGRA_VGPU_CMD_RESUME = 83,
 };
 
 struct tegra_vgpu_connect_params {
diff --git a/drivers/gpu/nvgpu/os/linux/module.c b/drivers/gpu/nvgpu/os/linux/module.c
index af71cc81..4516490b 100644
--- a/drivers/gpu/nvgpu/os/linux/module.c
+++ b/drivers/gpu/nvgpu/os/linux/module.c
@@ -967,9 +967,15 @@ static int gk20a_pm_suspend(struct device *dev)
         int idle_usage_count = 0;
 
         if (!g->power_on) {
+                if (platform->suspend)
+                        ret = platform->suspend(dev);
+                if (ret)
+                        return ret;
+
                 if (!pm_runtime_enabled(dev))
-                        gk20a_pm_railgate(dev);
-                return 0;
+                        ret = gk20a_pm_railgate(dev);
+
+                return ret;
         }
 
         if (nvgpu_atomic_read(&g->usage_count) > idle_usage_count)
@@ -980,25 +986,41 @@ static int gk20a_pm_suspend(struct device *dev)
                 return ret;
 
         if (platform->suspend)
-                platform->suspend(dev);
+                ret = platform->suspend(dev);
+        if (ret)
+                return ret;
 
         g->suspended = true;
 
-        return 0;
+        return ret;
 }
 
 static int gk20a_pm_resume(struct device *dev)
 {
+        struct gk20a_platform *platform = dev_get_drvdata(dev);
         struct gk20a *g = get_gk20a(dev);
         int ret = 0;
 
         if (!g->suspended) {
+                if (platform->resume)
+                        ret = platform->resume(dev);
+                if (ret)
+                        return ret;
+
                 if (!pm_runtime_enabled(dev))
-                        gk20a_pm_unrailgate(dev);
-                return 0;
+                        ret = gk20a_pm_unrailgate(dev);
+
+                return ret;
         }
 
+        if (platform->resume)
+                ret = platform->resume(dev);
+        if (ret)
+                return ret;
+
         ret = gk20a_pm_runtime_resume(dev);
+        if (ret)
+                return ret;
 
         g->suspended = false;
 
diff --git a/drivers/gpu/nvgpu/os/linux/platform_gk20a.h b/drivers/gpu/nvgpu/os/linux/platform_gk20a.h
index 9a99b7fe..d9725e4c 100644
--- a/drivers/gpu/nvgpu/os/linux/platform_gk20a.h
+++ b/drivers/gpu/nvgpu/os/linux/platform_gk20a.h
@@ -158,6 +158,9 @@ struct gk20a_platform {
         /* Device is going to be suspended */
         int (*suspend)(struct device *);
 
+        /* Device is going to be resumed */
+        int (*resume)(struct device *);
+
         /* Called to turn off the device */
         int (*railgate)(struct device *dev);
 
diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c b/drivers/gpu/nvgpu/os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c
index 66911626..8eada9df 100644
--- a/drivers/gpu/nvgpu/os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c
+++ b/drivers/gpu/nvgpu/os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c
@@ -22,6 +22,7 @@
 #include "os/linux/vgpu/clk_vgpu.h"
 #include "os/linux/platform_gk20a.h"
 #include "os/linux/os_linux.h"
+#include "os/linux/vgpu/vgpu_linux.h"
 
 static int gv11b_vgpu_probe(struct device *dev)
 {
@@ -94,4 +95,8 @@ struct gk20a_platform gv11b_vgpu_tegra_platform = {
         .devfreq_governor = "userspace",
 
         .virtual_dev = true,
+
+        /* power management callbacks */
+        .suspend = vgpu_tegra_suspend,
+        .resume = vgpu_tegra_resume,
 };
diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/platform_vgpu_tegra.c b/drivers/gpu/nvgpu/os/linux/vgpu/platform_vgpu_tegra.c
index e4819e7d..44879a45 100644
--- a/drivers/gpu/nvgpu/os/linux/vgpu/platform_vgpu_tegra.c
+++ b/drivers/gpu/nvgpu/os/linux/vgpu/platform_vgpu_tegra.c
@@ -1,7 +1,7 @@
 /*
  * Tegra Virtualized GPU Platform Interface
  *
- * Copyright (c) 2014-2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -21,6 +21,7 @@
 #include "gk20a/gk20a.h"
 #include "os/linux/platform_gk20a.h"
 #include "clk_vgpu.h"
+#include "vgpu_linux.h"
 
 static int gk20a_tegra_probe(struct device *dev)
 {
@@ -66,4 +67,8 @@ struct gk20a_platform vgpu_tegra_platform = {
         .devfreq_governor = "userspace",
 
         .virtual_dev = true,
+
+        /* power management callbacks */
+        .suspend = vgpu_tegra_suspend,
+        .resume = vgpu_tegra_resume,
 };
diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c
index a7612e54..25024e7d 100644
--- a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c
+++ b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c
@@ -147,7 +147,9 @@ int vgpu_pm_prepare_poweroff(struct device *dev)
         if (!g->power_on)
                 return 0;
 
-        ret = gk20a_channel_suspend(g);
+        if (g->ops.fifo.channel_suspend)
+                ret = g->ops.fifo.channel_suspend(g);
+
         if (ret)
                 return ret;
 
@@ -473,3 +475,35 @@ bool vgpu_is_reduced_bar1(struct gk20a *g)
 
         return resource_size(l->bar1_mem) == (resource_size_t)f->userd.size;
 }
+
+int vgpu_tegra_suspend(struct device *dev)
+{
+        struct tegra_vgpu_cmd_msg msg = {};
+        struct gk20a *g = get_gk20a(dev);
+        int err = 0;
+
+        msg.cmd = TEGRA_VGPU_CMD_SUSPEND;
+        msg.handle = vgpu_get_handle(g);
+        err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+        err = err ? err : msg.ret;
+        if (err)
+                nvgpu_err(g, "vGPU suspend failed\n");
+
+        return err;
+}
+
+int vgpu_tegra_resume(struct device *dev)
+{
+        struct tegra_vgpu_cmd_msg msg = {};
+        struct gk20a *g = get_gk20a(dev);
+        int err = 0;
+
+        msg.cmd = TEGRA_VGPU_CMD_RESUME;
+        msg.handle = vgpu_get_handle(g);
+        err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+        err = err ? err : msg.ret;
+        if (err)
+                nvgpu_err(g, "vGPU resume failed\n");
+
+        return err;
+}
diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.h b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.h
index 38379cf2..ff7d3a66 100644
--- a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.h
+++ b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.h
@@ -33,6 +33,9 @@ int vgpu_remove(struct platform_device *dev);
 
 void vgpu_create_sysfs(struct device *dev);
 void vgpu_remove_sysfs(struct device *dev);
+
+int vgpu_tegra_suspend(struct device *dev);
+int vgpu_tegra_resume(struct device *dev);
 #else
 /* define placeholders for functions used outside of vgpu */
 
@@ -52,6 +55,14 @@ static inline int vgpu_remove(struct platform_device *dev)
 {
         return -ENOSYS;
 }
+static inline int vgpu_tegra_suspend(struct device *dev)
+{
+        return -ENOSYS;
+}
+static inline int vgpu_tegra_resume(struct device *dev)
+{
+        return -ENOSYS;
+}
 #endif
 
 #endif