8 files changed, 84 insertions, 47 deletions

diff --git a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
index 92f88333..e6e16511 100644
--- a/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/flcn_gk20a.c
@@ -361,11 +361,11 @@ static int gk20a_flcn_copy_to_imem(struct nvgpu_falcon *flcn, u32 dst,
                         words, dst, blk, tag);
 
         gk20a_writel(g, base_addr + falcon_falcon_imemc_r(port),
-                falcon_falcon_imemc_offs_f(dst >> 2) |
-                falcon_falcon_imemc_blk_f(blk) |
-                /* Set Auto-Increment on write */
-                falcon_falcon_imemc_aincw_f(1) |
-                sec << 28);
+                        falcon_falcon_imemc_offs_f(dst >> 2) |
+                        falcon_falcon_imemc_blk_f(blk) |
+                        /* Set Auto-Increment on write */
+                        falcon_falcon_imemc_aincw_f(1) |
+                        falcon_falcon_imemc_secure_f(sec ? 1U : 0U));
 
         for (i = 0; i < words; i++) {
                 if (i % 64 == 0) {
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
index bc659a7b..8b3253a1 100644
--- a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
@@ -34,6 +34,7 @@
 #include <nvgpu/soc.h>
 #include <nvgpu/io.h>
 #include <nvgpu/utils.h>
+#include <nvgpu/bitops.h>
 
 #include "gk20a/gk20a.h"
 #include "gk20a/gr_gk20a.h"
@@ -235,9 +236,10 @@ static int gr_gv11b_handle_l1_tag_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         l1_tag_ecc_corrected_err_status, is_l1_tag_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                l1_tag_corrected_err_count_delta +=
-                        (is_l1_tag_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_s());
+                if (is_l1_tag_ecc_corrected_total_err_overflow) {
+                        l1_tag_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_l1_tag_ecc_corrected_err_count[gpc][tpc].counter +=
                                                         l1_tag_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -250,9 +252,10 @@ static int gr_gv11b_handle_l1_tag_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         l1_tag_ecc_uncorrected_err_status, is_l1_tag_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                l1_tag_uncorrected_err_count_delta +=
-                        (is_l1_tag_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_s());
+                if (is_l1_tag_ecc_uncorrected_total_err_overflow) {
+                        l1_tag_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_l1_tag_ecc_uncorrected_err_count[gpc][tpc].counter +=
                                                         l1_tag_uncorrected_err_count_delta;
                 gk20a_writel(g,
@@ -328,9 +331,10 @@ static int gr_gv11b_handle_lrf_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         lrf_ecc_corrected_err_status, is_lrf_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                lrf_corrected_err_count_delta +=
-                        (is_lrf_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_s());
+                if (is_lrf_ecc_corrected_total_err_overflow) {
+                        lrf_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_lrf_ecc_single_err_count[gpc][tpc].counter +=
                                                         lrf_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -343,9 +347,10 @@ static int gr_gv11b_handle_lrf_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         lrf_ecc_uncorrected_err_status, is_lrf_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                lrf_uncorrected_err_count_delta +=
-                        (is_lrf_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_s());
+                if (is_lrf_ecc_uncorrected_total_err_overflow) {
+                        lrf_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_lrf_ecc_double_err_count[gpc][tpc].counter +=
                                                         lrf_uncorrected_err_count_delta;
                 gk20a_writel(g,
@@ -488,9 +493,10 @@ static int gr_gv11b_handle_cbu_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         cbu_ecc_corrected_err_status, is_cbu_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                cbu_corrected_err_count_delta +=
-                        (is_cbu_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_s());
+                if (is_cbu_ecc_corrected_total_err_overflow) {
+                        cbu_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_cbu_ecc_corrected_err_count[gpc][tpc].counter +=
                                                         cbu_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -503,9 +509,10 @@ static int gr_gv11b_handle_cbu_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         cbu_ecc_uncorrected_err_status, is_cbu_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                cbu_uncorrected_err_count_delta +=
-                        (is_cbu_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_s());
+                if (is_cbu_ecc_uncorrected_total_err_overflow) {
+                        cbu_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_cbu_ecc_uncorrected_err_count[gpc][tpc].counter +=
                                                         cbu_uncorrected_err_count_delta;
                 gk20a_writel(g,
@@ -569,9 +576,10 @@ static int gr_gv11b_handle_l1_data_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         l1_data_ecc_corrected_err_status, is_l1_data_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                l1_data_corrected_err_count_delta +=
-                        (is_l1_data_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_s());
+                if (is_l1_data_ecc_corrected_total_err_overflow) {
+                        l1_data_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_l1_data_ecc_corrected_err_count[gpc][tpc].counter +=
                                                         l1_data_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -584,9 +592,10 @@ static int gr_gv11b_handle_l1_data_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         l1_data_ecc_uncorrected_err_status, is_l1_data_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                l1_data_uncorrected_err_count_delta +=
-                        (is_l1_data_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_s());
+                if (is_l1_data_ecc_uncorrected_total_err_overflow) {
+                        l1_data_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_l1_data_ecc_uncorrected_err_count[gpc][tpc].counter +=
                                                         l1_data_uncorrected_err_count_delta;
                 gk20a_writel(g,
@@ -654,9 +663,10 @@ static int gr_gv11b_handle_icache_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         icache_ecc_corrected_err_status, is_icache_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                icache_corrected_err_count_delta +=
-                        (is_icache_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_s());
+                if (is_icache_ecc_corrected_total_err_overflow) {
+                        icache_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_icache_ecc_corrected_err_count[gpc][tpc].counter +=
                                                         icache_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -669,9 +679,10 @@ static int gr_gv11b_handle_icache_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         icache_ecc_uncorrected_err_status, is_icache_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                icache_uncorrected_err_count_delta +=
-                        (is_icache_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_s());
+                if (is_icache_ecc_uncorrected_total_err_overflow) {
+                        icache_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.sm_icache_ecc_uncorrected_err_count[gpc][tpc].counter +=
                                                         icache_uncorrected_err_count_delta;
                 gk20a_writel(g,
@@ -759,9 +770,10 @@ int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         gcc_l15_ecc_corrected_err_status, is_gcc_l15_ecc_corrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                gcc_l15_corrected_err_count_delta +=
-                        (is_gcc_l15_ecc_corrected_total_err_overflow <<
-                         gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_s());
+                if (is_gcc_l15_ecc_corrected_total_err_overflow) {
+                        gcc_l15_corrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_s());
+                }
                 g->ecc.gr.gcc_l15_ecc_corrected_err_count[gpc].counter +=
                                                         gcc_l15_corrected_err_count_delta;
                 gk20a_writel(g,
@@ -774,9 +786,10 @@ int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc,
                         gcc_l15_ecc_uncorrected_err_status, is_gcc_l15_ecc_uncorrected_total_err_overflow);
 
                 /* HW uses 16-bits counter */
-                gcc_l15_uncorrected_err_count_delta +=
-                        (is_gcc_l15_ecc_uncorrected_total_err_overflow <<
-                         gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_s());
+                if (is_gcc_l15_ecc_uncorrected_total_err_overflow) {
+                        gcc_l15_uncorrected_err_count_delta +=
+                                BIT32(gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_s());
+                }
                 g->ecc.gr.gcc_l15_ecc_uncorrected_err_count[gpc].counter +=
                                                         gcc_l15_uncorrected_err_count_delta;
                 gk20a_writel(g,
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_falcon_gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_falcon_gk20a.h
index 27fb5884..7b4d87b0 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_falcon_gk20a.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gk20a/hw_falcon_gk20a.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -336,6 +336,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_falcon_gm20b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_falcon_gm20b.h
index a17c9a9a..c5985685 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_falcon_gm20b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gm20b/hw_falcon_gm20b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -356,6 +356,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_falcon_gp106.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_falcon_gp106.h
index 6740b2a6..d899e3f3 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_falcon_gp106.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp106/hw_falcon_gp106.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -356,6 +356,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_falcon_gp10b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_falcon_gp10b.h
index 918f262b..6dc401d6 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_falcon_gp10b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gp10b/hw_falcon_gp10b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -356,6 +356,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_falcon_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_falcon_gv100.h
index 122956bb..3492d68c 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_falcon_gv100.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_falcon_gv100.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -356,6 +356,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_falcon_gv11b.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_falcon_gv11b.h
index 4bb8f2de..31e883e5 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_falcon_gv11b.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv11b/hw_falcon_gv11b.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -356,6 +356,10 @@ static inline u32 falcon_falcon_imemc_aincw_f(u32 v)
 {
         return (v & 0x1U) << 24U;
 }
+static inline u32 falcon_falcon_imemc_secure_f(u32 v)
+{
+        return (v & 0x1U) << 28U;
+}
 static inline u32 falcon_falcon_imemd_r(u32 i)
 {
         return 0x00000184U + i*16U;