summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gv11b
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/gv11b')
-rw-r--r--drivers/gpu/nvgpu/gv11b/acr_gv11b.c294
-rw-r--r--drivers/gpu/nvgpu/gv11b/acr_gv11b.h30
-rw-r--r--drivers/gpu/nvgpu/gv11b/ce_gv11b.c110
-rw-r--r--drivers/gpu/nvgpu/gv11b/ce_gv11b.h35
-rw-r--r--drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c206
-rw-r--r--drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h34
-rw-r--r--drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c99
-rw-r--r--drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h28
-rw-r--r--drivers/gpu/nvgpu/gv11b/ecc_gv11b.h64
-rw-r--r--drivers/gpu/nvgpu/gv11b/fb_gv11b.c1555
-rw-r--r--drivers/gpu/nvgpu/gv11b/fb_gv11b.h72
-rw-r--r--drivers/gpu/nvgpu/gv11b/fifo_gv11b.c1907
-rw-r--r--drivers/gpu/nvgpu/gv11b/fifo_gv11b.h117
-rw-r--r--drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c72
-rw-r--r--drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h36
-rw-r--r--drivers/gpu/nvgpu/gv11b/gr_gv11b.c3639
-rw-r--r--drivers/gpu/nvgpu/gv11b/gr_gv11b.h215
-rw-r--r--drivers/gpu/nvgpu/gv11b/gv11b.c38
-rw-r--r--drivers/gpu/nvgpu/gv11b/gv11b.h32
-rw-r--r--drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c748
-rw-r--r--drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h99
-rw-r--r--drivers/gpu/nvgpu/gv11b/hal_gv11b.c778
-rw-r--r--drivers/gpu/nvgpu/gv11b/hal_gv11b.h31
-rw-r--r--drivers/gpu/nvgpu/gv11b/ltc_gv11b.c205
-rw-r--r--drivers/gpu/nvgpu/gv11b/ltc_gv11b.h34
-rw-r--r--drivers/gpu/nvgpu/gv11b/mc_gv11b.c92
-rw-r--r--drivers/gpu/nvgpu/gv11b/mc_gv11b.h30
-rw-r--r--drivers/gpu/nvgpu/gv11b/mm_gv11b.c330
-rw-r--r--drivers/gpu/nvgpu/gv11b/mm_gv11b.h46
-rw-r--r--drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c549
-rw-r--r--drivers/gpu/nvgpu/gv11b/pmu_gv11b.c283
-rw-r--r--drivers/gpu/nvgpu/gv11b/pmu_gv11b.h37
-rw-r--r--drivers/gpu/nvgpu/gv11b/regops_gv11b.c1548
-rw-r--r--drivers/gpu/nvgpu/gv11b/regops_gv11b.h42
-rw-r--r--drivers/gpu/nvgpu/gv11b/subctx_gv11b.c185
-rw-r--r--drivers/gpu/nvgpu/gv11b/subctx_gv11b.h34
-rw-r--r--drivers/gpu/nvgpu/gv11b/therm_gv11b.c75
-rw-r--r--drivers/gpu/nvgpu/gv11b/therm_gv11b.h28
38 files changed, 13757 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
new file mode 100644
index 00000000..b245dbc6
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c
@@ -0,0 +1,294 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifdef CONFIG_DEBUG_FS
24#include <linux/debugfs.h>
25#endif
26
27#include <nvgpu/types.h>
28#include <linux/platform/tegra/mc.h>
29
30#include <nvgpu/dma.h>
31#include <nvgpu/gmmu.h>
32#include <nvgpu/timers.h>
33#include <nvgpu/nvgpu_common.h>
34#include <nvgpu/kmem.h>
35#include <nvgpu/nvgpu_mem.h>
36#include <nvgpu/acr/nvgpu_acr.h>
37#include <nvgpu/firmware.h>
38#include <nvgpu/mm.h>
39
40#include "gk20a/gk20a.h"
41#include "acr_gv11b.h"
42#include "pmu_gv11b.h"
43#include "gk20a/pmu_gk20a.h"
44#include "gm20b/mm_gm20b.h"
45#include "gm20b/acr_gm20b.h"
46#include "gp106/acr_gp106.h"
47
48#include <nvgpu/hw/gv11b/hw_pwr_gv11b.h>
49
50/*Defines*/
51#define gv11b_dbg_pmu(fmt, arg...) \
52 gk20a_dbg(gpu_dbg_pmu, fmt, ##arg)
53
54static void flcn64_set_dma(struct falc_u64 *dma_addr, u64 value)
55{
56 dma_addr->lo |= u64_lo32(value);
57 dma_addr->hi |= u64_hi32(value);
58}
59/*Externs*/
60
61/*Forwards*/
62
63/*Loads ACR bin to FB mem and bootstraps PMU with bootloader code
64 * start and end are addresses of ucode blob in non-WPR region*/
65int gv11b_bootstrap_hs_flcn(struct gk20a *g)
66{
67 struct mm_gk20a *mm = &g->mm;
68 struct vm_gk20a *vm = mm->pmu.vm;
69 int err = 0;
70 u64 *acr_dmem;
71 u32 img_size_in_bytes = 0;
72 u32 status, size, index;
73 u64 start;
74 struct acr_desc *acr = &g->acr;
75 struct nvgpu_firmware *acr_fw = acr->acr_fw;
76 struct flcn_bl_dmem_desc_v1 *bl_dmem_desc = &acr->bl_dmem_desc_v1;
77 u32 *acr_ucode_header_t210_load;
78 u32 *acr_ucode_data_t210_load;
79
80 start = nvgpu_mem_get_addr(g, &acr->ucode_blob);
81 size = acr->ucode_blob.size;
82
83 gv11b_dbg_pmu("acr ucode blob start %llx\n", start);
84 gv11b_dbg_pmu("acr ucode blob size %x\n", size);
85
86 gv11b_dbg_pmu("");
87
88 if (!acr_fw) {
89 /*First time init case*/
90 acr_fw = nvgpu_request_firmware(g,
91 GM20B_HSBIN_PMU_UCODE_IMAGE, 0);
92 if (!acr_fw) {
93 nvgpu_err(g, "pmu ucode get fail");
94 return -ENOENT;
95 }
96 acr->acr_fw = acr_fw;
97 acr->hsbin_hdr = (struct bin_hdr *)acr_fw->data;
98 acr->fw_hdr = (struct acr_fw_header *)(acr_fw->data +
99 acr->hsbin_hdr->header_offset);
100 acr_ucode_data_t210_load = (u32 *)(acr_fw->data +
101 acr->hsbin_hdr->data_offset);
102 acr_ucode_header_t210_load = (u32 *)(acr_fw->data +
103 acr->fw_hdr->hdr_offset);
104 img_size_in_bytes = ALIGN((acr->hsbin_hdr->data_size), 256);
105
106 gv11b_dbg_pmu("sig dbg offset %u\n",
107 acr->fw_hdr->sig_dbg_offset);
108 gv11b_dbg_pmu("sig dbg size %u\n", acr->fw_hdr->sig_dbg_size);
109 gv11b_dbg_pmu("sig prod offset %u\n",
110 acr->fw_hdr->sig_prod_offset);
111 gv11b_dbg_pmu("sig prod size %u\n",
112 acr->fw_hdr->sig_prod_size);
113 gv11b_dbg_pmu("patch loc %u\n", acr->fw_hdr->patch_loc);
114 gv11b_dbg_pmu("patch sig %u\n", acr->fw_hdr->patch_sig);
115 gv11b_dbg_pmu("header offset %u\n", acr->fw_hdr->hdr_offset);
116 gv11b_dbg_pmu("header size %u\n", acr->fw_hdr->hdr_size);
117
118 /* Lets patch the signatures first.. */
119 if (acr_ucode_patch_sig(g, acr_ucode_data_t210_load,
120 (u32 *)(acr_fw->data +
121 acr->fw_hdr->sig_prod_offset),
122 (u32 *)(acr_fw->data +
123 acr->fw_hdr->sig_dbg_offset),
124 (u32 *)(acr_fw->data +
125 acr->fw_hdr->patch_loc),
126 (u32 *)(acr_fw->data +
127 acr->fw_hdr->patch_sig)) < 0) {
128 nvgpu_err(g, "patch signatures fail");
129 err = -1;
130 goto err_release_acr_fw;
131 }
132 err = nvgpu_dma_alloc_map_sys(vm, img_size_in_bytes,
133 &acr->acr_ucode);
134 if (err) {
135 err = -ENOMEM;
136 goto err_release_acr_fw;
137 }
138
139 for (index = 0; index < 9; index++)
140 gv11b_dbg_pmu("acr_ucode_header_t210_load %u\n",
141 acr_ucode_header_t210_load[index]);
142
143 acr_dmem = (u64 *)
144 &(((u8 *)acr_ucode_data_t210_load)[
145 acr_ucode_header_t210_load[2]]);
146 acr->acr_dmem_desc_v1 = (struct flcn_acr_desc_v1 *)((u8 *)(
147 acr->acr_ucode.cpu_va) + acr_ucode_header_t210_load[2]);
148 ((struct flcn_acr_desc_v1 *)acr_dmem)->nonwpr_ucode_blob_start =
149 (start);
150 ((struct flcn_acr_desc_v1 *)acr_dmem)->nonwpr_ucode_blob_size =
151 size;
152 ((struct flcn_acr_desc_v1 *)acr_dmem)->regions.no_regions = 2;
153 ((struct flcn_acr_desc_v1 *)acr_dmem)->wpr_offset = 0;
154
155 nvgpu_mem_wr_n(g, &acr->acr_ucode, 0,
156 acr_ucode_data_t210_load, img_size_in_bytes);
157 /*
158 * In order to execute this binary, we will be using
159 * a bootloader which will load this image into PMU IMEM/DMEM.
160 * Fill up the bootloader descriptor for PMU HAL to use..
161 * TODO: Use standard descriptor which the generic bootloader is
162 * checked in.
163 */
164 bl_dmem_desc->signature[0] = 0;
165 bl_dmem_desc->signature[1] = 0;
166 bl_dmem_desc->signature[2] = 0;
167 bl_dmem_desc->signature[3] = 0;
168 bl_dmem_desc->ctx_dma = GK20A_PMU_DMAIDX_VIRT;
169 flcn64_set_dma(&bl_dmem_desc->code_dma_base,
170 acr->acr_ucode.gpu_va);
171 bl_dmem_desc->non_sec_code_off = acr_ucode_header_t210_load[0];
172 bl_dmem_desc->non_sec_code_size = acr_ucode_header_t210_load[1];
173 bl_dmem_desc->sec_code_off = acr_ucode_header_t210_load[5];
174 bl_dmem_desc->sec_code_size = acr_ucode_header_t210_load[6];
175 bl_dmem_desc->code_entry_point = 0; /* Start at 0th offset */
176 flcn64_set_dma(&bl_dmem_desc->data_dma_base,
177 acr->acr_ucode.gpu_va +
178 acr_ucode_header_t210_load[2]);
179 bl_dmem_desc->data_size = acr_ucode_header_t210_load[3];
180 } else
181 acr->acr_dmem_desc_v1->nonwpr_ucode_blob_size = 0;
182 status = pmu_exec_gen_bl(g, bl_dmem_desc, 1);
183 if (status != 0) {
184 err = status;
185 goto err_free_ucode_map;
186 }
187
188 return 0;
189err_free_ucode_map:
190 nvgpu_dma_unmap_free(vm, &acr->acr_ucode);
191err_release_acr_fw:
192 nvgpu_release_firmware(g, acr_fw);
193 acr->acr_fw = NULL;
194
195 return err;
196}
197
198static int bl_bootstrap(struct nvgpu_pmu *pmu,
199 struct flcn_bl_dmem_desc_v1 *pbl_desc, u32 bl_sz)
200{
201 struct gk20a *g = gk20a_from_pmu(pmu);
202 struct acr_desc *acr = &g->acr;
203 struct mm_gk20a *mm = &g->mm;
204 u32 virt_addr = 0;
205 struct hsflcn_bl_desc *pmu_bl_gm10x_desc = g->acr.pmu_hsbl_desc;
206 u32 dst;
207
208 gk20a_dbg_fn("");
209
210 gk20a_writel(g, pwr_falcon_itfen_r(),
211 gk20a_readl(g, pwr_falcon_itfen_r()) |
212 pwr_falcon_itfen_ctxen_enable_f());
213 gk20a_writel(g, pwr_pmu_new_instblk_r(),
214 pwr_pmu_new_instblk_ptr_f(
215 nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) |
216 pwr_pmu_new_instblk_valid_f(1) |
217 pwr_pmu_new_instblk_target_sys_ncoh_f());
218
219 /*copy bootloader interface structure to dmem*/
220 nvgpu_flcn_copy_to_dmem(pmu->flcn, 0, (u8 *)pbl_desc,
221 sizeof(struct flcn_bl_dmem_desc_v1), 0);
222
223 /* copy bootloader to TOP of IMEM */
224 dst = (pwr_falcon_hwcfg_imem_size_v(
225 gk20a_readl(g, pwr_falcon_hwcfg_r())) << 8) - bl_sz;
226
227 nvgpu_flcn_copy_to_imem(pmu->flcn, dst,
228 (u8 *)(acr->hsbl_ucode.cpu_va), bl_sz, 0, 0,
229 pmu_bl_gm10x_desc->bl_start_tag);
230
231 gv11b_dbg_pmu("Before starting falcon with BL\n");
232
233 virt_addr = pmu_bl_gm10x_desc->bl_start_tag << 8;
234
235 nvgpu_flcn_bootstrap(pmu->flcn, virt_addr);
236
237 return 0;
238}
239
240int gv11b_init_pmu_setup_hw1(struct gk20a *g,
241 void *desc, u32 bl_sz)
242{
243
244 struct nvgpu_pmu *pmu = &g->pmu;
245 int err;
246
247 gk20a_dbg_fn("");
248
249 nvgpu_mutex_acquire(&pmu->isr_mutex);
250 nvgpu_flcn_reset(pmu->flcn);
251 pmu->isr_enabled = true;
252 nvgpu_mutex_release(&pmu->isr_mutex);
253
254 /* setup apertures - virtual */
255 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
256 pwr_fbif_transcfg_mem_type_physical_f() |
257 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
258 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT),
259 pwr_fbif_transcfg_mem_type_virtual_f());
260 /* setup apertures - physical */
261 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID),
262 pwr_fbif_transcfg_mem_type_physical_f() |
263 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
264 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH),
265 pwr_fbif_transcfg_mem_type_physical_f() |
266 pwr_fbif_transcfg_target_coherent_sysmem_f());
267 gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH),
268 pwr_fbif_transcfg_mem_type_physical_f() |
269 pwr_fbif_transcfg_target_noncoherent_sysmem_f());
270
271 /*Copying pmu cmdline args*/
272 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq(pmu,
273 g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_PWRCLK));
274 g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode(pmu, 1);
275 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size(
276 pmu, GK20A_PMU_TRACE_BUFSIZE);
277 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base(pmu);
278 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx(
279 pmu, GK20A_PMU_DMAIDX_VIRT);
280 nvgpu_flcn_copy_to_dmem(pmu->flcn, g->acr.pmu_args,
281 (u8 *)(g->ops.pmu_ver.get_pmu_cmdline_args_ptr(pmu)),
282 g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu), 0);
283 /*disable irqs for hs falcon booting as we will poll for halt*/
284 nvgpu_mutex_acquire(&pmu->isr_mutex);
285 pmu_enable_irq(pmu, false);
286 pmu->isr_enabled = false;
287 nvgpu_mutex_release(&pmu->isr_mutex);
288 /*Clearing mailbox register used to reflect capabilities*/
289 gk20a_writel(g, pwr_falcon_mailbox1_r(), 0);
290 err = bl_bootstrap(pmu, desc, bl_sz);
291 if (err)
292 return err;
293 return 0;
294}
diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.h b/drivers/gpu/nvgpu/gv11b/acr_gv11b.h
new file mode 100644
index 00000000..72b3ec35
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef __ACR_GV11B_H_
24#define __ACR_GV11B_H_
25
26
27int gv11b_bootstrap_hs_flcn(struct gk20a *g);
28int gv11b_init_pmu_setup_hw1(struct gk20a *g,
29 void *desc, u32 bl_sz);
30#endif /*__PMU_GP106_H_*/
diff --git a/drivers/gpu/nvgpu/gv11b/ce_gv11b.c b/drivers/gpu/nvgpu/gv11b/ce_gv11b.c
new file mode 100644
index 00000000..86518ac7
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/ce_gv11b.c
@@ -0,0 +1,110 @@
1/*
2 * Volta GPU series Copy Engine.
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "nvgpu/log.h"
26#include "nvgpu/bitops.h"
27
28#include "gk20a/gk20a.h"
29
30#include "gp10b/ce_gp10b.h"
31
32#include "ce_gv11b.h"
33
34#include <nvgpu/hw/gv11b/hw_ce_gv11b.h>
35#include <nvgpu/hw/gv11b/hw_top_gv11b.h>
36
37u32 gv11b_ce_get_num_pce(struct gk20a *g)
38{
39 /* register contains a bitmask indicating which physical copy
40 * engines are present (and not floorswept).
41 */
42 u32 num_pce;
43 u32 ce_pce_map = gk20a_readl(g, ce_pce_map_r());
44
45 num_pce = hweight32(ce_pce_map);
46 nvgpu_log_info(g, "num PCE: %d", num_pce);
47 return num_pce;
48}
49
50void gv11b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
51{
52 u32 ce_intr = gk20a_readl(g, ce_intr_status_r(inst_id));
53 u32 clear_intr = 0;
54
55 nvgpu_log(g, gpu_dbg_intr, "ce isr 0x%08x 0x%08x", ce_intr, inst_id);
56
57 /* An INVALID_CONFIG interrupt will be generated if a floorswept
58 * PCE is assigned to a valid LCE in the NV_CE_PCE2LCE_CONFIG
59 * registers. This is a fatal error and the LCE will have to be
60 * reset to get back to a working state.
61 */
62 if (ce_intr & ce_intr_status_invalid_config_pending_f()) {
63 nvgpu_log(g, gpu_dbg_intr,
64 "ce: inst %d: invalid config", inst_id);
65 clear_intr |= ce_intr_status_invalid_config_reset_f();
66 }
67
68 /* A MTHD_BUFFER_FAULT interrupt will be triggered if any access
69 * to a method buffer during context load or save encounters a fault.
70 * This is a fatal interrupt and will require at least the LCE to be
71 * reset before operations can start again, if not the entire GPU.
72 */
73 if (ce_intr & ce_intr_status_mthd_buffer_fault_pending_f()) {
74 nvgpu_log(g, gpu_dbg_intr,
75 "ce: inst %d: mthd buffer fault", inst_id);
76 clear_intr |= ce_intr_status_mthd_buffer_fault_reset_f();
77 }
78
79 gk20a_writel(g, ce_intr_status_r(inst_id), clear_intr);
80
81 gp10b_ce_isr(g, inst_id, pri_base);
82}
83
84u32 gv11b_ce_get_num_lce(struct gk20a *g)
85{
86 u32 reg_val, num_lce;
87
88 reg_val = gk20a_readl(g, top_num_ces_r());
89 num_lce = top_num_ces_value_v(reg_val);
90 nvgpu_log_info(g, "num LCE: %d", num_lce);
91
92 return num_lce;
93}
94
95void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g)
96{
97 u32 reg_val, num_lce, lce, clear_intr;
98
99 num_lce = gv11b_ce_get_num_lce(g);
100
101 for (lce = 0; lce < num_lce; lce++) {
102 reg_val = gk20a_readl(g, ce_intr_status_r(lce));
103 if (reg_val & ce_intr_status_mthd_buffer_fault_pending_f()) {
104 nvgpu_log(g, gpu_dbg_intr,
105 "ce: lce %d: mthd buffer fault", lce);
106 clear_intr = ce_intr_status_mthd_buffer_fault_reset_f();
107 gk20a_writel(g, ce_intr_status_r(lce), clear_intr);
108 }
109 }
110}
diff --git a/drivers/gpu/nvgpu/gv11b/ce_gv11b.h b/drivers/gpu/nvgpu/gv11b/ce_gv11b.h
new file mode 100644
index 00000000..a0c7e0b1
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/ce_gv11b.h
@@ -0,0 +1,35 @@
1/*
2 *
3 * Volta GPU series copy engine
4 *
5 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 * DEALINGS IN THE SOFTWARE.
24 */
25#ifndef __CE_GV11B_H__
26#define __CE_GV11B_H__
27
28struct gk20a;
29
30void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g);
31u32 gv11b_ce_get_num_lce(struct gk20a *g);
32u32 gv11b_ce_get_num_pce(struct gk20a *g);
33void gv11b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
34
35#endif /*__CE2_GV11B_H__*/
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
new file mode 100644
index 00000000..2eb45a88
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c
@@ -0,0 +1,206 @@
1/*
2 * GV11B Cycle stats snapshots support
3 *
4 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/dma-mapping.h>
26#include <linux/dma-buf.h>
27
28#include <nvgpu/bitops.h>
29#include <nvgpu/kmem.h>
30#include <nvgpu/lock.h>
31#include <nvgpu/dma.h>
32#include <nvgpu/mm.h>
33
34#include "gk20a/gk20a.h"
35#include "gk20a/css_gr_gk20a.h"
36#include "css_gr_gv11b.h"
37
38#include <nvgpu/log.h>
39#include <nvgpu/bug.h>
40
41#include <nvgpu/hw/gv11b/hw_perf_gv11b.h>
42#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
43
44
45/* reports whether the hw queue overflowed */
46static inline bool css_hw_get_overflow_status(struct gk20a *g)
47{
48 const u32 st = perf_pmasys_control_membuf_status_overflowed_f();
49 return st == (gk20a_readl(g, perf_pmasys_control_r()) & st);
50}
51
52/* returns how many pending snapshot entries are pending */
53static inline u32 css_hw_get_pending_snapshots(struct gk20a *g)
54{
55 return gk20a_readl(g, perf_pmasys_mem_bytes_r()) /
56 sizeof(struct gk20a_cs_snapshot_fifo_entry);
57}
58
59/* informs hw how many snapshots have been processed (frees up fifo space) */
60static inline void gv11b_css_hw_set_handled_snapshots(struct gk20a *g, u32 done)
61{
62 if (done > 0) {
63 gk20a_writel(g, perf_pmasys_mem_bump_r(),
64 done * sizeof(struct gk20a_cs_snapshot_fifo_entry));
65 }
66}
67
68/* disable streaming to memory */
69static void gv11b_css_hw_reset_streaming(struct gk20a *g)
70{
71 u32 engine_status;
72
73 /* reset the perfmon */
74 g->ops.mc.reset(g, mc_enable_perfmon_enabled_f());
75
76 /* RBUFEMPTY must be set -- otherwise we'll pick up */
77 /* snapshot that have been queued up from earlier */
78 engine_status = gk20a_readl(g, perf_pmasys_enginestatus_r());
79
80 /* turn off writes */
81 gk20a_writel(g, perf_pmasys_control_r(),
82 perf_pmasys_control_membuf_clear_status_doit_f());
83
84 /* pointing all pending snapshots as handled */
85 gv11b_css_hw_set_handled_snapshots(g, css_hw_get_pending_snapshots(g));
86}
87
88int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch,
89 struct gk20a_cs_snapshot_client *cs_client)
90{
91 struct gk20a *g = ch->g;
92 struct gr_gk20a *gr = &g->gr;
93 struct gk20a_cs_snapshot *data = gr->cs_data;
94 u32 snapshot_size = cs_client->snapshot_size;
95 int ret;
96
97 u32 virt_addr_lo;
98 u32 virt_addr_hi;
99 u32 inst_pa_page;
100
101 if (data->hw_snapshot)
102 return 0;
103
104 if (snapshot_size < CSS_MIN_HW_SNAPSHOT_SIZE)
105 snapshot_size = CSS_MIN_HW_SNAPSHOT_SIZE;
106
107 ret = nvgpu_dma_alloc_map_sys(g->mm.pmu.vm, snapshot_size,
108 &data->hw_memdesc);
109 if (ret)
110 return ret;
111
112 /* perf output buffer may not cross a 4GB boundary - with a separate */
113 /* va smaller than that, it won't but check anyway */
114 if (!data->hw_memdesc.cpu_va ||
115 data->hw_memdesc.size < snapshot_size ||
116 data->hw_memdesc.gpu_va + u64_lo32(snapshot_size) > SZ_4G) {
117 ret = -EFAULT;
118 goto failed_allocation;
119 }
120
121 data->hw_snapshot =
122 (struct gk20a_cs_snapshot_fifo_entry *)data->hw_memdesc.cpu_va;
123 data->hw_end = data->hw_snapshot +
124 snapshot_size / sizeof(struct gk20a_cs_snapshot_fifo_entry);
125 data->hw_get = data->hw_snapshot;
126 memset(data->hw_snapshot, 0xff, snapshot_size);
127
128 virt_addr_lo = u64_lo32(data->hw_memdesc.gpu_va);
129 virt_addr_hi = u64_hi32(data->hw_memdesc.gpu_va);
130
131 gv11b_css_hw_reset_streaming(g);
132
133 gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo);
134 gk20a_writel(g, perf_pmasys_outbaseupper_r(),
135 perf_pmasys_outbaseupper_ptr_f(virt_addr_hi));
136 gk20a_writel(g, perf_pmasys_outsize_r(), snapshot_size);
137
138 /* this field is aligned to 4K */
139 inst_pa_page = nvgpu_inst_block_addr(g, &g->mm.hwpm.inst_block) >> 12;
140
141 gk20a_writel(g, perf_pmasys_mem_block_r(),
142 perf_pmasys_mem_block_base_f(inst_pa_page) |
143 perf_pmasys_mem_block_valid_true_f() |
144 nvgpu_aperture_mask(g, &g->mm.hwpm.inst_block,
145 perf_pmasys_mem_block_target_sys_ncoh_f(),
146 perf_pmasys_mem_block_target_lfb_f()));
147
148
149 gk20a_dbg_info("cyclestats: buffer for hardware snapshots enabled\n");
150
151 return 0;
152
153failed_allocation:
154 if (data->hw_memdesc.size) {
155 nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc);
156 memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc));
157 }
158 data->hw_snapshot = NULL;
159
160 return ret;
161}
162
163void gv11b_css_hw_disable_snapshot(struct gr_gk20a *gr)
164{
165 struct gk20a *g = gr->g;
166 struct gk20a_cs_snapshot *data = gr->cs_data;
167
168 if (!data->hw_snapshot)
169 return;
170
171 gv11b_css_hw_reset_streaming(g);
172
173 gk20a_writel(g, perf_pmasys_outbase_r(), 0);
174 gk20a_writel(g, perf_pmasys_outbaseupper_r(),
175 perf_pmasys_outbaseupper_ptr_f(0));
176 gk20a_writel(g, perf_pmasys_outsize_r(), 0);
177
178 gk20a_writel(g, perf_pmasys_mem_block_r(),
179 perf_pmasys_mem_block_base_f(0) |
180 perf_pmasys_mem_block_valid_false_f() |
181 perf_pmasys_mem_block_target_f(0));
182
183 nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc);
184 memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc));
185 data->hw_snapshot = NULL;
186
187 gk20a_dbg_info("cyclestats: buffer for hardware snapshots disabled\n");
188}
189
190int gv11b_css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending,
191 bool *hw_overflow)
192{
193 struct gk20a *g = ch->g;
194 struct gr_gk20a *gr = &g->gr;
195 struct gk20a_cs_snapshot *css = gr->cs_data;
196
197 if (!css->hw_snapshot)
198 return -EINVAL;
199
200 *pending = css_hw_get_pending_snapshots(g);
201 if (!*pending)
202 return 0;
203
204 *hw_overflow = css_hw_get_overflow_status(g);
205 return 0;
206}
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h
new file mode 100644
index 00000000..6b11a62e
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h
@@ -0,0 +1,34 @@
1/*
2 * GV11B Cycle stats snapshots support
3 *
4 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef CSS_GR_GV11B_H
26#define CSS_GR_GV11B_H
27
28int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch,
29 struct gk20a_cs_snapshot_client *cs_client);
30void gv11b_css_hw_disable_snapshot(struct gr_gk20a *gr);
31int gv11b_css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending,
32 bool *hw_overflow);
33
34#endif /* CSS_GR_GV11B_H */
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
new file mode 100644
index 00000000..a02c2ddd
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c
@@ -0,0 +1,99 @@
1/*
2 * Tegra GV11B GPU Debugger/Profiler Driver
3 *
4 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <uapi/linux/nvgpu.h>
26
27#include <nvgpu/log.h>
28#include "gk20a/gk20a.h"
29#include <nvgpu/hw/gv11b/hw_perf_gv11b.h>
30
31int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size)
32{
33 struct mm_gk20a *mm = &g->mm;
34 u32 virt_addr_lo;
35 u32 virt_addr_hi;
36 u32 inst_pa_page;
37 int err;
38
39 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, "");
40 err = gk20a_busy(g);
41 if (err) {
42 nvgpu_err(g, "failed to poweron");
43 return err;
44 }
45
46 err = gk20a_alloc_inst_block(g, &mm->perfbuf.inst_block);
47 if (err)
48 return err;
49
50 g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0);
51
52 virt_addr_lo = u64_lo32(offset);
53 virt_addr_hi = u64_hi32(offset);
54
55 gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo);
56 gk20a_writel(g, perf_pmasys_outbaseupper_r(),
57 perf_pmasys_outbaseupper_ptr_f(virt_addr_hi));
58 gk20a_writel(g, perf_pmasys_outsize_r(), size);
59
60 /* this field is aligned to 4K */
61 inst_pa_page = nvgpu_inst_block_addr(g, &mm->perfbuf.inst_block) >> 12;
62
63 gk20a_writel(g, perf_pmasys_mem_block_r(),
64 perf_pmasys_mem_block_base_f(inst_pa_page) |
65 perf_pmasys_mem_block_valid_true_f() |
66 nvgpu_aperture_mask(g, &mm->perfbuf.inst_block,
67+ perf_pmasys_mem_block_target_sys_ncoh_f(),
68+ perf_pmasys_mem_block_target_lfb_f()));
69
70 gk20a_idle(g);
71 return 0;
72}
73
74/* must be called with dbg_sessions_lock held */
75int gv11b_perfbuf_disable_locked(struct gk20a *g)
76{
77 int err;
78
79 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, "");
80 err = gk20a_busy(g);
81 if (err) {
82 nvgpu_err(g, "failed to poweron");
83 return err;
84 }
85
86 gk20a_writel(g, perf_pmasys_outbase_r(), 0);
87 gk20a_writel(g, perf_pmasys_outbaseupper_r(),
88 perf_pmasys_outbaseupper_ptr_f(0));
89 gk20a_writel(g, perf_pmasys_outsize_r(), 0);
90
91 gk20a_writel(g, perf_pmasys_mem_block_r(),
92 perf_pmasys_mem_block_base_f(0) |
93 perf_pmasys_mem_block_valid_false_f() |
94 perf_pmasys_mem_block_target_f(0));
95
96 gk20a_idle(g);
97
98 return 0;
99}
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h
new file mode 100644
index 00000000..88771a49
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h
@@ -0,0 +1,28 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#ifndef DBG_GPU_GV11B_H
23#define DBG_GPU_GV11B_H
24
25int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size);
26int gv11b_perfbuf_disable_locked(struct gk20a *g);
27
28#endif /* DBG_GPU_GV11B_H */
diff --git a/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h b/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h
new file mode 100644
index 00000000..94b25c02
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h
@@ -0,0 +1,64 @@
1/*
2 * GV11B GPU ECC
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef _NVGPU_ECC_GV11B_H_
26#define _NVGPU_ECC_GV11B_H_
27
28struct ecc_gr_t19x {
29 struct gk20a_ecc_stat sm_l1_tag_corrected_err_count;
30 struct gk20a_ecc_stat sm_l1_tag_uncorrected_err_count;
31 struct gk20a_ecc_stat sm_cbu_corrected_err_count;
32 struct gk20a_ecc_stat sm_cbu_uncorrected_err_count;
33 struct gk20a_ecc_stat sm_l1_data_corrected_err_count;
34 struct gk20a_ecc_stat sm_l1_data_uncorrected_err_count;
35 struct gk20a_ecc_stat sm_icache_corrected_err_count;
36 struct gk20a_ecc_stat sm_icache_uncorrected_err_count;
37 struct gk20a_ecc_stat gcc_l15_corrected_err_count;
38 struct gk20a_ecc_stat gcc_l15_uncorrected_err_count;
39 struct gk20a_ecc_stat fecs_corrected_err_count;
40 struct gk20a_ecc_stat fecs_uncorrected_err_count;
41 struct gk20a_ecc_stat gpccs_corrected_err_count;
42 struct gk20a_ecc_stat gpccs_uncorrected_err_count;
43 struct gk20a_ecc_stat mmu_l1tlb_corrected_err_count;
44 struct gk20a_ecc_stat mmu_l1tlb_uncorrected_err_count;
45};
46
47struct ecc_ltc_t19x {
48 struct gk20a_ecc_stat l2_cache_corrected_err_count;
49 struct gk20a_ecc_stat l2_cache_uncorrected_err_count;
50};
51
52/* TODO: PMU and FB ECC features are still under embargo */
53struct ecc_eng_t19x {
54 /* FB */
55 struct gk20a_ecc_stat mmu_l2tlb_corrected_err_count;
56 struct gk20a_ecc_stat mmu_l2tlb_uncorrected_err_count;
57 struct gk20a_ecc_stat mmu_hubtlb_corrected_err_count;
58 struct gk20a_ecc_stat mmu_hubtlb_uncorrected_err_count;
59 struct gk20a_ecc_stat mmu_fillunit_corrected_err_count;
60 struct gk20a_ecc_stat mmu_fillunit_uncorrected_err_count;
61 /* PMU */
62};
63
64#endif
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c
new file mode 100644
index 00000000..ec487bdf
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c
@@ -0,0 +1,1555 @@
1/*
2 * GV11B FB
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26
27#include <nvgpu/dma.h>
28#include <nvgpu/log.h>
29#include <nvgpu/enabled.h>
30#include <nvgpu/gmmu.h>
31#include <nvgpu/barrier.h>
32#include <nvgpu/soc.h>
33
34#include "gk20a/gk20a.h"
35#include "gk20a/mm_gk20a.h"
36
37#include "gp10b/fb_gp10b.h"
38
39#include "gv11b/fifo_gv11b.h"
40#include "gv11b/fb_gv11b.h"
41#include "gv11b/ce_gv11b.h"
42
43#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
44#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
45#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
46#include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h>
47
48static int gv11b_fb_fix_page_fault(struct gk20a *g,
49 struct mmu_fault_info *mmfault);
50
51static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
52 u32 invalidate_replay_val);
53
54static void gv11b_init_nvlink_soc_credits(struct gk20a *g)
55{
56 if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) {
57 nvgpu_info(g, "nvlink soc credits init done by bpmp");
58 } else {
59 /* MSS_NVLINK_1_BASE */
60 void __iomem *soc1 = ioremap(0x01f20010, 4096);
61 /* MSS_NVLINK_2_BASE */
62 void __iomem *soc2 = ioremap(0x01f40010, 4096);
63 /* MSS_NVLINK_3_BASE */
64 void __iomem *soc3 = ioremap(0x01f60010, 4096);
65 /* MSS_NVLINK_4_BASE */
66 void __iomem *soc4 = ioremap(0x01f80010, 4096);
67 u32 val;
68
69 nvgpu_info(g, "init nvlink soc credits");
70
71 val = readl_relaxed(soc1);
72 writel_relaxed(val, soc1);
73 val = readl_relaxed(soc1 + 4);
74 writel_relaxed(val, soc1 + 4);
75
76 val = readl_relaxed(soc2);
77 writel_relaxed(val, soc2);
78 val = readl_relaxed(soc2 + 4);
79 writel_relaxed(val, soc2 + 4);
80
81 val = readl_relaxed(soc3);
82 writel_relaxed(val, soc3);
83 val = readl_relaxed(soc3 + 4);
84 writel_relaxed(val, soc3 + 4);
85
86 val = readl_relaxed(soc4);
87 writel_relaxed(val, soc4);
88 val = readl_relaxed(soc4 + 4);
89 writel_relaxed(val, soc4 + 4);
90 }
91}
92
93void gv11b_fb_init_fs_state(struct gk20a *g)
94{
95 nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb");
96
97 nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x",
98 gk20a_readl(g, fb_fbhub_num_active_ltcs_r()));
99
100 nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u",
101 fb_mmu_num_active_ltcs_count_v(
102 gk20a_readl(g, fb_mmu_num_active_ltcs_r())));
103}
104
105void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr)
106{
107 u32 max_size = gr->max_comptag_mem;
108 /* one tag line covers 64KB */
109 u32 max_comptag_lines = max_size << 4;
110 u32 compbit_base_post_divide;
111 u64 compbit_base_post_multiply64;
112 u64 compbit_store_iova;
113 u64 compbit_base_post_divide64;
114
115 if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL))
116 compbit_store_iova = nvgpu_mem_get_phys_addr(g,
117 &gr->compbit_store.mem);
118 else
119 compbit_store_iova = nvgpu_mem_get_addr(g,
120 &gr->compbit_store.mem);
121
122 compbit_base_post_divide64 = compbit_store_iova >>
123 fb_mmu_cbc_base_address_alignment_shift_v();
124
125 do_div(compbit_base_post_divide64, g->ltc_count);
126 compbit_base_post_divide = u64_lo32(compbit_base_post_divide64);
127
128 compbit_base_post_multiply64 = ((u64)compbit_base_post_divide *
129 g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v();
130
131 if (compbit_base_post_multiply64 < compbit_store_iova)
132 compbit_base_post_divide++;
133
134 if (g->ops.ltc.cbc_fix_config)
135 compbit_base_post_divide =
136 g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide);
137
138 gk20a_writel(g, fb_mmu_cbc_base_r(),
139 fb_mmu_cbc_base_address_f(compbit_base_post_divide));
140
141 nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte,
142 "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n",
143 (u32)(compbit_store_iova >> 32),
144 (u32)(compbit_store_iova & 0xffffffff),
145 compbit_base_post_divide);
146 nvgpu_log(g, gpu_dbg_fn, "cbc base %x",
147 gk20a_readl(g, fb_mmu_cbc_base_r()));
148
149 gr->compbit_store.base_hw = compbit_base_post_divide;
150
151 g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate,
152 0, max_comptag_lines - 1);
153
154}
155
156void gv11b_fb_reset(struct gk20a *g)
157{
158 if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) {
159 nvgpu_log(g, gpu_dbg_info, "mc_elpg_enable set by bpmp");
160 } else {
161 u32 mc_elpg_enable_val;
162
163 nvgpu_log(g, gpu_dbg_info, "enable xbar, pfb and hub");
164 mc_elpg_enable_val = mc_elpg_enable_xbar_enabled_f() |
165 mc_elpg_enable_pfb_enabled_f() |
166 mc_elpg_enable_hub_enabled_f();
167 mc_elpg_enable_val |= gk20a_readl(g, mc_elpg_enable_r());
168 gk20a_writel(g, mc_elpg_enable_r(), mc_elpg_enable_val);
169
170 }
171 /* fs hub should be out of reset by now */
172 gv11b_init_nvlink_soc_credits(g);
173}
174
175static const char * const invalid_str = "invalid";
176
177static const char *const fault_type_descs_gv11b[] = {
178 "invalid pde",
179 "invalid pde size",
180 "invalid pte",
181 "limit violation",
182 "unbound inst block",
183 "priv violation",
184 "write",
185 "read",
186 "pitch mask violation",
187 "work creation",
188 "unsupported aperture",
189 "compression failure",
190 "unsupported kind",
191 "region violation",
192 "poison",
193 "atomic"
194};
195
196static const char *const fault_client_type_descs_gv11b[] = {
197 "gpc",
198 "hub",
199};
200
201static const char *const fault_access_type_descs_gv11b[] = {
202 "virt read",
203 "virt write",
204 "virt atomic strong",
205 "virt prefetch",
206 "virt atomic weak",
207 "xxx",
208 "xxx",
209 "xxx",
210 "phys read",
211 "phys write",
212 "phys atomic",
213 "phys prefetch",
214};
215
216static const char *const hub_client_descs_gv11b[] = {
217 "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu",
218 "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2",
219 "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc",
220 "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb",
221 "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1",
222 "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8",
223 "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3",
224 "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0",
225 "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp",
226 "dont care"
227};
228
229static const char *const gpc_client_descs_gv11b[] = {
230 "t1 0", "t1 1", "t1 2", "t1 3",
231 "t1 4", "t1 5", "t1 6", "t1 7",
232 "pe 0", "pe 1", "pe 2", "pe 3",
233 "pe 4", "pe 5", "pe 6", "pe 7",
234 "rast", "gcc", "gpccs",
235 "prop 0", "prop 1", "prop 2", "prop 3",
236 "gpm",
237 "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3",
238 "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7",
239 "utlb",
240 "t1 8", "t1 9", "t1 10", "t1 11",
241 "t1 12", "t1 13", "t1 14", "t1 15",
242 "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3",
243 "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7",
244 "pe 8", "pe 9", "tpccs 8", "tpccs 9",
245 "t1 16", "t1 17", "t1 18", "t1 19",
246 "pe 10", "pe 11", "tpccs 10", "tpccs 11",
247 "t1 20", "t1 21", "t1 22", "t1 23",
248 "pe 12", "pe 13", "tpccs 12", "tpccs 13",
249 "t1 24", "t1 25", "t1 26", "t1 27",
250 "pe 14", "pe 15", "tpccs 14", "tpccs 15",
251 "t1 28", "t1 29", "t1 30", "t1 31",
252 "pe 16", "pe 17", "tpccs 16", "tpccs 17",
253 "t1 32", "t1 33", "t1 34", "t1 35",
254 "pe 18", "pe 19", "tpccs 18", "tpccs 19",
255 "t1 36", "t1 37", "t1 38", "t1 39",
256};
257
258u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g,
259 unsigned int index)
260{
261 u32 reg_val;
262
263 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
264 return fb_mmu_fault_buffer_size_enable_v(reg_val);
265}
266
267static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g,
268 unsigned int index, u32 next)
269{
270 u32 reg_val;
271
272 nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next);
273
274 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
275 reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(),
276 fb_mmu_fault_buffer_get_ptr_f(next));
277
278 /* while the fault is being handled it is possible for overflow
279 * to happen,
280 */
281 if (reg_val & fb_mmu_fault_buffer_get_overflow_m())
282 reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f();
283
284 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
285
286 /* make sure get ptr update is visible to everyone to avoid
287 * reading already read entry
288 */
289 nvgpu_mb();
290}
291
292static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g,
293 unsigned int index)
294{
295 u32 reg_val;
296
297 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
298 return fb_mmu_fault_buffer_get_ptr_v(reg_val);
299}
300
301static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g,
302 unsigned int index)
303{
304 u32 reg_val;
305
306 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_put_r(index));
307 return fb_mmu_fault_buffer_put_ptr_v(reg_val);
308}
309
310static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g,
311 unsigned int index)
312{
313 u32 reg_val;
314
315 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
316 return fb_mmu_fault_buffer_size_val_v(reg_val);
317}
318
319static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g,
320 unsigned int index, u32 *get_idx)
321{
322 u32 put_idx;
323
324 *get_idx = gv11b_fb_fault_buffer_get_index(g, index);
325 put_idx = gv11b_fb_fault_buffer_put_index(g, index);
326
327 return *get_idx == put_idx;
328}
329
330static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g,
331 unsigned int index)
332{
333 u32 get_idx, put_idx, entries;
334
335
336 get_idx = gv11b_fb_fault_buffer_get_index(g, index);
337
338 put_idx = gv11b_fb_fault_buffer_put_index(g, index);
339
340 entries = gv11b_fb_fault_buffer_size_val(g, index);
341
342 return get_idx == ((put_idx + 1) % entries);
343}
344
345void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g,
346 unsigned int index, unsigned int state)
347{
348 u32 fault_status;
349 u32 reg_val;
350
351 nvgpu_log_fn(g, " ");
352
353 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
354 if (state) {
355 if (gv11b_fb_is_fault_buf_enabled(g, index)) {
356 nvgpu_log_info(g, "fault buffer is already enabled");
357 } else {
358 reg_val |= fb_mmu_fault_buffer_size_enable_true_f();
359 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index),
360 reg_val);
361 }
362
363 } else {
364 struct nvgpu_timeout timeout;
365 u32 delay = GR_IDLE_CHECK_DEFAULT;
366
367 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
368 NVGPU_TIMER_CPU_TIMER);
369
370 reg_val &= (~(fb_mmu_fault_buffer_size_enable_m()));
371 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val);
372
373 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
374
375 do {
376 if (!(fault_status & fb_mmu_fault_status_busy_true_f()))
377 break;
378 /*
379 * Make sure fault buffer is disabled.
380 * This is to avoid accessing fault buffer by hw
381 * during the window BAR2 is being unmapped by s/w
382 */
383 nvgpu_log_info(g, "fault status busy set, check again");
384 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
385
386 nvgpu_usleep_range(delay, delay * 2);
387 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
388 } while (!nvgpu_timeout_expired_msg(&timeout,
389 "fault status busy set"));
390 }
391}
392
393void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index)
394{
395 u32 addr_lo;
396 u32 addr_hi;
397
398 nvgpu_log_fn(g, " ");
399
400 gv11b_fb_fault_buf_set_state_hw(g, index,
401 FAULT_BUF_DISABLED);
402 addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >>
403 ram_in_base_shift_v());
404 addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va);
405
406 gk20a_writel(g, fb_mmu_fault_buffer_lo_r(index),
407 fb_mmu_fault_buffer_lo_addr_f(addr_lo));
408
409 gk20a_writel(g, fb_mmu_fault_buffer_hi_r(index),
410 fb_mmu_fault_buffer_hi_addr_f(addr_hi));
411
412 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index),
413 fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) |
414 fb_mmu_fault_buffer_size_overflow_intr_enable_f());
415
416 gv11b_fb_fault_buf_set_state_hw(g, index, FAULT_BUF_ENABLED);
417}
418
419static void gv11b_fb_intr_en_set(struct gk20a *g,
420 unsigned int index, u32 mask)
421{
422 u32 reg_val;
423
424 reg_val = gk20a_readl(g, fb_niso_intr_en_set_r(index));
425 reg_val |= mask;
426 gk20a_writel(g, fb_niso_intr_en_set_r(index), reg_val);
427}
428
429static void gv11b_fb_intr_en_clr(struct gk20a *g,
430 unsigned int index, u32 mask)
431{
432 u32 reg_val;
433
434 reg_val = gk20a_readl(g, fb_niso_intr_en_clr_r(index));
435 reg_val |= mask;
436 gk20a_writel(g, fb_niso_intr_en_clr_r(index), reg_val);
437}
438
439static u32 gv11b_fb_get_hub_intr_clr_mask(struct gk20a *g,
440 unsigned int intr_type)
441{
442 u32 mask = 0;
443
444 if (intr_type & HUB_INTR_TYPE_OTHER) {
445 mask |=
446 fb_niso_intr_en_clr_mmu_other_fault_notify_m();
447 }
448
449 if (intr_type & HUB_INTR_TYPE_NONREPLAY) {
450 mask |=
451 fb_niso_intr_en_clr_mmu_nonreplayable_fault_notify_m() |
452 fb_niso_intr_en_clr_mmu_nonreplayable_fault_overflow_m();
453 }
454
455 if (intr_type & HUB_INTR_TYPE_REPLAY) {
456 mask |=
457 fb_niso_intr_en_clr_mmu_replayable_fault_notify_m() |
458 fb_niso_intr_en_clr_mmu_replayable_fault_overflow_m();
459 }
460
461 if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) {
462 mask |=
463 fb_niso_intr_en_clr_mmu_ecc_uncorrected_error_notify_m();
464 }
465
466 if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) {
467 mask |=
468 fb_niso_intr_en_clr_hub_access_counter_notify_m() |
469 fb_niso_intr_en_clr_hub_access_counter_error_m();
470 }
471
472 return mask;
473}
474
475static u32 gv11b_fb_get_hub_intr_en_mask(struct gk20a *g,
476 unsigned int intr_type)
477{
478 u32 mask = 0;
479
480 if (intr_type & HUB_INTR_TYPE_OTHER) {
481 mask |=
482 fb_niso_intr_en_set_mmu_other_fault_notify_m();
483 }
484
485 if (intr_type & HUB_INTR_TYPE_NONREPLAY) {
486 mask |=
487 fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
488 fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m();
489 }
490
491 if (intr_type & HUB_INTR_TYPE_REPLAY) {
492 mask |=
493 fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
494 fb_niso_intr_en_set_mmu_replayable_fault_overflow_m();
495 }
496
497 if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) {
498 mask |=
499 fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m();
500 }
501
502 if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) {
503 mask |=
504 fb_niso_intr_en_set_hub_access_counter_notify_m() |
505 fb_niso_intr_en_set_hub_access_counter_error_m();
506 }
507
508 return mask;
509}
510
511void gv11b_fb_enable_hub_intr(struct gk20a *g,
512 unsigned int index, unsigned int intr_type)
513{
514 u32 mask = 0;
515
516 mask = gv11b_fb_get_hub_intr_en_mask(g, intr_type);
517
518 if (mask)
519 gv11b_fb_intr_en_set(g, index, mask);
520}
521
522void gv11b_fb_disable_hub_intr(struct gk20a *g,
523 unsigned int index, unsigned int intr_type)
524{
525 u32 mask = 0;
526
527 mask = gv11b_fb_get_hub_intr_clr_mask(g, intr_type);
528
529 if (mask)
530 gv11b_fb_intr_en_clr(g, index, mask);
531}
532
533static void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status)
534{
535 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
536 u32 corrected_delta, uncorrected_delta;
537 u32 corrected_overflow, uncorrected_overflow;
538
539 ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r());
540 corrected_cnt = gk20a_readl(g,
541 fb_mmu_l2tlb_ecc_corrected_err_count_r());
542 uncorrected_cnt = gk20a_readl(g,
543 fb_mmu_l2tlb_ecc_uncorrected_err_count_r());
544
545 corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v(
546 corrected_cnt);
547 uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v(
548 uncorrected_cnt);
549 corrected_overflow = ecc_status &
550 fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m();
551
552 uncorrected_overflow = ecc_status &
553 fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m();
554
555 /* clear the interrupt */
556 if ((corrected_delta > 0) || corrected_overflow)
557 gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0);
558 if ((uncorrected_delta > 0) || uncorrected_overflow)
559 gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0);
560
561 gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(),
562 fb_mmu_l2tlb_ecc_status_reset_clear_f());
563
564 /* Handle overflow */
565 if (corrected_overflow)
566 corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s());
567 if (uncorrected_overflow)
568 uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s());
569
570
571 g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0] +=
572 corrected_delta;
573 g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0] +=
574 uncorrected_delta;
575
576 if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m())
577 nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
578 if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m())
579 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
580 if (corrected_overflow || uncorrected_overflow)
581 nvgpu_info(g, "mmu l2tlb ecc counter overflow!");
582
583 nvgpu_log(g, gpu_dbg_intr,
584 "ecc error address: 0x%x", ecc_addr);
585 nvgpu_log(g, gpu_dbg_intr,
586 "ecc error count corrected: %d, uncorrected %d",
587 g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0],
588 g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0]);
589}
590
591static void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status)
592{
593 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
594 u32 corrected_delta, uncorrected_delta;
595 u32 corrected_overflow, uncorrected_overflow;
596
597 ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r());
598 corrected_cnt = gk20a_readl(g,
599 fb_mmu_hubtlb_ecc_corrected_err_count_r());
600 uncorrected_cnt = gk20a_readl(g,
601 fb_mmu_hubtlb_ecc_uncorrected_err_count_r());
602
603 corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v(
604 corrected_cnt);
605 uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v(
606 uncorrected_cnt);
607 corrected_overflow = ecc_status &
608 fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m();
609
610 uncorrected_overflow = ecc_status &
611 fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m();
612
613 /* clear the interrupt */
614 if ((corrected_delta > 0) || corrected_overflow)
615 gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0);
616 if ((uncorrected_delta > 0) || uncorrected_overflow)
617 gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0);
618
619 gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(),
620 fb_mmu_hubtlb_ecc_status_reset_clear_f());
621
622 /* Handle overflow */
623 if (corrected_overflow)
624 corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s());
625 if (uncorrected_overflow)
626 uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s());
627
628
629 g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0] +=
630 corrected_delta;
631 g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0] +=
632 uncorrected_delta;
633
634 if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m())
635 nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
636 if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m())
637 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
638 if (corrected_overflow || uncorrected_overflow)
639 nvgpu_info(g, "mmu hubtlb ecc counter overflow!");
640
641 nvgpu_log(g, gpu_dbg_intr,
642 "ecc error address: 0x%x", ecc_addr);
643 nvgpu_log(g, gpu_dbg_intr,
644 "ecc error count corrected: %d, uncorrected %d",
645 g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0],
646 g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0]);
647}
648
649static void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status)
650{
651 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
652 u32 corrected_delta, uncorrected_delta;
653 u32 corrected_overflow, uncorrected_overflow;
654
655 ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r());
656 corrected_cnt = gk20a_readl(g,
657 fb_mmu_fillunit_ecc_corrected_err_count_r());
658 uncorrected_cnt = gk20a_readl(g,
659 fb_mmu_fillunit_ecc_uncorrected_err_count_r());
660
661 corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v(
662 corrected_cnt);
663 uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v(
664 uncorrected_cnt);
665 corrected_overflow = ecc_status &
666 fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m();
667
668 uncorrected_overflow = ecc_status &
669 fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m();
670
671 /* clear the interrupt */
672 if ((corrected_delta > 0) || corrected_overflow)
673 gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0);
674 if ((uncorrected_delta > 0) || uncorrected_overflow)
675 gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0);
676
677 gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(),
678 fb_mmu_fillunit_ecc_status_reset_clear_f());
679
680 /* Handle overflow */
681 if (corrected_overflow)
682 corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s());
683 if (uncorrected_overflow)
684 uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s());
685
686
687 g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0] +=
688 corrected_delta;
689 g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0] +=
690 uncorrected_delta;
691
692 if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m())
693 nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error");
694 if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m())
695 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error");
696 if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m())
697 nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error");
698 if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m())
699 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error");
700
701 if (corrected_overflow || uncorrected_overflow)
702 nvgpu_info(g, "mmu fillunit ecc counter overflow!");
703
704 nvgpu_log(g, gpu_dbg_intr,
705 "ecc error address: 0x%x", ecc_addr);
706 nvgpu_log(g, gpu_dbg_intr,
707 "ecc error count corrected: %d, uncorrected %d",
708 g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0],
709 g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0]);
710}
711
712static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault)
713{
714 if (WARN_ON(mmfault->fault_type >=
715 ARRAY_SIZE(fault_type_descs_gv11b)))
716 mmfault->fault_type_desc = invalid_str;
717 else
718 mmfault->fault_type_desc =
719 fault_type_descs_gv11b[mmfault->fault_type];
720
721 if (WARN_ON(mmfault->client_type >=
722 ARRAY_SIZE(fault_client_type_descs_gv11b)))
723 mmfault->client_type_desc = invalid_str;
724 else
725 mmfault->client_type_desc =
726 fault_client_type_descs_gv11b[mmfault->client_type];
727
728 mmfault->client_id_desc = invalid_str;
729 if (mmfault->client_type ==
730 gmmu_fault_client_type_hub_v()) {
731
732 if (!(WARN_ON(mmfault->client_id >=
733 ARRAY_SIZE(hub_client_descs_gv11b))))
734 mmfault->client_id_desc =
735 hub_client_descs_gv11b[mmfault->client_id];
736 } else if (mmfault->client_type ==
737 gmmu_fault_client_type_gpc_v()) {
738 if (!(WARN_ON(mmfault->client_id >=
739 ARRAY_SIZE(gpc_client_descs_gv11b))))
740 mmfault->client_id_desc =
741 gpc_client_descs_gv11b[mmfault->client_id];
742 }
743
744}
745
746static void gv11b_fb_print_fault_info(struct gk20a *g,
747 struct mmu_fault_info *mmfault)
748{
749 if (mmfault && mmfault->valid) {
750 nvgpu_err(g, "[MMU FAULT] "
751 "mmu engine id: %d, "
752 "ch id: %d, "
753 "fault addr: 0x%llx, "
754 "fault addr aperture: %d, "
755 "fault type: %s, "
756 "access type: %s, ",
757 mmfault->mmu_engine_id,
758 mmfault->chid,
759 mmfault->fault_addr,
760 mmfault->fault_addr_aperture,
761 mmfault->fault_type_desc,
762 fault_access_type_descs_gv11b[mmfault->access_type]);
763 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
764 "mmu engine id: %d, "
765 "faulted act eng id if any: 0x%x, "
766 "faulted veid if any: 0x%x, "
767 "faulted pbdma id if any: 0x%x, "
768 "fault addr: 0x%llx, ",
769 mmfault->mmu_engine_id,
770 mmfault->faulted_engine,
771 mmfault->faulted_subid,
772 mmfault->faulted_pbdma,
773 mmfault->fault_addr);
774 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
775 "fault addr aperture: %d, "
776 "fault type: %s, "
777 "access type: %s, "
778 "inst ptr: 0x%llx, "
779 "inst ptr aperture: %d, ",
780 mmfault->fault_addr_aperture,
781 mmfault->fault_type_desc,
782 fault_access_type_descs_gv11b[mmfault->access_type],
783 mmfault->inst_ptr,
784 mmfault->inst_aperture);
785 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
786 "ch id: %d, "
787 "timestamp hi:lo 0x%08x:0x%08x, "
788 "client type: %s, "
789 "client id: %s, "
790 "gpc id if client type is gpc: %d, ",
791 mmfault->chid,
792 mmfault->timestamp_hi, mmfault->timestamp_lo,
793 mmfault->client_type_desc,
794 mmfault->client_id_desc,
795 mmfault->gpc_id);
796 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
797 "protected mode: %d, "
798 "replayable fault: %d, "
799 "replayable fault en: %d ",
800 mmfault->protected_mode,
801 mmfault->replayable_fault,
802 mmfault->replay_fault_en);
803 }
804}
805
806/*
807 *Fault buffer format
808 *
809 * 31 28 24 23 16 15 8 7 4 0
810 *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-.
811 *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0|
812 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
813 *| inst_hi |
814 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
815 *| addr_31_12 | |AP |
816 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
817 *| addr_63_32 |
818 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
819 *| timestamp_lo |
820 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
821 *| timestamp_hi |
822 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
823 *| (reserved) | engine_id |
824 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
825 *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype|
826 */
827
828static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g,
829 struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault)
830{
831 u32 rd32_val;
832 u32 addr_lo, addr_hi;
833 u64 inst_ptr;
834 u32 chid = FIFO_INVAL_CHANNEL_ID;
835 struct channel_gk20a *refch;
836
837 memset(mmfault, 0, sizeof(*mmfault));
838
839 rd32_val = nvgpu_mem_rd32(g, mem, offset +
840 gmmu_fault_buf_entry_inst_lo_w());
841 addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val);
842 addr_lo = addr_lo << ram_in_base_shift_v();
843
844 addr_hi = nvgpu_mem_rd32(g, mem, offset +
845 gmmu_fault_buf_entry_inst_hi_w());
846 addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi);
847
848 inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo);
849
850 /* refch will be put back after fault is handled */
851 refch = gk20a_refch_from_inst_ptr(g, inst_ptr);
852 if (refch)
853 chid = refch->chid;
854
855 /* it is ok to continue even if refch is NULL */
856 mmfault->refch = refch;
857 mmfault->chid = chid;
858 mmfault->inst_ptr = inst_ptr;
859 mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val);
860
861 rd32_val = nvgpu_mem_rd32(g, mem, offset +
862 gmmu_fault_buf_entry_addr_lo_w());
863
864 mmfault->fault_addr_aperture =
865 gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val);
866 addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val);
867 addr_lo = addr_lo << ram_in_base_shift_v();
868
869 rd32_val = nvgpu_mem_rd32(g, mem, offset +
870 gmmu_fault_buf_entry_addr_hi_w());
871 addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val);
872 mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo);
873
874 rd32_val = nvgpu_mem_rd32(g, mem, offset +
875 gmmu_fault_buf_entry_timestamp_lo_w());
876 mmfault->timestamp_lo =
877 gmmu_fault_buf_entry_timestamp_lo_v(rd32_val);
878
879 rd32_val = nvgpu_mem_rd32(g, mem, offset +
880 gmmu_fault_buf_entry_timestamp_hi_w());
881 mmfault->timestamp_hi =
882 gmmu_fault_buf_entry_timestamp_hi_v(rd32_val);
883
884 rd32_val = nvgpu_mem_rd32(g, mem, offset +
885 gmmu_fault_buf_entry_engine_id_w());
886
887 mmfault->mmu_engine_id =
888 gmmu_fault_buf_entry_engine_id_v(rd32_val);
889 gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id,
890 &mmfault->faulted_engine, &mmfault->faulted_subid,
891 &mmfault->faulted_pbdma);
892
893 rd32_val = nvgpu_mem_rd32(g, mem, offset +
894 gmmu_fault_buf_entry_fault_type_w());
895 mmfault->client_id =
896 gmmu_fault_buf_entry_client_v(rd32_val);
897 mmfault->replayable_fault =
898 gmmu_fault_buf_entry_replayable_fault_v(rd32_val);
899
900 mmfault->fault_type =
901 gmmu_fault_buf_entry_fault_type_v(rd32_val);
902 mmfault->access_type =
903 gmmu_fault_buf_entry_access_type_v(rd32_val);
904
905 mmfault->client_type =
906 gmmu_fault_buf_entry_mmu_client_type_v(rd32_val);
907
908 mmfault->gpc_id =
909 gmmu_fault_buf_entry_gpc_id_v(rd32_val);
910 mmfault->protected_mode =
911 gmmu_fault_buf_entry_protected_mode_v(rd32_val);
912
913 mmfault->replay_fault_en =
914 gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val);
915
916 mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val);
917
918 rd32_val = nvgpu_mem_rd32(g, mem, offset +
919 gmmu_fault_buf_entry_fault_type_w());
920 rd32_val &= ~(gmmu_fault_buf_entry_valid_m());
921 nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(),
922 rd32_val);
923
924 gv11b_fb_parse_mmfault(mmfault);
925}
926
927static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g,
928 struct mmu_fault_info *mmfault, u32 *invalidate_replay_val)
929{
930 unsigned int id_type;
931 u32 num_lce, act_eng_bitmask = 0;
932 int err = 0;
933 u32 id = ((u32)~0);
934
935 if (!mmfault->valid)
936 return;
937
938 gv11b_fb_print_fault_info(g, mmfault);
939
940 num_lce = gv11b_ce_get_num_lce(g);
941 if ((mmfault->mmu_engine_id >=
942 gmmu_fault_mmu_eng_id_ce0_v()) &&
943 (mmfault->mmu_engine_id <
944 gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) {
945 /* CE page faults are not reported as replayable */
946 nvgpu_log(g, gpu_dbg_intr, "CE Faulted");
947 err = gv11b_fb_fix_page_fault(g, mmfault);
948 gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch,
949 mmfault->faulted_pbdma, mmfault->faulted_engine);
950 if (!err) {
951 nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed");
952 *invalidate_replay_val = 0;
953 /* refch in mmfault is assigned at the time of copying
954 * fault info from snap reg or bar2 fault buf
955 */
956 gk20a_channel_put(mmfault->refch);
957 return;
958 }
959 /* Do recovery. Channel recovery needs refch */
960 nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed");
961 }
962
963 if (!mmfault->replayable_fault) {
964 if (mmfault->fault_type ==
965 gmmu_fault_type_unbound_inst_block_v()) {
966 /*
967 * Bug 1847172: When an engine faults due to an unbound
968 * instance block, the fault cannot be isolated to a
969 * single context so we need to reset the entire runlist
970 */
971 id_type = ID_TYPE_UNKNOWN;
972 nvgpu_log(g, gpu_dbg_intr, "UNBOUND INST BLOCK MMU FAULT");
973
974 } else if (mmfault->refch) {
975 if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) {
976 id = mmfault->refch->tsgid;
977 id_type = ID_TYPE_TSG;
978 } else {
979 id = mmfault->chid;
980 id_type = ID_TYPE_CHANNEL;
981 }
982 } else {
983 id_type = ID_TYPE_UNKNOWN;
984 }
985 if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID)
986 act_eng_bitmask = BIT(mmfault->faulted_engine);
987
988 g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask,
989 id, id_type, RC_TYPE_MMU_FAULT, mmfault);
990 } else {
991 if (mmfault->fault_type == gmmu_fault_type_pte_v()) {
992 nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix");
993 err = gv11b_fb_fix_page_fault(g, mmfault);
994 if (err)
995 *invalidate_replay_val |=
996 fb_mmu_invalidate_replay_cancel_global_f();
997 else
998 *invalidate_replay_val |=
999 fb_mmu_invalidate_replay_start_ack_all_f();
1000 } else {
1001 /* cancel faults other than invalid pte */
1002 *invalidate_replay_val |=
1003 fb_mmu_invalidate_replay_cancel_global_f();
1004 }
1005 /* refch in mmfault is assigned at the time of copying
1006 * fault info from snap reg or bar2 fault buf
1007 */
1008 gk20a_channel_put(mmfault->refch);
1009 }
1010}
1011
1012static void gv11b_fb_replay_or_cancel_faults(struct gk20a *g,
1013 u32 invalidate_replay_val)
1014{
1015 int err = 0;
1016
1017 nvgpu_log_fn(g, " ");
1018
1019 if (invalidate_replay_val &
1020 fb_mmu_invalidate_replay_cancel_global_f()) {
1021 /*
1022 * cancel faults so that next time it faults as
1023 * replayable faults and channel recovery can be done
1024 */
1025 err = gv11b_fb_mmu_invalidate_replay(g,
1026 fb_mmu_invalidate_replay_cancel_global_f());
1027 } else if (invalidate_replay_val &
1028 fb_mmu_invalidate_replay_start_ack_all_f()) {
1029 /* pte valid is fixed. replay faulting request */
1030 err = gv11b_fb_mmu_invalidate_replay(g,
1031 fb_mmu_invalidate_replay_start_ack_all_f());
1032 }
1033}
1034
1035static void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g,
1036 u32 fault_status, unsigned int index)
1037{
1038 u32 get_indx, offset, rd32_val, entries;
1039 struct nvgpu_mem *mem;
1040 struct mmu_fault_info *mmfault;
1041 u32 invalidate_replay_val = 0;
1042 u64 prev_fault_addr = 0ULL;
1043 u64 next_fault_addr = 0ULL;
1044
1045 if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) {
1046 nvgpu_log(g, gpu_dbg_intr,
1047 "SPURIOUS mmu fault: reg index:%d", index);
1048 return;
1049 }
1050 nvgpu_info(g, "%s MMU FAULT" ,
1051 index == REPLAY_REG_INDEX ? "REPLAY" : "NON-REPLAY");
1052
1053 nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx);
1054
1055 mem = &g->mm.hw_fault_buf[index];
1056 mmfault = g->mm.fault_info[index];
1057
1058 entries = gv11b_fb_fault_buffer_size_val(g, index);
1059 nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries);
1060
1061 offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32);
1062 nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset);
1063
1064 rd32_val = nvgpu_mem_rd32(g, mem,
1065 offset + gmmu_fault_buf_entry_valid_w());
1066 nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val);
1067
1068 while ((rd32_val & gmmu_fault_buf_entry_valid_m())) {
1069
1070 nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val);
1071
1072 gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault);
1073
1074 get_indx = (get_indx + 1) % entries;
1075 nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx);
1076
1077 gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx);
1078
1079 offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32);
1080 nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset);
1081
1082 rd32_val = nvgpu_mem_rd32(g, mem,
1083 offset + gmmu_fault_buf_entry_valid_w());
1084
1085 if (index == REPLAY_REG_INDEX && mmfault->fault_addr != 0ULL) {
1086 /* fault_addr "0" is not supposed to be fixed ever.
1087 * For the first time when prev = 0, next = 0 and
1088 * fault addr is also 0 then handle_mmu_fault_common will
1089 * not be called. Fix by checking fault_addr not equal to 0
1090 */
1091 prev_fault_addr = next_fault_addr;
1092 next_fault_addr = mmfault->fault_addr;
1093 if (prev_fault_addr == next_fault_addr) {
1094 nvgpu_log(g, gpu_dbg_intr, "pte is fixed");
1095 if (mmfault->refch)
1096 gk20a_channel_put(mmfault->refch);
1097 /* pte already fixed for this addr */
1098 continue;
1099 }
1100 }
1101
1102 gv11b_fb_handle_mmu_fault_common(g, mmfault,
1103 &invalidate_replay_val);
1104
1105 }
1106 if (index == REPLAY_REG_INDEX && invalidate_replay_val)
1107 gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val);
1108}
1109
1110static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g,
1111 u32 fault_status, struct mmu_fault_info *mmfault)
1112{
1113 u32 reg_val;
1114 u32 addr_lo, addr_hi;
1115 u64 inst_ptr;
1116 int chid = FIFO_INVAL_CHANNEL_ID;
1117 struct channel_gk20a *refch;
1118
1119 memset(mmfault, 0, sizeof(*mmfault));
1120
1121 if (!(fault_status & fb_mmu_fault_status_valid_set_f())) {
1122
1123 nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set");
1124 return;
1125 }
1126
1127 reg_val = gk20a_readl(g, fb_mmu_fault_inst_lo_r());
1128 addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val);
1129 addr_lo = addr_lo << ram_in_base_shift_v();
1130
1131 addr_hi = gk20a_readl(g, fb_mmu_fault_inst_hi_r());
1132 addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi);
1133 inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo);
1134
1135 /* refch will be put back after fault is handled */
1136 refch = gk20a_refch_from_inst_ptr(g, inst_ptr);
1137 if (refch)
1138 chid = refch->chid;
1139
1140 /* It is still ok to continue if refch is NULL */
1141 mmfault->refch = refch;
1142 mmfault->chid = chid;
1143 mmfault->inst_ptr = inst_ptr;
1144 mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val);
1145 mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val);
1146
1147 gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id,
1148 &mmfault->faulted_engine, &mmfault->faulted_subid,
1149 &mmfault->faulted_pbdma);
1150
1151 reg_val = gk20a_readl(g, fb_mmu_fault_addr_lo_r());
1152 addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val);
1153 addr_lo = addr_lo << ram_in_base_shift_v();
1154
1155 mmfault->fault_addr_aperture =
1156 fb_mmu_fault_addr_lo_phys_aperture_v(reg_val);
1157
1158 addr_hi = gk20a_readl(g, fb_mmu_fault_addr_hi_r());
1159 addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi);
1160 mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo);
1161
1162 reg_val = gk20a_readl(g, fb_mmu_fault_info_r());
1163 mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val);
1164 mmfault->replayable_fault =
1165 fb_mmu_fault_info_replayable_fault_v(reg_val);
1166 mmfault->client_id = fb_mmu_fault_info_client_v(reg_val);
1167 mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val);
1168 mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val);
1169 mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val);
1170 mmfault->protected_mode =
1171 fb_mmu_fault_info_protected_mode_v(reg_val);
1172 mmfault->replay_fault_en =
1173 fb_mmu_fault_info_replayable_fault_en_v(reg_val);
1174
1175 mmfault->valid = fb_mmu_fault_info_valid_v(reg_val);
1176
1177 fault_status &= ~(fb_mmu_fault_status_valid_m());
1178 gk20a_writel(g, fb_mmu_fault_status_r(), fault_status);
1179
1180 gv11b_fb_parse_mmfault(mmfault);
1181
1182}
1183
1184static void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g,
1185 u32 fault_status)
1186{
1187 u32 reg_val;
1188 unsigned int index = REPLAY_REG_INDEX;
1189
1190 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
1191
1192 if (fault_status &
1193 fb_mmu_fault_status_replayable_getptr_corrupted_m()) {
1194
1195 nvgpu_err(g, "replayable getptr corrupted set");
1196
1197 gv11b_fb_fault_buf_configure_hw(g, index);
1198
1199 reg_val = set_field(reg_val,
1200 fb_mmu_fault_buffer_get_getptr_corrupted_m(),
1201 fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
1202 }
1203
1204 if (fault_status &
1205 fb_mmu_fault_status_replayable_overflow_m()) {
1206 bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
1207
1208 nvgpu_err(g, "replayable overflow: buffer full:%s",
1209 buffer_full?"true":"false");
1210
1211 reg_val = set_field(reg_val,
1212 fb_mmu_fault_buffer_get_overflow_m(),
1213 fb_mmu_fault_buffer_get_overflow_clear_f());
1214 }
1215
1216 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
1217}
1218
1219static void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g,
1220 u32 fault_status)
1221{
1222 u32 reg_val;
1223 unsigned int index = NONREPLAY_REG_INDEX;
1224
1225 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
1226
1227 if (fault_status &
1228 fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) {
1229
1230 nvgpu_err(g, "non replayable getptr corrupted set");
1231
1232 gv11b_fb_fault_buf_configure_hw(g, index);
1233
1234 reg_val = set_field(reg_val,
1235 fb_mmu_fault_buffer_get_getptr_corrupted_m(),
1236 fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
1237 }
1238
1239 if (fault_status &
1240 fb_mmu_fault_status_non_replayable_overflow_m()) {
1241
1242 bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
1243
1244 nvgpu_err(g, "non replayable overflow: buffer full:%s",
1245 buffer_full?"true":"false");
1246
1247 reg_val = set_field(reg_val,
1248 fb_mmu_fault_buffer_get_overflow_m(),
1249 fb_mmu_fault_buffer_get_overflow_clear_f());
1250 }
1251
1252 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
1253}
1254
1255static void gv11b_fb_handle_bar2_fault(struct gk20a *g,
1256 struct mmu_fault_info *mmfault, u32 fault_status)
1257{
1258 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX,
1259 HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY);
1260
1261
1262 if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) {
1263 if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX))
1264 gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX);
1265 }
1266
1267 if (fault_status & fb_mmu_fault_status_replayable_error_m()) {
1268 if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX))
1269 gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX);
1270 }
1271 gv11b_ce_mthd_buffer_fault_in_bar2_fault(g);
1272
1273 g->ops.mm.init_bar2_mm_hw_setup(g);
1274
1275 if (mmfault->refch) {
1276 gk20a_channel_put(mmfault->refch);
1277 mmfault->refch = NULL;
1278 }
1279 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX,
1280 HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY);
1281}
1282
1283static void gv11b_fb_handle_other_fault_notify(struct gk20a *g,
1284 u32 fault_status)
1285{
1286 struct mmu_fault_info *mmfault;
1287 u32 invalidate_replay_val = 0;
1288
1289 mmfault = g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY];
1290
1291 gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault);
1292
1293 /* BAR2/Physical faults will not be snapped in hw fault buf */
1294 if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) {
1295 nvgpu_err(g, "BAR2 MMU FAULT");
1296 gv11b_fb_handle_bar2_fault(g, mmfault, fault_status);
1297
1298 } else if (mmfault->mmu_engine_id ==
1299 gmmu_fault_mmu_eng_id_physical_v()) {
1300 /* usually means VPR or out of bounds physical accesses */
1301 nvgpu_err(g, "PHYSICAL MMU FAULT");
1302
1303 } else {
1304 gv11b_fb_handle_mmu_fault_common(g, mmfault,
1305 &invalidate_replay_val);
1306
1307 if (invalidate_replay_val)
1308 gv11b_fb_replay_or_cancel_faults(g,
1309 invalidate_replay_val);
1310 }
1311}
1312
1313static void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status)
1314{
1315 u32 dropped_faults = 0;
1316
1317 dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() |
1318 fb_mmu_fault_status_dropped_bar1_virt_set_f() |
1319 fb_mmu_fault_status_dropped_bar2_phys_set_f() |
1320 fb_mmu_fault_status_dropped_bar2_virt_set_f() |
1321 fb_mmu_fault_status_dropped_ifb_phys_set_f() |
1322 fb_mmu_fault_status_dropped_ifb_virt_set_f() |
1323 fb_mmu_fault_status_dropped_other_phys_set_f()|
1324 fb_mmu_fault_status_dropped_other_virt_set_f();
1325
1326 if (fault_status & dropped_faults) {
1327 nvgpu_err(g, "dropped mmu fault (0x%08x)",
1328 fault_status & dropped_faults);
1329 gk20a_writel(g, fb_mmu_fault_status_r(), dropped_faults);
1330 }
1331}
1332
1333
1334static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr)
1335{
1336 u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
1337
1338 nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status);
1339
1340 if (niso_intr &
1341 fb_niso_intr_mmu_other_fault_notify_m()) {
1342
1343 gv11b_fb_handle_dropped_mmu_fault(g, fault_status);
1344
1345 gv11b_fb_handle_other_fault_notify(g, fault_status);
1346 }
1347
1348 if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) {
1349
1350 if (niso_intr &
1351 fb_niso_intr_mmu_nonreplayable_fault_notify_m()) {
1352
1353 gv11b_fb_handle_mmu_nonreplay_replay_fault(g,
1354 fault_status, NONREPLAY_REG_INDEX);
1355
1356 /*
1357 * When all the faults are processed,
1358 * GET and PUT will have same value and mmu fault status
1359 * bit will be reset by HW
1360 */
1361 }
1362 if (niso_intr &
1363 fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) {
1364
1365 gv11b_fb_handle_nonreplay_fault_overflow(g,
1366 fault_status);
1367 }
1368
1369 }
1370
1371 if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) {
1372
1373 if (niso_intr &
1374 fb_niso_intr_mmu_replayable_fault_notify_m()) {
1375
1376 gv11b_fb_handle_mmu_nonreplay_replay_fault(g,
1377 fault_status, REPLAY_REG_INDEX);
1378 }
1379 if (niso_intr &
1380 fb_niso_intr_mmu_replayable_fault_overflow_m()) {
1381
1382 gv11b_fb_handle_replay_fault_overflow(g,
1383 fault_status);
1384 }
1385
1386 }
1387
1388 nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status");
1389 gk20a_writel(g, fb_mmu_fault_status_r(),
1390 fb_mmu_fault_status_valid_clear_f());
1391}
1392
1393void gv11b_fb_hub_isr(struct gk20a *g)
1394{
1395 u32 status, niso_intr;
1396
1397 nvgpu_mutex_acquire(&g->mm.hub_isr_mutex);
1398
1399 niso_intr = gk20a_readl(g, fb_niso_intr_r());
1400
1401 nvgpu_info(g, "enter hub isr, niso_intr = 0x%08x", niso_intr);
1402
1403 if (niso_intr &
1404 (fb_niso_intr_hub_access_counter_notify_m() |
1405 fb_niso_intr_hub_access_counter_error_m())) {
1406
1407 nvgpu_info(g, "hub access counter notify/error");
1408 }
1409 if (niso_intr &
1410 fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) {
1411
1412 nvgpu_info(g, "ecc uncorrected error notify");
1413
1414 /* disable interrupts during handling */
1415 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX,
1416 HUB_INTR_TYPE_ECC_UNCORRECTED);
1417
1418 status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r());
1419 if (status)
1420 gv11b_handle_l2tlb_ecc_isr(g, status);
1421
1422 status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r());
1423 if (status)
1424 gv11b_handle_hubtlb_ecc_isr(g, status);
1425
1426 status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r());
1427 if (status)
1428 gv11b_handle_fillunit_ecc_isr(g, status);
1429
1430 /* re-enable interrupts after handling */
1431 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX,
1432 HUB_INTR_TYPE_ECC_UNCORRECTED);
1433
1434 }
1435 if (niso_intr &
1436 (fb_niso_intr_mmu_other_fault_notify_m() |
1437 fb_niso_intr_mmu_replayable_fault_notify_m() |
1438 fb_niso_intr_mmu_replayable_fault_overflow_m() |
1439 fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
1440 fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) {
1441
1442 nvgpu_info(g, "MMU Fault");
1443 gv11b_fb_handle_mmu_fault(g, niso_intr);
1444 }
1445
1446 nvgpu_mutex_release(&g->mm.hub_isr_mutex);
1447}
1448
1449bool gv11b_fb_mmu_fault_pending(struct gk20a *g)
1450{
1451 if (gk20a_readl(g, fb_niso_intr_r()) &
1452 (fb_niso_intr_mmu_other_fault_notify_m() |
1453 fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() |
1454 fb_niso_intr_mmu_replayable_fault_notify_m() |
1455 fb_niso_intr_mmu_replayable_fault_overflow_m() |
1456 fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
1457 fb_niso_intr_mmu_nonreplayable_fault_overflow_m()))
1458 return true;
1459
1460 return false;
1461}
1462
1463static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
1464 u32 invalidate_replay_val)
1465{
1466 int err = -ETIMEDOUT;
1467 u32 reg_val;
1468 struct nvgpu_timeout timeout;
1469
1470 gk20a_dbg_fn("");
1471
1472 nvgpu_mutex_acquire(&g->mm.tlb_lock);
1473
1474 reg_val = gk20a_readl(g, fb_mmu_invalidate_r());
1475
1476 reg_val |= fb_mmu_invalidate_all_va_true_f() |
1477 fb_mmu_invalidate_all_pdb_true_f() |
1478 invalidate_replay_val |
1479 fb_mmu_invalidate_trigger_true_f();
1480
1481 gk20a_writel(g, fb_mmu_invalidate_r(), reg_val);
1482
1483 /* retry 200 times */
1484 nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER);
1485 do {
1486 reg_val = gk20a_readl(g, fb_mmu_ctrl_r());
1487 if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) !=
1488 fb_mmu_ctrl_pri_fifo_empty_false_f()) {
1489 err = 0;
1490 break;
1491 }
1492 nvgpu_udelay(5);
1493 } while (!nvgpu_timeout_expired_msg(&timeout,
1494 "invalidate replay failed on 0x%llx"));
1495 if (err)
1496 nvgpu_err(g, "invalidate replay timedout");
1497
1498 nvgpu_mutex_release(&g->mm.tlb_lock);
1499
1500 return err;
1501}
1502
1503static int gv11b_fb_fix_page_fault(struct gk20a *g,
1504 struct mmu_fault_info *mmfault)
1505{
1506 int err = 0;
1507 u32 pte[2];
1508
1509 if (mmfault->refch == NULL) {
1510 nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL");
1511 return -EINVAL;
1512 }
1513
1514 err = __nvgpu_get_pte(g,
1515 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1516 if (err) {
1517 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found");
1518 return err;
1519 }
1520 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1521 "pte: %#08x %#08x", pte[1], pte[0]);
1522
1523 if (pte[0] == 0x0 && pte[1] == 0x0) {
1524 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1525 "pte all zeros, do not set valid");
1526 return -1;
1527 }
1528 if (pte[0] & gmmu_new_pte_valid_true_f()) {
1529 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1530 "pte valid already set");
1531 return -1;
1532 }
1533
1534 pte[0] |= gmmu_new_pte_valid_true_f();
1535 if (pte[0] & gmmu_new_pte_read_only_true_f())
1536 pte[0] &= ~(gmmu_new_pte_read_only_true_f());
1537 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1538 "new pte: %#08x %#08x", pte[1], pte[0]);
1539
1540 err = __nvgpu_set_pte(g,
1541 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1542 if (err) {
1543 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed");
1544 return err;
1545 }
1546 /* invalidate tlb so that GMMU does not use old cached translation */
1547 g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem);
1548
1549 err = __nvgpu_get_pte(g,
1550 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1551 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1552 "pte after tlb invalidate: %#08x %#08x",
1553 pte[1], pte[0]);
1554 return err;
1555}
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.h b/drivers/gpu/nvgpu/gv11b/fb_gv11b.h
new file mode 100644
index 00000000..d9a6fa77
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.h
@@ -0,0 +1,72 @@
1/*
2 * GV11B FB
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef _NVGPU_GV11B_FB
26#define _NVGPU_GV11B_FB
27
28#define STALL_REG_INDEX 0
29#define NONSTALL_REG_INDEX 1
30
31#define NONREPLAY_REG_INDEX 0
32#define REPLAY_REG_INDEX 1
33
34#define FAULT_BUF_DISABLED 0
35#define FAULT_BUF_ENABLED 1
36
37#define FAULT_BUF_INVALID 0
38#define FAULT_BUF_VALID 1
39
40#define HUB_INTR_TYPE_OTHER 1 /* bit 0 */
41#define HUB_INTR_TYPE_NONREPLAY 2 /* bit 1 */
42#define HUB_INTR_TYPE_REPLAY 4 /* bit 2 */
43#define HUB_INTR_TYPE_ECC_UNCORRECTED 8 /* bit 3 */
44#define HUB_INTR_TYPE_ACCESS_COUNTER 16 /* bit 4 */
45#define HUB_INTR_TYPE_ALL (HUB_INTR_TYPE_OTHER | \
46 HUB_INTR_TYPE_NONREPLAY | \
47 HUB_INTR_TYPE_REPLAY | \
48 HUB_INTR_TYPE_ECC_UNCORRECTED | \
49 HUB_INTR_TYPE_ACCESS_COUNTER)
50
51#define FAULT_TYPE_OTHER_AND_NONREPLAY 0
52#define FAULT_TYPE_REPLAY 1
53
54struct gk20a;
55
56void gv11b_fb_init_fs_state(struct gk20a *g);
57void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr);
58void gv11b_fb_reset(struct gk20a *g);
59void gv11b_fb_hub_isr(struct gk20a *g);
60
61u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g,
62 unsigned int index);
63void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g,
64 unsigned int index, unsigned int state);
65void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index);
66void gv11b_fb_enable_hub_intr(struct gk20a *g,
67 unsigned int index, unsigned int intr_type);
68void gv11b_fb_disable_hub_intr(struct gk20a *g,
69 unsigned int index, unsigned int intr_type);
70bool gv11b_fb_mmu_fault_pending(struct gk20a *g);
71
72#endif
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
new file mode 100644
index 00000000..f87c6dea
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
@@ -0,0 +1,1907 @@
1/*
2 * GV11B fifo
3 *
4 * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24#include <linux/delay.h>
25#include <linux/types.h>
26
27#include <nvgpu/semaphore.h>
28#include <nvgpu/timers.h>
29#include <nvgpu/log.h>
30#include <nvgpu/dma.h>
31#include <nvgpu/nvgpu_mem.h>
32#include <nvgpu/gmmu.h>
33#include <nvgpu/soc.h>
34#include <nvgpu/debug.h>
35#include <nvgpu/nvhost_t19x.h>
36#include <nvgpu/barrier.h>
37#include <nvgpu/mm.h>
38#include <nvgpu/ctxsw_trace.h>
39
40#include "gk20a/gk20a.h"
41#include "gk20a/fifo_gk20a.h"
42#include "gk20a/channel_gk20a.h"
43
44#include "gp10b/fifo_gp10b.h"
45
46#include <nvgpu/hw/gv11b/hw_pbdma_gv11b.h>
47#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
48#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
49#include <nvgpu/hw/gv11b/hw_ccsr_gv11b.h>
50#include <nvgpu/hw/gv11b/hw_usermode_gv11b.h>
51#include <nvgpu/hw/gv11b/hw_top_gv11b.h>
52#include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h>
53#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
54#include <nvgpu/hw/gv11b/hw_gr_gv11b.h>
55
56#include "fifo_gv11b.h"
57#include "subctx_gv11b.h"
58#include "gr_gv11b.h"
59#include "mc_gv11b.h"
60
61#define PBDMA_SUBDEVICE_ID 1
62
63static void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g,
64 struct channel_gk20a *ch, struct nvgpu_mem *mem);
65
66void gv11b_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist)
67{
68
69 u32 runlist_entry_0 = ram_rl_entry_type_tsg_v();
70
71 if (tsg->timeslice_timeout)
72 runlist_entry_0 |=
73 ram_rl_entry_tsg_timeslice_scale_f(tsg->timeslice_scale) |
74 ram_rl_entry_tsg_timeslice_timeout_f(tsg->timeslice_timeout);
75 else
76 runlist_entry_0 |=
77 ram_rl_entry_tsg_timeslice_scale_f(
78 ram_rl_entry_tsg_timeslice_scale_3_v()) |
79 ram_rl_entry_tsg_timeslice_timeout_f(
80 ram_rl_entry_tsg_timeslice_timeout_128_v());
81
82 runlist[0] = runlist_entry_0;
83 runlist[1] = ram_rl_entry_tsg_length_f(tsg->num_active_channels);
84 runlist[2] = ram_rl_entry_tsg_tsgid_f(tsg->tsgid);
85 runlist[3] = 0;
86
87 gk20a_dbg_info("gv11b tsg runlist [0] %x [1] %x [2] %x [3] %x\n",
88 runlist[0], runlist[1], runlist[2], runlist[3]);
89
90}
91
92void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist)
93{
94 struct gk20a *g = c->g;
95 u32 addr_lo, addr_hi;
96 u32 runlist_entry;
97
98 /* Time being use 0 pbdma sequencer */
99 runlist_entry = ram_rl_entry_type_channel_v() |
100 ram_rl_entry_chan_runqueue_selector_f(
101 c->t19x.runqueue_sel) |
102 ram_rl_entry_chan_userd_target_f(
103 nvgpu_aperture_mask(g, &g->fifo.userd,
104 ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(),
105 ram_rl_entry_chan_userd_target_vid_mem_v())) |
106 ram_rl_entry_chan_inst_target_f(
107 nvgpu_aperture_mask(g, &c->inst_block,
108 ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(),
109 ram_rl_entry_chan_inst_target_vid_mem_v()));
110
111 addr_lo = u64_lo32(c->userd_iova) >>
112 ram_rl_entry_chan_userd_ptr_align_shift_v();
113 addr_hi = u64_hi32(c->userd_iova);
114 runlist[0] = runlist_entry | ram_rl_entry_chan_userd_ptr_lo_f(addr_lo);
115 runlist[1] = ram_rl_entry_chan_userd_ptr_hi_f(addr_hi);
116
117 addr_lo = u64_lo32(nvgpu_inst_block_addr(g, &c->inst_block)) >>
118 ram_rl_entry_chan_inst_ptr_align_shift_v();
119 addr_hi = u64_hi32(nvgpu_inst_block_addr(g, &c->inst_block));
120
121 runlist[2] = ram_rl_entry_chan_inst_ptr_lo_f(addr_lo) |
122 ram_rl_entry_chid_f(c->chid);
123 runlist[3] = ram_rl_entry_chan_inst_ptr_hi_f(addr_hi);
124
125 gk20a_dbg_info("gv11b channel runlist [0] %x [1] %x [2] %x [3] %x\n",
126 runlist[0], runlist[1], runlist[2], runlist[3]);
127}
128
129static void gv11b_userd_writeback_config(struct gk20a *g)
130{
131 gk20a_writel(g, fifo_userd_writeback_r(), fifo_userd_writeback_timer_f(
132 fifo_userd_writeback_timer_100us_v()));
133
134
135}
136
137int channel_gv11b_setup_ramfc(struct channel_gk20a *c,
138 u64 gpfifo_base, u32 gpfifo_entries,
139 unsigned long acquire_timeout, u32 flags)
140{
141 struct gk20a *g = c->g;
142 struct nvgpu_mem *mem = &c->inst_block;
143 u32 data;
144
145 gk20a_dbg_fn("");
146
147 nvgpu_memset(g, mem, 0, 0, ram_fc_size_val_v());
148
149 nvgpu_mem_wr32(g, mem, ram_fc_gp_base_w(),
150 pbdma_gp_base_offset_f(
151 u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s())));
152
153 nvgpu_mem_wr32(g, mem, ram_fc_gp_base_hi_w(),
154 pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) |
155 pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
156
157 nvgpu_mem_wr32(g, mem, ram_fc_signature_w(),
158 c->g->ops.fifo.get_pbdma_signature(c->g));
159
160 nvgpu_mem_wr32(g, mem, ram_fc_pb_header_w(),
161 pbdma_pb_header_priv_user_f() |
162 pbdma_pb_header_method_zero_f() |
163 pbdma_pb_header_subchannel_zero_f() |
164 pbdma_pb_header_level_main_f() |
165 pbdma_pb_header_first_true_f() |
166 pbdma_pb_header_type_inc_f());
167
168 nvgpu_mem_wr32(g, mem, ram_fc_subdevice_w(),
169 pbdma_subdevice_id_f(PBDMA_SUBDEVICE_ID) |
170 pbdma_subdevice_status_active_f() |
171 pbdma_subdevice_channel_dma_enable_f());
172
173 nvgpu_mem_wr32(g, mem, ram_fc_target_w(),
174 pbdma_target_eng_ctx_valid_true_f() |
175 pbdma_target_ce_ctx_valid_true_f() |
176 pbdma_target_engine_sw_f());
177
178 nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(),
179 g->ops.fifo.pbdma_acquire_val(acquire_timeout));
180
181 nvgpu_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(),
182 pbdma_runlist_timeslice_timeout_128_f() |
183 pbdma_runlist_timeslice_timescale_3_f() |
184 pbdma_runlist_timeslice_enable_true_f());
185
186
187 nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid));
188
189 if (c->t19x.subctx_id == CHANNEL_INFO_VEID0)
190 nvgpu_mem_wr32(g, mem, ram_fc_set_channel_info_w(),
191 pbdma_set_channel_info_scg_type_graphics_compute0_f() |
192 pbdma_set_channel_info_veid_f(c->t19x.subctx_id));
193 else
194 nvgpu_mem_wr32(g, mem, ram_fc_set_channel_info_w(),
195 pbdma_set_channel_info_scg_type_compute1_f() |
196 pbdma_set_channel_info_veid_f(c->t19x.subctx_id));
197
198 gv11b_fifo_init_ramfc_eng_method_buffer(g, c, mem);
199
200 if (c->is_privileged_channel) {
201 /* Set privilege level for channel */
202 nvgpu_mem_wr32(g, mem, ram_fc_config_w(),
203 pbdma_config_auth_level_privileged_f());
204
205 gk20a_fifo_setup_ramfc_for_privileged_channel(c);
206 }
207
208 /* Enable userd writeback */
209 data = nvgpu_mem_rd32(g, mem, ram_fc_config_w());
210 data = data | pbdma_config_userd_writeback_enable_f();
211 nvgpu_mem_wr32(g, mem, ram_fc_config_w(),data);
212
213 gv11b_userd_writeback_config(g);
214
215 return channel_gp10b_commit_userd(c);
216}
217
218
219static void gv11b_ring_channel_doorbell(struct channel_gk20a *c)
220{
221 struct fifo_gk20a *f = &c->g->fifo;
222 u32 hw_chid = f->channel_base + c->chid;
223
224 gk20a_dbg_info("channel ring door bell %d\n", c->chid);
225
226 gv11b_usermode_writel(c->g, usermode_notify_channel_pending_r(),
227 usermode_notify_channel_pending_id_f(hw_chid));
228}
229
230u32 gv11b_userd_gp_get(struct gk20a *g, struct channel_gk20a *c)
231{
232 struct nvgpu_mem *userd_mem = &g->fifo.userd;
233 u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32));
234
235 return nvgpu_mem_rd32(g, userd_mem,
236 offset + ram_userd_gp_get_w());
237}
238
239u64 gv11b_userd_pb_get(struct gk20a *g, struct channel_gk20a *c)
240{
241 struct nvgpu_mem *userd_mem = &g->fifo.userd;
242 u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32));
243 u32 lo = nvgpu_mem_rd32(g, userd_mem, offset + ram_userd_get_w());
244 u32 hi = nvgpu_mem_rd32(g, userd_mem, offset + ram_userd_get_hi_w());
245
246 return ((u64)hi << 32) | lo;
247}
248
249void gv11b_userd_gp_put(struct gk20a *g, struct channel_gk20a *c)
250{
251 struct nvgpu_mem *userd_mem = &g->fifo.userd;
252 u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32));
253
254 nvgpu_mem_wr32(g, userd_mem, offset + ram_userd_gp_put_w(),
255 c->gpfifo.put);
256 /* commit everything to cpu */
257 nvgpu_smp_mb();
258
259 gv11b_ring_channel_doorbell(c);
260}
261
262void channel_gv11b_unbind(struct channel_gk20a *ch)
263{
264 struct gk20a *g = ch->g;
265
266 gk20a_dbg_fn("");
267
268 if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) {
269 gk20a_writel(g, ccsr_channel_inst_r(ch->chid),
270 ccsr_channel_inst_ptr_f(0) |
271 ccsr_channel_inst_bind_false_f());
272
273 gk20a_writel(g, ccsr_channel_r(ch->chid),
274 ccsr_channel_enable_clr_true_f() |
275 ccsr_channel_pbdma_faulted_reset_f() |
276 ccsr_channel_eng_faulted_reset_f());
277 }
278}
279
280u32 gv11b_fifo_get_num_fifos(struct gk20a *g)
281{
282 return ccsr_channel__size_1_v();
283}
284
285bool gv11b_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid)
286{
287 return (engine_subid == gmmu_fault_client_type_gpc_v());
288}
289
290void gv11b_dump_channel_status_ramfc(struct gk20a *g,
291 struct gk20a_debug_output *o,
292 u32 chid,
293 struct ch_state *ch_state)
294{
295 u32 channel = gk20a_readl(g, ccsr_channel_r(chid));
296 u32 status = ccsr_channel_status_v(channel);
297 u32 *inst_mem;
298 struct channel_gk20a *c = g->fifo.channel + chid;
299 struct nvgpu_semaphore_int *hw_sema = NULL;
300
301 if (c->hw_sema)
302 hw_sema = c->hw_sema;
303
304 if (!ch_state)
305 return;
306
307 inst_mem = &ch_state->inst_block[0];
308
309 gk20a_debug_output(o, "%d-%s, pid %d, refs: %d: ", chid,
310 g->name,
311 ch_state->pid,
312 ch_state->refs);
313 gk20a_debug_output(o, "channel status: %s in use %s %s\n",
314 ccsr_channel_enable_v(channel) ? "" : "not",
315 gk20a_decode_ccsr_chan_status(status),
316 ccsr_channel_busy_v(channel) ? "busy" : "not busy");
317 gk20a_debug_output(o, "RAMFC : TOP: %016llx PUT: %016llx GET: %016llx "
318 "FETCH: %016llx\nHEADER: %08x COUNT: %08x\n"
319 "SEMAPHORE: addr hi: %08x addr lo: %08x\n"
320 "payload %08x execute %08x\n",
321 (u64)inst_mem[ram_fc_pb_top_level_get_w()] +
322 ((u64)inst_mem[ram_fc_pb_top_level_get_hi_w()] << 32ULL),
323 (u64)inst_mem[ram_fc_pb_put_w()] +
324 ((u64)inst_mem[ram_fc_pb_put_hi_w()] << 32ULL),
325 (u64)inst_mem[ram_fc_pb_get_w()] +
326 ((u64)inst_mem[ram_fc_pb_get_hi_w()] << 32ULL),
327 (u64)inst_mem[ram_fc_pb_fetch_w()] +
328 ((u64)inst_mem[ram_fc_pb_fetch_hi_w()] << 32ULL),
329 inst_mem[ram_fc_pb_header_w()],
330 inst_mem[ram_fc_pb_count_w()],
331 inst_mem[ram_fc_sem_addr_hi_w()],
332 inst_mem[ram_fc_sem_addr_lo_w()],
333 inst_mem[ram_fc_sem_payload_lo_w()],
334 inst_mem[ram_fc_sem_execute_w()]);
335 if (hw_sema)
336 gk20a_debug_output(o, "SEMA STATE: value: 0x%08x next_val: 0x%08x addr: 0x%010llx\n",
337 __nvgpu_semaphore_read(hw_sema),
338 nvgpu_atomic_read(&hw_sema->next_value),
339 nvgpu_hw_sema_addr(hw_sema));
340 gk20a_debug_output(o, "\n");
341}
342
343void gv11b_dump_eng_status(struct gk20a *g,
344 struct gk20a_debug_output *o)
345{
346 u32 i, host_num_engines;
347
348 host_num_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
349
350 for (i = 0; i < host_num_engines; i++) {
351 u32 status = gk20a_readl(g, fifo_engine_status_r(i));
352 u32 ctx_status = fifo_engine_status_ctx_status_v(status);
353
354 gk20a_debug_output(o, "%s eng %d: ", g->name, i);
355 gk20a_debug_output(o,
356 "id: %d (%s), next_id: %d (%s), ctx status: %s ",
357 fifo_engine_status_id_v(status),
358 fifo_engine_status_id_type_v(status) ?
359 "tsg" : "channel",
360 fifo_engine_status_next_id_v(status),
361 fifo_engine_status_next_id_type_v(status) ?
362 "tsg" : "channel",
363 gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status));
364
365 if (fifo_engine_status_eng_reload_v(status))
366 gk20a_debug_output(o, "ctx_reload ");
367 if (fifo_engine_status_faulted_v(status))
368 gk20a_debug_output(o, "faulted ");
369 if (fifo_engine_status_engine_v(status))
370 gk20a_debug_output(o, "busy ");
371 gk20a_debug_output(o, "\n");
372 }
373 gk20a_debug_output(o, "\n");
374}
375
376u32 gv11b_fifo_intr_0_error_mask(struct gk20a *g)
377{
378 u32 intr_0_error_mask =
379 fifo_intr_0_bind_error_pending_f() |
380 fifo_intr_0_sched_error_pending_f() |
381 fifo_intr_0_chsw_error_pending_f() |
382 fifo_intr_0_fb_flush_timeout_pending_f() |
383 fifo_intr_0_lb_error_pending_f();
384
385 return intr_0_error_mask;
386}
387
388u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g)
389{
390 return gk20a_get_gr_idle_timeout(g);
391}
392
393static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id,
394 u32 pbdma_id, unsigned int timeout_rc_type)
395{
396 struct nvgpu_timeout timeout;
397 unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */
398 u32 pbdma_stat;
399 u32 chan_stat;
400 int ret = -EBUSY;
401
402 /* timeout in milli seconds */
403 nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g),
404 NVGPU_TIMER_CPU_TIMER);
405
406 nvgpu_log(g, gpu_dbg_info, "wait preempt pbdma %d", pbdma_id);
407 /* Verify that ch/tsg is no longer on the pbdma */
408 do {
409 /*
410 * If the PBDMA has a stalling interrupt and receives a NACK,
411 * the PBDMA won't save out until the STALLING interrupt is
412 * cleared. Stalling interrupt need not be directly addressed,
413 * as simply clearing of the interrupt bit will be sufficient
414 * to allow the PBDMA to save out. If the stalling interrupt
415 * was due to a SW method or another deterministic failure,
416 * the PBDMA will assert it when the channel is reloaded
417 * or resumed. Note that the fault will still be
418 * reported to SW.
419 */
420
421 gk20a_fifo_handle_pbdma_intr(g, &g->fifo, pbdma_id, RC_NO);
422
423 pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(pbdma_id));
424 chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat);
425
426 if (chan_stat ==
427 fifo_pbdma_status_chan_status_valid_v() ||
428 chan_stat ==
429 fifo_pbdma_status_chan_status_chsw_save_v()) {
430
431 if (id != fifo_pbdma_status_id_v(pbdma_stat)) {
432 ret = 0;
433 break;
434 }
435
436 } else if (chan_stat ==
437 fifo_pbdma_status_chan_status_chsw_load_v()) {
438
439 if (id != fifo_pbdma_status_next_id_v(pbdma_stat)) {
440 ret = 0;
441 break;
442 }
443
444 } else if (chan_stat ==
445 fifo_pbdma_status_chan_status_chsw_switch_v()) {
446
447 if ((id != fifo_pbdma_status_next_id_v(pbdma_stat)) &&
448 (id != fifo_pbdma_status_id_v(pbdma_stat))) {
449 ret = 0;
450 break;
451 }
452 } else {
453 /* pbdma status is invalid i.e. it is not loaded */
454 ret = 0;
455 break;
456 }
457
458 usleep_range(delay, delay * 2);
459 delay = min_t(unsigned long,
460 delay << 1, GR_IDLE_CHECK_MAX);
461 } while (!nvgpu_timeout_expired_msg(&timeout,
462 "preempt timeout pbdma"));
463 return ret;
464}
465
466static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id,
467 u32 act_eng_id, u32 *reset_eng_bitmask,
468 unsigned int timeout_rc_type)
469{
470 struct nvgpu_timeout timeout;
471 unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */
472 u32 eng_stat;
473 u32 ctx_stat;
474 int ret = -EBUSY;
475 bool stall_intr = false;
476
477 /* timeout in milli seconds */
478 nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g),
479 NVGPU_TIMER_CPU_TIMER);
480
481 nvgpu_log(g, gpu_dbg_info, "wait preempt act engine id: %u",
482 act_eng_id);
483 /* Check if ch/tsg has saved off the engine or if ctxsw is hung */
484 do {
485 eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id));
486 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
487
488 if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) {
489 stall_intr = true;
490 nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr,
491 "stall intr set, "
492 "preemption will not finish");
493 }
494 if (ctx_stat ==
495 fifo_engine_status_ctx_status_ctxsw_switch_v()) {
496 /* Eng save hasn't started yet. Continue polling */
497
498 } else if (ctx_stat ==
499 fifo_engine_status_ctx_status_valid_v() ||
500 ctx_stat ==
501 fifo_engine_status_ctx_status_ctxsw_save_v()) {
502
503 if (id == fifo_engine_status_id_v(eng_stat)) {
504 if (stall_intr ||
505 timeout_rc_type == PREEMPT_TIMEOUT_NORC) {
506 /* preemption will not finish */
507 *reset_eng_bitmask |= BIT(act_eng_id);
508 ret = 0;
509 break;
510 }
511 } else {
512 /* context is not running on the engine */
513 ret = 0;
514 break;
515 }
516
517 } else if (ctx_stat ==
518 fifo_engine_status_ctx_status_ctxsw_load_v()) {
519
520 if (id == fifo_engine_status_next_id_v(eng_stat)) {
521
522 if (stall_intr ||
523 timeout_rc_type == PREEMPT_TIMEOUT_NORC) {
524 /* preemption will not finish */
525 *reset_eng_bitmask |= BIT(act_eng_id);
526 ret = 0;
527 break;
528 }
529 } else {
530 /* context is not running on the engine */
531 ret = 0;
532 break;
533 }
534
535 } else {
536 /* Preempt should be finished */
537 ret = 0;
538 break;
539 }
540 nvgpu_usleep_range(delay, delay * 2);
541 delay = min_t(unsigned long,
542 delay << 1, GR_IDLE_CHECK_MAX);
543 } while (!nvgpu_timeout_expired_msg(&timeout,
544 "preempt timeout eng"));
545 return ret;
546}
547
548static void gv11b_reset_eng_faulted_ch(struct gk20a *g, u32 chid)
549{
550 u32 reg_val;
551
552 reg_val = gk20a_readl(g, ccsr_channel_r(chid));
553 reg_val |= ccsr_channel_eng_faulted_reset_f();
554 gk20a_writel(g, ccsr_channel_r(chid), reg_val);
555}
556
557static void gv11b_reset_eng_faulted_tsg(struct tsg_gk20a *tsg)
558{
559 struct gk20a *g = tsg->g;
560 struct channel_gk20a *ch;
561
562 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
563 list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
564 gv11b_reset_eng_faulted_ch(g, ch->chid);
565 }
566 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
567}
568
569static void gv11b_reset_pbdma_faulted_ch(struct gk20a *g, u32 chid)
570{
571 u32 reg_val;
572
573 reg_val = gk20a_readl(g, ccsr_channel_r(chid));
574 reg_val |= ccsr_channel_pbdma_faulted_reset_f();
575 gk20a_writel(g, ccsr_channel_r(chid), reg_val);
576}
577
578static void gv11b_reset_pbdma_faulted_tsg(struct tsg_gk20a *tsg)
579{
580 struct gk20a *g = tsg->g;
581 struct channel_gk20a *ch;
582
583 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
584 list_for_each_entry(ch, &tsg->ch_list, ch_entry) {
585 gv11b_reset_pbdma_faulted_ch(g, ch->chid);
586 }
587 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
588}
589
590void gv11b_fifo_reset_pbdma_and_eng_faulted(struct gk20a *g,
591 struct channel_gk20a *refch,
592 u32 faulted_pbdma, u32 faulted_engine)
593{
594 struct tsg_gk20a *tsg;
595
596 nvgpu_log(g, gpu_dbg_intr, "reset faulted pbdma:0x%x eng:0x%x",
597 faulted_pbdma, faulted_engine);
598
599 if (gk20a_is_channel_marked_as_tsg(refch)) {
600 tsg = &g->fifo.tsg[refch->tsgid];
601 if (faulted_pbdma != FIFO_INVAL_PBDMA_ID)
602 gv11b_reset_pbdma_faulted_tsg(tsg);
603 if (faulted_engine != FIFO_INVAL_ENGINE_ID)
604 gv11b_reset_eng_faulted_tsg(tsg);
605 } else {
606 if (faulted_pbdma != FIFO_INVAL_PBDMA_ID)
607 gv11b_reset_pbdma_faulted_ch(g, refch->chid);
608 if (faulted_engine != FIFO_INVAL_ENGINE_ID)
609 gv11b_reset_eng_faulted_ch(g, refch->chid);
610 }
611}
612
613static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask,
614 u32 id, unsigned int id_type, unsigned int rc_type,
615 struct mmu_fault_info *mmfault)
616{
617 u32 runlists_mask = 0;
618 struct fifo_gk20a *f = &g->fifo;
619 struct fifo_runlist_info_gk20a *runlist;
620 u32 pbdma_bitmask = 0;
621
622 if (id_type != ID_TYPE_UNKNOWN) {
623 if (id_type == ID_TYPE_TSG)
624 runlists_mask |= fifo_sched_disable_runlist_m(
625 f->tsg[id].runlist_id);
626 else
627 runlists_mask |= fifo_sched_disable_runlist_m(
628 f->channel[id].runlist_id);
629 }
630
631 if (rc_type == RC_TYPE_MMU_FAULT && mmfault) {
632 if (mmfault->faulted_pbdma != FIFO_INVAL_PBDMA_ID)
633 pbdma_bitmask = BIT(mmfault->faulted_pbdma);
634
635 for (id = 0; id < f->max_runlists; id++) {
636
637 runlist = &f->runlist_info[id];
638
639 if (runlist->eng_bitmask & act_eng_bitmask)
640 runlists_mask |=
641 fifo_sched_disable_runlist_m(id);
642
643 if (runlist->pbdma_bitmask & pbdma_bitmask)
644 runlists_mask |=
645 fifo_sched_disable_runlist_m(id);
646 }
647 }
648
649 if (id_type == ID_TYPE_UNKNOWN) {
650 for (id = 0; id < f->max_runlists; id++) {
651 if (act_eng_bitmask) {
652 /* eng ids are known */
653 runlist = &f->runlist_info[id];
654 if (runlist->eng_bitmask & act_eng_bitmask)
655 runlists_mask |=
656 fifo_sched_disable_runlist_m(id);
657 } else {
658 runlists_mask |=
659 fifo_sched_disable_runlist_m(id);
660 }
661 }
662 }
663 gk20a_dbg_info("runlists_mask = %08x", runlists_mask);
664 return runlists_mask;
665}
666
667static void gv11b_fifo_runlist_event_intr_disable(struct gk20a *g)
668{
669 u32 reg_val;
670
671 reg_val = gk20a_readl(g, fifo_intr_en_0_r());
672 reg_val &= fifo_intr_0_runlist_event_pending_f();
673 gk20a_writel(g, fifo_intr_en_0_r(), reg_val);
674}
675
676static void gv11b_fifo_runlist_event_intr_enable(struct gk20a *g)
677{
678 u32 reg_val;
679
680 reg_val = gk20a_readl(g, fifo_intr_en_0_r());
681 reg_val |= fifo_intr_0_runlist_event_pending_f();
682 gk20a_writel(g, fifo_intr_en_0_r(), reg_val);
683}
684
685static void gv11b_fifo_issue_runlist_preempt(struct gk20a *g,
686 u32 runlists_mask)
687{
688 u32 reg_val;
689
690 /* issue runlist preempt */
691 reg_val = gk20a_readl(g, fifo_runlist_preempt_r());
692 reg_val |= runlists_mask;
693 gk20a_writel(g, fifo_runlist_preempt_r(), reg_val);
694}
695
696static int gv11b_fifo_poll_runlist_preempt_pending(struct gk20a *g,
697 u32 runlists_mask)
698{
699 struct nvgpu_timeout timeout;
700 u32 delay = GR_IDLE_CHECK_DEFAULT;
701 int ret = -EBUSY;
702
703 nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g),
704 NVGPU_TIMER_CPU_TIMER);
705 do {
706 if (!((gk20a_readl(g, fifo_runlist_preempt_r())) &
707 runlists_mask)) {
708 ret = 0;
709 break;
710 }
711
712 nvgpu_usleep_range(delay, delay * 2);
713 delay = min_t(unsigned long,
714 delay << 1, GR_IDLE_CHECK_MAX);
715 } while (!nvgpu_timeout_expired_msg(&timeout,
716 "runlist preempt timeout"));
717 return ret;
718}
719
720int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id,
721 unsigned int id_type, unsigned int timeout_rc_type)
722{
723 struct fifo_gk20a *f = &g->fifo;
724 unsigned long runlist_served_pbdmas;
725 unsigned long runlist_served_engines;
726 u32 pbdma_id;
727 u32 act_eng_id;
728 u32 runlist_id;
729 int func_ret;
730 int ret = 0;
731 u32 tsgid;
732
733 if (id_type == ID_TYPE_TSG) {
734 runlist_id = f->tsg[id].runlist_id;
735 tsgid = id;
736 } else {
737 runlist_id = f->channel[id].runlist_id;
738 tsgid = f->channel[id].tsgid;
739 }
740
741 nvgpu_log_info(g, "Check preempt pending for tsgid = %u", tsgid);
742
743 runlist_served_pbdmas = f->runlist_info[runlist_id].pbdma_bitmask;
744 runlist_served_engines = f->runlist_info[runlist_id].eng_bitmask;
745
746 for_each_set_bit(pbdma_id, &runlist_served_pbdmas, f->num_pbdma) {
747
748 func_ret = gv11b_fifo_poll_pbdma_chan_status(g, tsgid, pbdma_id,
749 timeout_rc_type);
750 if (func_ret != 0) {
751 gk20a_dbg_info("preempt timeout pbdma %d", pbdma_id);
752 ret |= func_ret;
753 }
754 }
755
756 f->runlist_info[runlist_id].reset_eng_bitmask = 0;
757
758 for_each_set_bit(act_eng_id, &runlist_served_engines, f->max_engines) {
759
760 func_ret = gv11b_fifo_poll_eng_ctx_status(g, tsgid, act_eng_id,
761 &f->runlist_info[runlist_id].reset_eng_bitmask,
762 timeout_rc_type);
763
764 if (func_ret != 0) {
765 gk20a_dbg_info("preempt timeout engine %d", act_eng_id);
766 ret |= func_ret;
767 }
768 }
769
770 return ret;
771}
772
773int gv11b_fifo_preempt_channel(struct gk20a *g, u32 chid)
774{
775 struct fifo_gk20a *f = &g->fifo;
776 u32 tsgid;
777
778 tsgid = f->channel[chid].tsgid;
779 nvgpu_log_info(g, "chid:%d tsgid:%d", chid, tsgid);
780
781 /* Preempt tsg. Channel preempt is NOOP */
782 return g->ops.fifo.preempt_tsg(g, tsgid);
783}
784
785static int __locked_fifo_preempt_runlists(struct gk20a *g, u32 runlists_mask)
786{
787 int ret;
788
789 /*
790 * Disable runlist event interrupt as it will get
791 * triggered after runlist preempt finishes
792 */
793 gv11b_fifo_runlist_event_intr_disable(g);
794
795 /* issue runlist preempt */
796 gv11b_fifo_issue_runlist_preempt(g, runlists_mask);
797
798 /* poll for runlist preempt done */
799 ret = gv11b_fifo_poll_runlist_preempt_pending(g, runlists_mask);
800
801 /* Clear outstanding runlist event */
802 gk20a_fifo_handle_runlist_event(g);
803
804 /* Enable runlist event interrupt*/
805 gv11b_fifo_runlist_event_intr_enable(g);
806
807 return ret;
808}
809
810/* TSG enable sequence applicable for Volta and onwards */
811int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg)
812{
813 struct gk20a *g = tsg->g;
814 struct channel_gk20a *ch;
815
816 nvgpu_rwsem_down_read(&tsg->ch_list_lock);
817 nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) {
818 g->ops.fifo.enable_channel(ch);
819 }
820 nvgpu_rwsem_up_read(&tsg->ch_list_lock);
821
822 return 0;
823}
824
825int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid)
826{
827 struct fifo_gk20a *f = &g->fifo;
828 u32 ret = 0;
829 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
830 u32 mutex_ret = 0;
831 u32 runlist_id;
832
833 gk20a_dbg_fn("%d", tsgid);
834
835 runlist_id = f->tsg[tsgid].runlist_id;
836 gk20a_dbg_fn("runlist_id %d", runlist_id);
837
838 nvgpu_mutex_acquire(&f->runlist_info[runlist_id].mutex);
839
840 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
841
842 ret = __locked_fifo_preempt(g, tsgid, true);
843
844 if (!mutex_ret)
845 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
846
847 nvgpu_mutex_release(&f->runlist_info[runlist_id].mutex);
848
849 return ret;
850}
851
852
853static int gv11b_fifo_preempt_runlists(struct gk20a *g, u32 runlists_mask)
854{
855 int ret = 0;
856 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
857 u32 mutex_ret = 0;
858 u32 runlist_id;
859
860 gk20a_dbg_fn("");
861
862 for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) {
863 if (runlists_mask & fifo_runlist_preempt_runlist_m(runlist_id))
864 nvgpu_mutex_acquire(&g->fifo.
865 runlist_info[runlist_id].mutex);
866 }
867
868 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
869
870 ret = __locked_fifo_preempt_runlists(g, runlists_mask);
871
872 if (!mutex_ret)
873 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
874
875 for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) {
876 if (runlists_mask & fifo_runlist_preempt_runlist_m(runlist_id))
877 nvgpu_mutex_release(&g->fifo.
878 runlist_info[runlist_id].mutex);
879 }
880
881 return ret;
882}
883
884static int __locked_fifo_preempt_ch_tsg(struct gk20a *g, u32 id,
885 unsigned int id_type, unsigned int timeout_rc_type)
886{
887 int ret;
888 struct fifo_gk20a *f = &g->fifo;
889
890 nvgpu_log_fn(g, "id:%d id_type:%d", id, id_type);
891
892 /* Issue tsg preempt. Channel preempt is noop */
893 if (id_type == ID_TYPE_CHANNEL)
894 gk20a_fifo_issue_preempt(g, f->channel[id].tsgid, true);
895 else
896 gk20a_fifo_issue_preempt(g, id, true);
897
898 /* wait for preempt */
899 ret = g->ops.fifo.is_preempt_pending(g, id, id_type,
900 timeout_rc_type);
901
902 if (ret && (timeout_rc_type == PREEMPT_TIMEOUT_RC))
903 __locked_fifo_preempt_timeout_rc(g, id, id_type);
904
905 return ret;
906}
907
908
909int gv11b_fifo_preempt_ch_tsg(struct gk20a *g, u32 id,
910 unsigned int id_type, unsigned int timeout_rc_type)
911{
912 struct fifo_gk20a *f = &g->fifo;
913 u32 ret = 0;
914 u32 token = PMU_INVALID_MUTEX_OWNER_ID;
915 u32 mutex_ret = 0;
916 u32 runlist_id;
917
918 if (id_type == ID_TYPE_TSG)
919 runlist_id = f->tsg[id].runlist_id;
920 else if (id_type == ID_TYPE_CHANNEL)
921 runlist_id = f->channel[id].runlist_id;
922 else
923 return -EINVAL;
924
925 if (runlist_id >= g->fifo.max_runlists) {
926 gk20a_dbg_info("runlist_id = %d", runlist_id);
927 return -EINVAL;
928 }
929
930 gk20a_dbg_fn("preempt id = %d, runlist_id = %d", id, runlist_id);
931
932 nvgpu_mutex_acquire(&f->runlist_info[runlist_id].mutex);
933
934 mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
935
936 ret = __locked_fifo_preempt_ch_tsg(g, id, id_type, timeout_rc_type);
937
938 if (!mutex_ret)
939 nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
940
941 nvgpu_mutex_release(&f->runlist_info[runlist_id].mutex);
942
943 return ret;
944
945}
946
947void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask,
948 u32 id, unsigned int id_type, unsigned int rc_type,
949 struct mmu_fault_info *mmfault)
950{
951 bool verbose = false;
952 struct tsg_gk20a *tsg = NULL;
953 struct channel_gk20a *refch = NULL;
954 u32 runlists_mask, runlist_id;
955 struct fifo_runlist_info_gk20a *runlist = NULL;
956 u32 engine_id, client_type = ~0;
957
958 gk20a_dbg_info("active engine ids bitmask =0x%x", act_eng_bitmask);
959 gk20a_dbg_info("hw id =%d", id);
960 gk20a_dbg_info("id_type =%d", id_type);
961 gk20a_dbg_info("rc_type =%d", rc_type);
962 gk20a_dbg_info("mmu_fault =0x%p", mmfault);
963
964 runlists_mask = gv11b_fifo_get_runlists_mask(g, act_eng_bitmask, id,
965 id_type, rc_type, mmfault);
966
967 gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_DISABLED,
968 !RUNLIST_INFO_MUTEX_LOCKED);
969
970 g->fifo.deferred_reset_pending = false;
971
972 /* Disable power management */
973 if (g->support_pmu && g->elpg_enabled) {
974 if (nvgpu_pmu_disable_elpg(g))
975 nvgpu_err(g, "failed to set disable elpg");
976 }
977 if (g->ops.clock_gating.slcg_gr_load_gating_prod)
978 g->ops.clock_gating.slcg_gr_load_gating_prod(g,
979 false);
980 if (g->ops.clock_gating.slcg_perf_load_gating_prod)
981 g->ops.clock_gating.slcg_perf_load_gating_prod(g,
982 false);
983 if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
984 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
985 false);
986
987 gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN);
988
989 if (rc_type == RC_TYPE_MMU_FAULT)
990 gk20a_debug_dump(g);
991
992 /* get the channel/TSG */
993 if (rc_type == RC_TYPE_MMU_FAULT && mmfault && mmfault->refch) {
994 refch = mmfault->refch;
995 client_type = mmfault->client_type;
996 if (gk20a_is_channel_marked_as_tsg(refch))
997 tsg = &g->fifo.tsg[refch->tsgid];
998 gv11b_fifo_reset_pbdma_and_eng_faulted(g, refch,
999 mmfault->faulted_pbdma,
1000 mmfault->faulted_engine);
1001 } else {
1002 if (id_type == ID_TYPE_TSG)
1003 tsg = &g->fifo.tsg[id];
1004 else if (id_type == ID_TYPE_CHANNEL)
1005 refch = gk20a_channel_get(&g->fifo.channel[id]);
1006 }
1007
1008 if (id_type == ID_TYPE_TSG || id_type == ID_TYPE_CHANNEL) {
1009 g->ops.fifo.preempt_ch_tsg(g, id, id_type,
1010 PREEMPT_TIMEOUT_NORC);
1011 } else {
1012 gv11b_fifo_preempt_runlists(g, runlists_mask);
1013 }
1014
1015 if (tsg) {
1016 if (!g->fifo.deferred_reset_pending) {
1017 if (rc_type == RC_TYPE_MMU_FAULT) {
1018 gk20a_fifo_set_ctx_mmu_error_tsg(g, tsg);
1019 verbose = gk20a_fifo_error_tsg(g, tsg);
1020 }
1021 }
1022 gk20a_fifo_abort_tsg(g, tsg->tsgid, false);
1023 if (refch)
1024 gk20a_channel_put(refch);
1025 } else if (refch) {
1026 if (!g->fifo.deferred_reset_pending) {
1027 if (rc_type == RC_TYPE_MMU_FAULT) {
1028 gk20a_fifo_set_ctx_mmu_error_ch(g, refch);
1029 verbose = gk20a_fifo_error_ch(g, refch);
1030 }
1031 }
1032 gk20a_channel_abort(refch, false);
1033 gk20a_channel_put(refch);
1034 } else {
1035 nvgpu_err(g, "id unknown, abort runlist");
1036 for (runlist_id = 0; runlist_id < g->fifo.max_runlists;
1037 runlist_id++) {
1038 if (runlists_mask & BIT(runlist_id))
1039 g->ops.fifo.update_runlist(g, runlist_id,
1040 FIFO_INVAL_CHANNEL_ID, false, true);
1041 }
1042 }
1043
1044 /* check if engine reset should be deferred */
1045 for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) {
1046
1047 runlist = &g->fifo.runlist_info[runlist_id];
1048 if ((runlists_mask & BIT(runlist_id)) &&
1049 runlist->reset_eng_bitmask) {
1050
1051 unsigned long __reset_eng_bitmask =
1052 runlist->reset_eng_bitmask;
1053
1054 for_each_set_bit(engine_id, &__reset_eng_bitmask, 32) {
1055 if ((refch || tsg) &&
1056 gk20a_fifo_should_defer_engine_reset(g,
1057 engine_id, client_type, false)) {
1058
1059 g->fifo.deferred_fault_engines |=
1060 BIT(engine_id);
1061
1062 /* handled during channel free */
1063 g->fifo.deferred_reset_pending = true;
1064 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
1065 "sm debugger attached,"
1066 " deferring channel recovery to channel free");
1067 } else {
1068 /*
1069 * if lock is already taken, a reset is
1070 * taking place so no need to repeat
1071 */
1072 if (nvgpu_mutex_tryacquire(
1073 &g->fifo.gr_reset_mutex)) {
1074
1075 gk20a_fifo_reset_engine(g,
1076 engine_id);
1077
1078 nvgpu_mutex_release(
1079 &g->fifo.gr_reset_mutex);
1080 }
1081 }
1082 }
1083 }
1084 }
1085
1086#ifdef CONFIG_GK20A_CTXSW_TRACE
1087 if (refch)
1088 gk20a_ctxsw_trace_channel_reset(g, refch);
1089 else if (tsg)
1090 gk20a_ctxsw_trace_tsg_reset(g, tsg);
1091#endif
1092
1093 gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_ENABLED,
1094 !RUNLIST_INFO_MUTEX_LOCKED);
1095
1096 /* It is safe to enable ELPG again. */
1097 if (g->support_pmu && g->elpg_enabled)
1098 nvgpu_pmu_enable_elpg(g);
1099}
1100
1101void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f)
1102{
1103 /*
1104 * These are all errors which indicate something really wrong
1105 * going on in the device
1106 */
1107 f->intr.pbdma.device_fatal_0 =
1108 pbdma_intr_0_memreq_pending_f() |
1109 pbdma_intr_0_memack_timeout_pending_f() |
1110 pbdma_intr_0_memack_extra_pending_f() |
1111 pbdma_intr_0_memdat_timeout_pending_f() |
1112 pbdma_intr_0_memdat_extra_pending_f() |
1113 pbdma_intr_0_memflush_pending_f() |
1114 pbdma_intr_0_memop_pending_f() |
1115 pbdma_intr_0_lbconnect_pending_f() |
1116 pbdma_intr_0_lback_timeout_pending_f() |
1117 pbdma_intr_0_lback_extra_pending_f() |
1118 pbdma_intr_0_lbdat_timeout_pending_f() |
1119 pbdma_intr_0_lbdat_extra_pending_f() |
1120 pbdma_intr_0_pri_pending_f();
1121
1122 /*
1123 * These are data parsing, framing errors or others which can be
1124 * recovered from with intervention... or just resetting the
1125 * channel
1126 */
1127 f->intr.pbdma.channel_fatal_0 =
1128 pbdma_intr_0_gpfifo_pending_f() |
1129 pbdma_intr_0_gpptr_pending_f() |
1130 pbdma_intr_0_gpentry_pending_f() |
1131 pbdma_intr_0_gpcrc_pending_f() |
1132 pbdma_intr_0_pbptr_pending_f() |
1133 pbdma_intr_0_pbentry_pending_f() |
1134 pbdma_intr_0_pbcrc_pending_f() |
1135 pbdma_intr_0_method_pending_f() |
1136 pbdma_intr_0_methodcrc_pending_f() |
1137 pbdma_intr_0_pbseg_pending_f() |
1138 pbdma_intr_0_clear_faulted_error_pending_f() |
1139 pbdma_intr_0_eng_reset_pending_f() |
1140 pbdma_intr_0_semaphore_pending_f() |
1141 pbdma_intr_0_signature_pending_f();
1142
1143 /* Can be used for sw-methods, or represents a recoverable timeout. */
1144 f->intr.pbdma.restartable_0 =
1145 pbdma_intr_0_device_pending_f();
1146}
1147
1148static u32 gv11b_fifo_intr_0_en_mask(struct gk20a *g)
1149{
1150 u32 intr_0_en_mask;
1151
1152 intr_0_en_mask = g->ops.fifo.intr_0_error_mask(g);
1153
1154 intr_0_en_mask |= fifo_intr_0_runlist_event_pending_f() |
1155 fifo_intr_0_pbdma_intr_pending_f() |
1156 fifo_intr_0_ctxsw_timeout_pending_f();
1157
1158 return intr_0_en_mask;
1159}
1160
1161int gv11b_init_fifo_reset_enable_hw(struct gk20a *g)
1162{
1163 u32 intr_stall;
1164 u32 mask;
1165 u32 timeout;
1166 unsigned int i;
1167 u32 host_num_pbdma = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA);
1168
1169 gk20a_dbg_fn("");
1170
1171 /* enable pmc pfifo */
1172 g->ops.mc.reset(g, mc_enable_pfifo_enabled_f());
1173
1174 if (g->ops.clock_gating.slcg_ce2_load_gating_prod)
1175 g->ops.clock_gating.slcg_ce2_load_gating_prod(g,
1176 g->slcg_enabled);
1177 if (g->ops.clock_gating.slcg_fifo_load_gating_prod)
1178 g->ops.clock_gating.slcg_fifo_load_gating_prod(g,
1179 g->slcg_enabled);
1180 if (g->ops.clock_gating.blcg_fifo_load_gating_prod)
1181 g->ops.clock_gating.blcg_fifo_load_gating_prod(g,
1182 g->blcg_enabled);
1183
1184 /* enable pbdma */
1185 mask = 0;
1186 for (i = 0; i < host_num_pbdma; ++i)
1187 mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i);
1188 gk20a_writel(g, mc_enable_pb_r(), mask);
1189
1190
1191 timeout = gk20a_readl(g, fifo_fb_timeout_r());
1192 nvgpu_log_info(g, "fifo_fb_timeout reg val = 0x%08x", timeout);
1193 if (!nvgpu_platform_is_silicon(g)) {
1194 timeout = set_field(timeout, fifo_fb_timeout_period_m(),
1195 fifo_fb_timeout_period_max_f());
1196 timeout = set_field(timeout, fifo_fb_timeout_detection_m(),
1197 fifo_fb_timeout_detection_disabled_f());
1198 nvgpu_log_info(g, "new fifo_fb_timeout reg val = 0x%08x",
1199 timeout);
1200 gk20a_writel(g, fifo_fb_timeout_r(), timeout);
1201 }
1202
1203 for (i = 0; i < host_num_pbdma; i++) {
1204 timeout = gk20a_readl(g, pbdma_timeout_r(i));
1205 nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x",
1206 timeout);
1207 if (!nvgpu_platform_is_silicon(g)) {
1208 timeout = set_field(timeout, pbdma_timeout_period_m(),
1209 pbdma_timeout_period_max_f());
1210 nvgpu_log_info(g, "new pbdma_timeout reg val = 0x%08x",
1211 timeout);
1212 gk20a_writel(g, pbdma_timeout_r(i), timeout);
1213 }
1214 }
1215
1216 /* clear and enable pbdma interrupt */
1217 for (i = 0; i < host_num_pbdma; i++) {
1218 gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF);
1219 gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF);
1220
1221 intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i));
1222 gk20a_dbg_info("pbdma id:%u, intr_en_0 0x%08x", i, intr_stall);
1223 gk20a_writel(g, pbdma_intr_en_0_r(i), intr_stall);
1224
1225 intr_stall = gk20a_readl(g, pbdma_intr_stall_1_r(i));
1226 gk20a_dbg_info("pbdma id:%u, intr_en_1 0x%08x", i, intr_stall);
1227 gk20a_writel(g, pbdma_intr_en_1_r(i), intr_stall);
1228 }
1229
1230 /* clear ctxsw timeout interrupts */
1231 gk20a_writel(g, fifo_intr_ctxsw_timeout_r(), ~0);
1232
1233 if (nvgpu_platform_is_silicon(g)) {
1234 /* enable ctxsw timeout */
1235 timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US;
1236 timeout = scale_ptimer(timeout,
1237 ptimer_scalingfactor10x(g->ptimer_src_freq));
1238 timeout |= fifo_eng_ctxsw_timeout_detection_enabled_f();
1239 gk20a_writel(g, fifo_eng_ctxsw_timeout_r(), timeout);
1240 } else {
1241 timeout = gk20a_readl(g, fifo_eng_ctxsw_timeout_r());
1242 nvgpu_log_info(g, "fifo_eng_ctxsw_timeout reg val = 0x%08x",
1243 timeout);
1244 timeout = set_field(timeout, fifo_eng_ctxsw_timeout_period_m(),
1245 fifo_eng_ctxsw_timeout_period_max_f());
1246 timeout = set_field(timeout,
1247 fifo_eng_ctxsw_timeout_detection_m(),
1248 fifo_eng_ctxsw_timeout_detection_disabled_f());
1249 nvgpu_log_info(g, "new fifo_eng_ctxsw_timeout reg val = 0x%08x",
1250 timeout);
1251 gk20a_writel(g, fifo_eng_ctxsw_timeout_r(), timeout);
1252 }
1253
1254 /* clear runlist interrupts */
1255 gk20a_writel(g, fifo_intr_runlist_r(), ~0);
1256
1257 /* clear and enable pfifo interrupt */
1258 gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF);
1259 mask = gv11b_fifo_intr_0_en_mask(g);
1260 gk20a_dbg_info("fifo_intr_en_0 0x%08x", mask);
1261 gk20a_writel(g, fifo_intr_en_0_r(), mask);
1262 gk20a_dbg_info("fifo_intr_en_1 = 0x80000000");
1263 gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000);
1264
1265 gk20a_dbg_fn("done");
1266
1267 return 0;
1268}
1269
1270static const char *const gv11b_sched_error_str[] = {
1271 "xxx-0",
1272 "xxx-1",
1273 "xxx-2",
1274 "xxx-3",
1275 "xxx-4",
1276 "engine_reset",
1277 "rl_ack_timeout",
1278 "rl_ack_extra",
1279 "rl_rdat_timeout",
1280 "rl_rdat_extra",
1281 "xxx-a",
1282 "xxx-b",
1283 "rl_req_timeout",
1284 "new_runlist",
1285 "code_config_while_busy",
1286 "xxx-f",
1287 "xxx-0x10",
1288 "xxx-0x11",
1289 "xxx-0x12",
1290 "xxx-0x13",
1291 "xxx-0x14",
1292 "xxx-0x15",
1293 "xxx-0x16",
1294 "xxx-0x17",
1295 "xxx-0x18",
1296 "xxx-0x19",
1297 "xxx-0x1a",
1298 "xxx-0x1b",
1299 "xxx-0x1c",
1300 "xxx-0x1d",
1301 "xxx-0x1e",
1302 "xxx-0x1f",
1303 "bad_tsg",
1304};
1305
1306bool gv11b_fifo_handle_sched_error(struct gk20a *g)
1307{
1308 u32 sched_error;
1309
1310 sched_error = gk20a_readl(g, fifo_intr_sched_error_r());
1311
1312 if (sched_error < ARRAY_SIZE(gv11b_sched_error_str))
1313 nvgpu_err(g, "fifo sched error :%s",
1314 gv11b_sched_error_str[sched_error]);
1315 else
1316 nvgpu_err(g, "fifo sched error code not supported");
1317
1318 if (sched_error == SCHED_ERROR_CODE_BAD_TSG ) {
1319 /* id is unknown, preempt all runlists and do recovery */
1320 gk20a_fifo_recover(g, 0, 0, false, false, false);
1321 }
1322
1323 return false;
1324}
1325
1326static u32 gv11b_fifo_ctxsw_timeout_info(struct gk20a *g, u32 active_eng_id)
1327{
1328 u32 tsgid = FIFO_INVAL_TSG_ID;
1329 u32 timeout_info;
1330 u32 ctx_status, info_status;
1331
1332 timeout_info = gk20a_readl(g,
1333 fifo_intr_ctxsw_timeout_info_r(active_eng_id));
1334
1335 /*
1336 * ctxsw_state and tsgid are snapped at the point of the timeout and
1337 * will not change while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit
1338 * is PENDING.
1339 */
1340 ctx_status = fifo_intr_ctxsw_timeout_info_ctxsw_state_v(timeout_info);
1341 if (ctx_status ==
1342 fifo_intr_ctxsw_timeout_info_ctxsw_state_load_v()) {
1343
1344 tsgid = fifo_intr_ctxsw_timeout_info_next_tsgid_v(timeout_info);
1345
1346 } else if (ctx_status ==
1347 fifo_intr_ctxsw_timeout_info_ctxsw_state_switch_v() ||
1348 ctx_status ==
1349 fifo_intr_ctxsw_timeout_info_ctxsw_state_save_v()) {
1350
1351 tsgid = fifo_intr_ctxsw_timeout_info_prev_tsgid_v(timeout_info);
1352 }
1353 gk20a_dbg_info("ctxsw timeout info: tsgid = %d", tsgid);
1354
1355 /*
1356 * STATUS indicates whether the context request ack was eventually
1357 * received and whether a subsequent request timed out. This field is
1358 * updated live while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit
1359 * is PENDING. STATUS starts in AWAITING_ACK, and progresses to
1360 * ACK_RECEIVED and finally ends with DROPPED_TIMEOUT.
1361 *
1362 * AWAITING_ACK - context request ack still not returned from engine.
1363 * ENG_WAS_RESET - The engine was reset via a PRI write to NV_PMC_ENABLE
1364 * or NV_PMC_ELPG_ENABLE prior to receiving the ack. Host will not
1365 * expect ctx ack to return, but if it is already in flight, STATUS will
1366 * transition shortly to ACK_RECEIVED unless the interrupt is cleared
1367 * first. Once the engine is reset, additional context switches can
1368 * occur; if one times out, STATUS will transition to DROPPED_TIMEOUT
1369 * if the interrupt isn't cleared first.
1370 * ACK_RECEIVED - The ack for the timed-out context request was
1371 * received between the point of the timeout and this register being
1372 * read. Note this STATUS can be reported during the load stage of the
1373 * same context switch that timed out if the timeout occurred during the
1374 * save half of a context switch. Additional context requests may have
1375 * completed or may be outstanding, but no further context timeout has
1376 * occurred. This simplifies checking for spurious context switch
1377 * timeouts.
1378 * DROPPED_TIMEOUT - The originally timed-out context request acked,
1379 * but a subsequent context request then timed out.
1380 * Information about the subsequent timeout is not stored; in fact, that
1381 * context request may also have already been acked by the time SW
1382 * SW reads this register. If not, there is a chance SW can get the
1383 * dropped information by clearing the corresponding
1384 * INTR_CTXSW_TIMEOUT_ENGINE bit and waiting for the timeout to occur
1385 * again. Note, however, that if the engine does time out again,
1386 * it may not be from the original request that caused the
1387 * DROPPED_TIMEOUT state, as that request may
1388 * be acked in the interim.
1389 */
1390 info_status = fifo_intr_ctxsw_timeout_info_status_v(timeout_info);
1391 if (info_status ==
1392 fifo_intr_ctxsw_timeout_info_status_awaiting_ack_v()) {
1393
1394 gk20a_dbg_info("ctxsw timeout info : awaiting ack");
1395
1396 } else if (info_status ==
1397 fifo_intr_ctxsw_timeout_info_status_eng_was_reset_v()) {
1398
1399 gk20a_dbg_info("ctxsw timeout info : eng was reset");
1400
1401 } else if (info_status ==
1402 fifo_intr_ctxsw_timeout_info_status_ack_received_v()) {
1403
1404 gk20a_dbg_info("ctxsw timeout info : ack received");
1405 /* no need to recover */
1406 tsgid = FIFO_INVAL_TSG_ID;
1407
1408 } else if (info_status ==
1409 fifo_intr_ctxsw_timeout_info_status_dropped_timeout_v()) {
1410
1411 gk20a_dbg_info("ctxsw timeout info : dropped timeout");
1412 /* no need to recover */
1413 tsgid = FIFO_INVAL_TSG_ID;
1414
1415 } else {
1416 gk20a_dbg_info("ctxsw timeout info status = %u", info_status);
1417 }
1418
1419 return tsgid;
1420}
1421
1422bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr)
1423{
1424 bool ret = false;
1425 u32 tsgid = FIFO_INVAL_TSG_ID;
1426 u32 engine_id, active_eng_id;
1427 u32 timeout_val, ctxsw_timeout_engines;
1428
1429
1430 if (!(fifo_intr & fifo_intr_0_ctxsw_timeout_pending_f()))
1431 return ret;
1432
1433 /* get ctxsw timedout engines */
1434 ctxsw_timeout_engines = gk20a_readl(g, fifo_intr_ctxsw_timeout_r());
1435 if (ctxsw_timeout_engines == 0) {
1436 nvgpu_err(g, "no eng ctxsw timeout pending");
1437 return ret;
1438 }
1439
1440 timeout_val = gk20a_readl(g, fifo_eng_ctxsw_timeout_r());
1441 timeout_val = fifo_eng_ctxsw_timeout_period_v(timeout_val);
1442
1443 gk20a_dbg_info("eng ctxsw timeout period = 0x%x", timeout_val);
1444
1445 for (engine_id = 0; engine_id < g->fifo.num_engines; engine_id++) {
1446 active_eng_id = g->fifo.active_engines_list[engine_id];
1447
1448 if (ctxsw_timeout_engines &
1449 fifo_intr_ctxsw_timeout_engine_pending_f(
1450 active_eng_id)) {
1451
1452 struct fifo_gk20a *f = &g->fifo;
1453 u32 ms = 0;
1454 bool verbose = false;
1455
1456 tsgid = gv11b_fifo_ctxsw_timeout_info(g, active_eng_id);
1457
1458 if (tsgid == FIFO_INVAL_TSG_ID)
1459 continue;
1460
1461 if (gk20a_fifo_check_tsg_ctxsw_timeout(
1462 &f->tsg[tsgid], &verbose, &ms)) {
1463 ret = true;
1464 nvgpu_err(g,
1465 "ctxsw timeout error:"
1466 "active engine id =%u, %s=%d, ms=%u",
1467 active_eng_id, "tsg", tsgid, ms);
1468
1469 /* Cancel all channels' timeout */
1470 gk20a_channel_timeout_restart_all_channels(g);
1471 gk20a_fifo_recover(g, BIT(active_eng_id), tsgid,
1472 true, true, verbose);
1473 } else {
1474 gk20a_dbg_info(
1475 "fifo is waiting for ctx switch: "
1476 "for %d ms, %s=%d", ms, "tsg", tsgid);
1477 }
1478 }
1479 }
1480 /* clear interrupt */
1481 gk20a_writel(g, fifo_intr_ctxsw_timeout_r(), ctxsw_timeout_engines);
1482 return ret;
1483}
1484
1485unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g,
1486 u32 pbdma_id, u32 pbdma_intr_0,
1487 u32 *handled, u32 *error_notifier)
1488{
1489 unsigned int rc_type = RC_TYPE_NO_RC;
1490
1491 rc_type = gk20a_fifo_handle_pbdma_intr_0(g, pbdma_id,
1492 pbdma_intr_0, handled, error_notifier);
1493
1494 if (pbdma_intr_0 & pbdma_intr_0_clear_faulted_error_pending_f()) {
1495 gk20a_dbg(gpu_dbg_intr, "clear faulted error on pbdma id %d",
1496 pbdma_id);
1497 gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
1498 *handled |= pbdma_intr_0_clear_faulted_error_pending_f();
1499 rc_type = RC_TYPE_PBDMA_FAULT;
1500 }
1501
1502 if (pbdma_intr_0 & pbdma_intr_0_eng_reset_pending_f()) {
1503 gk20a_dbg(gpu_dbg_intr, "eng reset intr on pbdma id %d",
1504 pbdma_id);
1505 *handled |= pbdma_intr_0_eng_reset_pending_f();
1506 rc_type = RC_TYPE_PBDMA_FAULT;
1507 }
1508
1509 return rc_type;
1510}
1511
1512/*
1513 * Pbdma which encountered the ctxnotvalid interrupt will stall and
1514 * prevent the channel which was loaded at the time the interrupt fired
1515 * from being swapped out until the interrupt is cleared.
1516 * CTXNOTVALID pbdma interrupt indicates error conditions related
1517 * to the *_CTX_VALID fields for a channel. The following
1518 * conditions trigger the interrupt:
1519 * * CTX_VALID bit for the targeted engine is FALSE
1520 * * At channel start/resume, all preemptible eng have CTX_VALID FALSE but:
1521 * - CTX_RELOAD is set in CCSR_CHANNEL_STATUS,
1522 * - PBDMA_TARGET_SHOULD_SEND_HOST_TSG_EVENT is TRUE, or
1523 * - PBDMA_TARGET_NEEDS_HOST_TSG_EVENT is TRUE
1524 * The field is left NOT_PENDING and the interrupt is not raised if the PBDMA is
1525 * currently halted. This allows SW to unblock the PBDMA and recover.
1526 * SW may read METHOD0, CHANNEL_STATUS and TARGET to determine whether the
1527 * interrupt was due to an engine method, CTX_RELOAD, SHOULD_SEND_HOST_TSG_EVENT
1528 * or NEEDS_HOST_TSG_EVENT. If METHOD0 VALID is TRUE, lazy context creation
1529 * can be used or the TSG may be destroyed.
1530 * If METHOD0 VALID is FALSE, the error is likely a bug in SW, and the TSG
1531 * will have to be destroyed.
1532 */
1533
1534unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g,
1535 u32 pbdma_id, u32 pbdma_intr_1,
1536 u32 *handled, u32 *error_notifier)
1537{
1538 unsigned int rc_type = RC_TYPE_PBDMA_FAULT;
1539 u32 pbdma_intr_1_current = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
1540
1541 /* minimize race with the gpu clearing the pending interrupt */
1542 if (!(pbdma_intr_1_current &
1543 pbdma_intr_1_ctxnotvalid_pending_f()))
1544 pbdma_intr_1 &= ~pbdma_intr_1_ctxnotvalid_pending_f();
1545
1546 if (pbdma_intr_1 == 0)
1547 return RC_TYPE_NO_RC;
1548
1549 if (pbdma_intr_1 & pbdma_intr_1_ctxnotvalid_pending_f()) {
1550 gk20a_dbg(gpu_dbg_intr, "ctxnotvalid intr on pbdma id %d",
1551 pbdma_id);
1552 nvgpu_err(g, "pbdma_intr_1(%d)= 0x%08x ",
1553 pbdma_id, pbdma_intr_1);
1554 *handled |= pbdma_intr_1_ctxnotvalid_pending_f();
1555 } else{
1556 /*
1557 * rest of the interrupts in _intr_1 are "host copy engine"
1558 * related, which is not supported. For now just make them
1559 * channel fatal.
1560 */
1561 nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x",
1562 pbdma_id, pbdma_intr_1);
1563 *handled |= pbdma_intr_1;
1564 }
1565
1566 return rc_type;
1567}
1568
1569static void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g,
1570 struct channel_gk20a *ch, struct nvgpu_mem *mem)
1571{
1572 struct tsg_gk20a *tsg;
1573 struct nvgpu_mem *method_buffer_per_runque;
1574
1575 tsg = tsg_gk20a_from_ch(ch);
1576 if (tsg == NULL) {
1577 nvgpu_err(g, "channel is not part of tsg");
1578 return;
1579 }
1580 if (tsg->eng_method_buffers == NULL) {
1581 nvgpu_log_info(g, "eng method buffer NULL");
1582 return;
1583 }
1584 if (tsg->runlist_id == gk20a_fifo_get_fast_ce_runlist_id(g))
1585 method_buffer_per_runque =
1586 &tsg->eng_method_buffers[ASYNC_CE_RUNQUE];
1587 else
1588 method_buffer_per_runque =
1589 &tsg->eng_method_buffers[GR_RUNQUE];
1590
1591 nvgpu_mem_wr32(g, mem, ram_in_eng_method_buffer_addr_lo_w(),
1592 u64_lo32(method_buffer_per_runque->gpu_va));
1593 nvgpu_mem_wr32(g, mem, ram_in_eng_method_buffer_addr_hi_w(),
1594 u64_hi32(method_buffer_per_runque->gpu_va));
1595
1596 nvgpu_log_info(g, "init ramfc with method buffer");
1597}
1598
1599unsigned int gv11b_fifo_get_eng_method_buffer_size(struct gk20a *g)
1600{
1601 unsigned int buffer_size;
1602
1603 buffer_size = ((9 + 1 + 3) * g->ops.ce2.get_num_pce(g)) + 2;
1604 buffer_size = (27 * 5 * buffer_size);
1605 buffer_size = roundup(buffer_size, PAGE_SIZE);
1606 nvgpu_log_info(g, "method buffer size in bytes %d", buffer_size);
1607
1608 return buffer_size;
1609}
1610
1611void gv11b_fifo_init_eng_method_buffers(struct gk20a *g,
1612 struct tsg_gk20a *tsg)
1613{
1614 struct vm_gk20a *vm = g->mm.bar2.vm;
1615 int err = 0;
1616 int i;
1617 unsigned int runque, method_buffer_size;
1618 unsigned int num_pbdma = g->fifo.num_pbdma;
1619
1620 if (tsg->eng_method_buffers != NULL)
1621 return;
1622
1623 method_buffer_size = gv11b_fifo_get_eng_method_buffer_size(g);
1624 if (method_buffer_size == 0) {
1625 nvgpu_info(g, "ce will hit MTHD_BUFFER_FAULT");
1626 return;
1627 }
1628
1629 tsg->eng_method_buffers = nvgpu_kzalloc(g,
1630 num_pbdma * sizeof(struct nvgpu_mem));
1631
1632 for (runque = 0; runque < num_pbdma; runque++) {
1633 err = nvgpu_dma_alloc_map_sys(vm, method_buffer_size,
1634 &tsg->eng_method_buffers[runque]);
1635 if (err)
1636 break;
1637 }
1638 if (err) {
1639 for (i = (runque - 1); i >= 0; i--)
1640 nvgpu_dma_unmap_free(vm,
1641 &tsg->eng_method_buffers[i]);
1642
1643 nvgpu_kfree(g, tsg->eng_method_buffers);
1644 tsg->eng_method_buffers = NULL;
1645 nvgpu_err(g, "could not alloc eng method buffers");
1646 return;
1647 }
1648 nvgpu_log_info(g, "eng method buffers allocated");
1649
1650}
1651
1652void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g,
1653 struct tsg_gk20a *tsg)
1654{
1655 struct vm_gk20a *vm = g->mm.bar2.vm;
1656 unsigned int runque;
1657
1658 if (tsg->eng_method_buffers == NULL)
1659 return;
1660
1661 for (runque = 0; runque < g->fifo.num_pbdma; runque++)
1662 nvgpu_dma_unmap_free(vm, &tsg->eng_method_buffers[runque]);
1663
1664 nvgpu_kfree(g, tsg->eng_method_buffers);
1665 tsg->eng_method_buffers = NULL;
1666
1667 nvgpu_log_info(g, "eng method buffers de-allocated");
1668}
1669
1670#ifdef CONFIG_TEGRA_GK20A_NVHOST
1671int gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c,
1672 u32 syncpt_id, struct nvgpu_mem *syncpt_buf)
1673{
1674 u32 nr_pages;
1675 int err = 0;
1676 struct gk20a *g = c->g;
1677 struct vm_gk20a *vm = c->vm;
1678
1679 /*
1680 * Add ro map for complete sync point shim range in vm
1681 * All channels sharing same vm will share same ro mapping.
1682 * Create rw map for current channel sync point
1683 */
1684 if (!vm->syncpt_ro_map_gpu_va) {
1685 vm->syncpt_ro_map_gpu_va = nvgpu_gmmu_map(c->vm,
1686 &g->syncpt_mem, g->syncpt_unit_size,
1687 0, gk20a_mem_flag_read_only,
1688 false, APERTURE_SYSMEM);
1689
1690 if (!vm->syncpt_ro_map_gpu_va) {
1691 nvgpu_err(g, "failed to ro map syncpt buffer");
1692 nvgpu_dma_free(g, &g->syncpt_mem);
1693 err = -ENOMEM;
1694 }
1695 }
1696
1697 nr_pages = DIV_ROUND_UP(g->syncpt_size, PAGE_SIZE);
1698 __nvgpu_mem_create_from_phys(g, syncpt_buf,
1699 (g->syncpt_unit_base +
1700 nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(syncpt_id)),
1701 nr_pages);
1702 syncpt_buf->gpu_va = nvgpu_gmmu_map(c->vm, syncpt_buf,
1703 g->syncpt_size, 0, gk20a_mem_flag_none,
1704 false, APERTURE_SYSMEM);
1705
1706 if (!syncpt_buf->gpu_va) {
1707 nvgpu_err(g, "failed to map syncpt buffer");
1708 nvgpu_dma_free(g, syncpt_buf);
1709 err = -ENOMEM;
1710 }
1711 return err;
1712}
1713
1714void gv11b_fifo_free_syncpt_buf(struct channel_gk20a *c,
1715 struct nvgpu_mem *syncpt_buf)
1716{
1717 nvgpu_gmmu_unmap(c->vm, syncpt_buf, syncpt_buf->gpu_va);
1718 nvgpu_dma_free(c->g, syncpt_buf);
1719}
1720
1721void gv11b_fifo_add_syncpt_wait_cmd(struct gk20a *g,
1722 struct priv_cmd_entry *cmd, u32 off,
1723 u32 id, u32 thresh, u64 gpu_va_base)
1724{
1725 u64 gpu_va = gpu_va_base +
1726 nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(id);
1727
1728 gk20a_dbg_fn("");
1729
1730 off = cmd->off + off;
1731
1732 /* semaphore_a */
1733 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010004);
1734 nvgpu_mem_wr32(g, cmd->mem, off++,
1735 (gpu_va >> 32) & 0xff);
1736 /* semaphore_b */
1737 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010005);
1738 /* offset */
1739 nvgpu_mem_wr32(g, cmd->mem, off++, gpu_va & 0xffffffff);
1740
1741 /* semaphore_c */
1742 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010006);
1743 /* payload */
1744 nvgpu_mem_wr32(g, cmd->mem, off++, thresh);
1745 /* semaphore_d */
1746 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010007);
1747 /* operation: acq_geq, switch_en */
1748 nvgpu_mem_wr32(g, cmd->mem, off++, 0x4 | (0x1 << 12));
1749}
1750
1751u32 gv11b_fifo_get_syncpt_wait_cmd_size(void)
1752{
1753 return 8;
1754}
1755
1756void gv11b_fifo_add_syncpt_incr_cmd(struct gk20a *g,
1757 bool wfi_cmd, struct priv_cmd_entry *cmd,
1758 u32 id, u64 gpu_va)
1759{
1760 u32 off = cmd->off;
1761
1762 gk20a_dbg_fn("");
1763
1764 /* semaphore_a */
1765 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010004);
1766 nvgpu_mem_wr32(g, cmd->mem, off++,
1767 (gpu_va >> 32) & 0xff);
1768 /* semaphore_b */
1769 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010005);
1770 /* offset */
1771 nvgpu_mem_wr32(g, cmd->mem, off++, gpu_va & 0xffffffff);
1772
1773 /* semaphore_c */
1774 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010006);
1775 /* payload */
1776 nvgpu_mem_wr32(g, cmd->mem, off++, 0x0);
1777 /* semaphore_d */
1778 nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010007);
1779
1780 /* operation: release, wfi */
1781 nvgpu_mem_wr32(g, cmd->mem, off++,
1782 0x2 | ((wfi_cmd ? 0x0 : 0x1) << 20));
1783 /* ignored */
1784 nvgpu_mem_wr32(g, cmd->mem, off++, 0);
1785}
1786
1787u32 gv11b_fifo_get_syncpt_incr_cmd_size(bool wfi_cmd)
1788{
1789 return 9;
1790}
1791#endif /* CONFIG_TEGRA_GK20A_NVHOST */
1792
1793int gv11b_init_fifo_setup_hw(struct gk20a *g)
1794{
1795 struct fifo_gk20a *f = &g->fifo;
1796
1797 f->t19x.max_subctx_count =
1798 gr_pri_fe_chip_def_info_max_veid_count_init_v();
1799 return 0;
1800}
1801
1802static u32 gv11b_mmu_fault_id_to_gr_veid(struct gk20a *g, u32 gr_eng_fault_id,
1803 u32 mmu_fault_id)
1804{
1805 struct fifo_gk20a *f = &g->fifo;
1806 u32 num_subctx;
1807 u32 veid = FIFO_INVAL_VEID;
1808
1809 num_subctx = f->t19x.max_subctx_count;
1810
1811 if (mmu_fault_id >= gr_eng_fault_id &&
1812 mmu_fault_id < (gr_eng_fault_id + num_subctx))
1813 veid = mmu_fault_id - gr_eng_fault_id;
1814
1815 return veid;
1816}
1817
1818static u32 gv11b_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g,
1819 u32 mmu_fault_id, u32 *veid)
1820{
1821 u32 engine_id;
1822 u32 active_engine_id;
1823 struct fifo_engine_info_gk20a *engine_info;
1824 struct fifo_gk20a *f = &g->fifo;
1825
1826
1827 for (engine_id = 0; engine_id < f->num_engines; engine_id++) {
1828 active_engine_id = f->active_engines_list[engine_id];
1829 engine_info = &g->fifo.engine_info[active_engine_id];
1830
1831 if (active_engine_id == ENGINE_GR_GK20A) {
1832 /* get faulted subctx id */
1833 *veid = gv11b_mmu_fault_id_to_gr_veid(g,
1834 engine_info->fault_id, mmu_fault_id);
1835 if (*veid != FIFO_INVAL_VEID)
1836 break;
1837 } else {
1838 if (engine_info->fault_id == mmu_fault_id)
1839 break;
1840 }
1841
1842 active_engine_id = FIFO_INVAL_ENGINE_ID;
1843 }
1844 return active_engine_id;
1845}
1846
1847static u32 gv11b_mmu_fault_id_to_pbdma_id(struct gk20a *g, u32 mmu_fault_id)
1848{
1849 u32 num_pbdma, reg_val, fault_id_pbdma0;
1850
1851 reg_val = gk20a_readl(g, fifo_cfg0_r());
1852 num_pbdma = fifo_cfg0_num_pbdma_v(reg_val);
1853 fault_id_pbdma0 = fifo_cfg0_pbdma_fault_id_v(reg_val);
1854
1855 if (mmu_fault_id >= fault_id_pbdma0 &&
1856 mmu_fault_id <= fault_id_pbdma0 + num_pbdma - 1)
1857 return mmu_fault_id - fault_id_pbdma0;
1858
1859 return FIFO_INVAL_PBDMA_ID;
1860}
1861
1862void gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(struct gk20a *g,
1863 u32 mmu_fault_id, u32 *active_engine_id, u32 *veid, u32 *pbdma_id)
1864{
1865 *active_engine_id = gv11b_mmu_fault_id_to_eng_id_and_veid(g,
1866 mmu_fault_id, veid);
1867
1868 if (*active_engine_id == FIFO_INVAL_ENGINE_ID)
1869 *pbdma_id = gv11b_mmu_fault_id_to_pbdma_id(g, mmu_fault_id);
1870 else
1871 *pbdma_id = FIFO_INVAL_PBDMA_ID;
1872}
1873
1874static bool gk20a_fifo_channel_status_is_eng_faulted(struct gk20a *g, u32 chid)
1875{
1876 u32 channel = gk20a_readl(g, ccsr_channel_r(chid));
1877
1878 return ccsr_channel_eng_faulted_v(channel) ==
1879 ccsr_channel_eng_faulted_true_v();
1880}
1881
1882void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch)
1883{
1884 struct gk20a *g = ch->g;
1885 struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid];
1886
1887 /*
1888 * If channel has FAULTED set, clear the CE method buffer
1889 * if saved out channel is same as faulted channel
1890 */
1891 if (!gk20a_fifo_channel_status_is_eng_faulted(g, ch->chid))
1892 return;
1893
1894 if (tsg->eng_method_buffers == NULL)
1895 return;
1896
1897 /*
1898 * CE method buffer format :
1899 * DWord0 = method count
1900 * DWord1 = channel id
1901 *
1902 * It is sufficient to write 0 to method count to invalidate
1903 */
1904 if ((u32)ch->chid ==
1905 nvgpu_mem_rd32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 1))
1906 nvgpu_mem_wr32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 0, 0);
1907}
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h
new file mode 100644
index 00000000..fc1ddf83
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h
@@ -0,0 +1,117 @@
1/*
2 * GV11B Fifo
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef FIFO_GV11B_H
26#define FIFO_GV11B_H
27
28#define FIFO_INVAL_PBDMA_ID ((u32)~0)
29#define FIFO_INVAL_VEID ((u32)~0)
30
31/* engine context-switch request occurred while the engine was in reset */
32#define SCHED_ERROR_CODE_ENGINE_RESET 0x00000005
33
34/*
35* ERROR_CODE_BAD_TSG indicates that Host encountered a badly formed TSG header
36* or a badly formed channel type runlist entry in the runlist. This is typically
37* caused by encountering a new TSG entry in the middle of a TSG definition.
38* A channel type entry having wrong runqueue selector can also cause this.
39* Additionally this error code can indicate when a channel is encountered on
40* the runlist which is outside of a TSG.
41*/
42#define SCHED_ERROR_CODE_BAD_TSG 0x00000020
43
44/* can be removed after runque support is added */
45
46#define GR_RUNQUE 0 /* pbdma 0 */
47#define ASYNC_CE_RUNQUE 2 /* pbdma 2 */
48
49#define CHANNEL_INFO_VEID0 0
50
51struct gpu_ops;
52
53void gv11b_fifo_reset_pbdma_and_eng_faulted(struct gk20a *g,
54 struct channel_gk20a *refch,
55 u32 faulted_pbdma, u32 faulted_engine);
56void gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(struct gk20a *g,
57 u32 mmu_fault_id, u32 *active_engine_id, u32 *veid, u32 *pbdma_id);
58
59void gv11b_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist);
60void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist);
61int channel_gv11b_setup_ramfc(struct channel_gk20a *c,
62 u64 gpfifo_base, u32 gpfifo_entries,
63 unsigned long acquire_timeout, u32 flags);
64u32 gv11b_userd_gp_get(struct gk20a *g, struct channel_gk20a *c);
65u64 gv11b_userd_pb_get(struct gk20a *g, struct channel_gk20a *c);
66void gv11b_userd_gp_put(struct gk20a *g, struct channel_gk20a *c);
67void channel_gv11b_unbind(struct channel_gk20a *ch);
68u32 gv11b_fifo_get_num_fifos(struct gk20a *g);
69bool gv11b_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid);
70void gv11b_dump_channel_status_ramfc(struct gk20a *g,
71 struct gk20a_debug_output *o,
72 u32 chid,
73 struct ch_state *ch_state);
74void gv11b_dump_eng_status(struct gk20a *g,
75 struct gk20a_debug_output *o);
76u32 gv11b_fifo_intr_0_error_mask(struct gk20a *g);
77int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id,
78 unsigned int id_type, unsigned int timeout_rc_type);
79int gv11b_fifo_preempt_channel(struct gk20a *g, u32 chid);
80int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid);
81int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg);
82int gv11b_fifo_preempt_ch_tsg(struct gk20a *g, u32 id,
83 unsigned int id_type, unsigned int timeout_rc_type);
84void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask,
85 u32 id, unsigned int id_type, unsigned int rc_type,
86 struct mmu_fault_info *mmfault);
87void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f);
88int gv11b_init_fifo_reset_enable_hw(struct gk20a *g);
89bool gv11b_fifo_handle_sched_error(struct gk20a *g);
90bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr);
91unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g,
92 u32 pbdma_id, u32 pbdma_intr_0,
93 u32 *handled, u32 *error_notifier);
94unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g,
95 u32 pbdma_id, u32 pbdma_intr_1,
96 u32 *handled, u32 *error_notifier);
97void gv11b_fifo_init_eng_method_buffers(struct gk20a *g,
98 struct tsg_gk20a *tsg);
99void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g,
100 struct tsg_gk20a *tsg);
101int gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c,
102 u32 syncpt_id, struct nvgpu_mem *syncpt_buf);
103void gv11b_fifo_free_syncpt_buf(struct channel_gk20a *c,
104 struct nvgpu_mem *syncpt_buf);
105void gv11b_fifo_add_syncpt_wait_cmd(struct gk20a *g,
106 struct priv_cmd_entry *cmd, u32 off,
107 u32 id, u32 thresh, u64 gpu_va_base);
108u32 gv11b_fifo_get_syncpt_wait_cmd_size(void);
109void gv11b_fifo_add_syncpt_incr_cmd(struct gk20a *g,
110 bool wfi_cmd, struct priv_cmd_entry *cmd,
111 u32 id, u64 gpu_va_base);
112u32 gv11b_fifo_get_syncpt_incr_cmd_size(bool wfi_cmd);
113int gv11b_init_fifo_setup_hw(struct gk20a *g);
114
115void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch);
116u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g);
117#endif
diff --git a/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c
new file mode 100644
index 00000000..514aadb1
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c
@@ -0,0 +1,72 @@
1/*
2 *
3 * GV11B Graphics Context
4 *
5 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 * DEALINGS IN THE SOFTWARE.
24 */
25
26#include "gk20a/gk20a.h"
27
28#include "gr_ctx_gv11b.h"
29
30int gr_gv11b_get_netlist_name(struct gk20a *g, int index, char *name)
31{
32 switch (index) {
33#ifdef GV11B_NETLIST_IMAGE_FW_NAME
34 case NETLIST_FINAL:
35 sprintf(name, GV11B_NETLIST_IMAGE_FW_NAME);
36 return 0;
37#endif
38#ifdef GK20A_NETLIST_IMAGE_A
39 case NETLIST_SLOT_A:
40 sprintf(name, GK20A_NETLIST_IMAGE_A);
41 return 0;
42#endif
43#ifdef GK20A_NETLIST_IMAGE_B
44 case NETLIST_SLOT_B:
45 sprintf(name, GK20A_NETLIST_IMAGE_B);
46 return 0;
47#endif
48#ifdef GK20A_NETLIST_IMAGE_C
49 case NETLIST_SLOT_C:
50 sprintf(name, GK20A_NETLIST_IMAGE_C);
51 return 0;
52#endif
53#ifdef GK20A_NETLIST_IMAGE_D
54 case NETLIST_SLOT_D:
55 sprintf(name, GK20A_NETLIST_IMAGE_D);
56 return 0;
57#endif
58 default:
59 return -1;
60 }
61
62 return -1;
63}
64
65bool gr_gv11b_is_firmware_defined(void)
66{
67#ifdef GV11B_NETLIST_IMAGE_FW_NAME
68 return true;
69#else
70 return false;
71#endif
72}
diff --git a/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h
new file mode 100644
index 00000000..0a95ab11
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h
@@ -0,0 +1,36 @@
1/*
2 * GV11B Graphics Context
3 *
4 * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24#ifndef __GR_CTX_GV11B_H__
25#define __GR_CTX_GV11B_H__
26
27#include "gk20a/gr_ctx_gk20a.h"
28
29/* Define netlist for silicon only */
30
31#define GV11B_NETLIST_IMAGE_FW_NAME GK20A_NETLIST_IMAGE_D
32
33int gr_gv11b_get_netlist_name(struct gk20a *g, int index, char *name);
34bool gr_gv11b_is_firmware_defined(void);
35
36#endif /*__GR_CTX_GV11B_H__*/
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
new file mode 100644
index 00000000..3d817d7e
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c
@@ -0,0 +1,3639 @@
1/*
2 * GV11b GPU GR
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/delay.h>
26#include <linux/version.h>
27#include <linux/vmalloc.h>
28#include <linux/tegra_gpu_t19x.h>
29#include <uapi/linux/nvgpu.h>
30
31#include <soc/tegra/fuse.h>
32
33#include <nvgpu/timers.h>
34#include <nvgpu/gmmu.h>
35#include <nvgpu/dma.h>
36#include <nvgpu/log.h>
37#include <nvgpu/debug.h>
38#include <nvgpu/enabled.h>
39
40#include "gk20a/gk20a.h"
41#include "gk20a/gr_gk20a.h"
42#include "gk20a/dbg_gpu_gk20a.h"
43#include "gk20a/regops_gk20a.h"
44#include "gk20a/gr_pri_gk20a.h"
45
46#include "gm20b/gr_gm20b.h"
47
48#include "gp10b/gr_gp10b.h"
49
50#include "gv11b/gr_gv11b.h"
51#include "gv11b/mm_gv11b.h"
52#include "gv11b/subctx_gv11b.h"
53
54#include <nvgpu/hw/gv11b/hw_gr_gv11b.h>
55#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
56#include <nvgpu/hw/gv11b/hw_proj_gv11b.h>
57#include <nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h>
58#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
59#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
60#include <nvgpu/hw/gv11b/hw_pbdma_gv11b.h>
61#include <nvgpu/hw/gv11b/hw_therm_gv11b.h>
62#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
63
64bool gr_gv11b_is_valid_class(struct gk20a *g, u32 class_num)
65{
66 bool valid = false;
67
68 switch (class_num) {
69 case VOLTA_COMPUTE_A:
70 case VOLTA_A:
71 case VOLTA_DMA_COPY_A:
72 valid = true;
73 break;
74
75 case MAXWELL_COMPUTE_B:
76 case MAXWELL_B:
77 case FERMI_TWOD_A:
78 case KEPLER_DMA_COPY_A:
79 case MAXWELL_DMA_COPY_A:
80 case PASCAL_COMPUTE_A:
81 case PASCAL_A:
82 case PASCAL_DMA_COPY_A:
83 valid = true;
84 break;
85
86 default:
87 break;
88 }
89 gk20a_dbg_info("class=0x%x valid=%d", class_num, valid);
90 return valid;
91}
92
93bool gr_gv11b_is_valid_gfx_class(struct gk20a *g, u32 class_num)
94{
95 bool valid = false;
96
97 switch (class_num) {
98 case VOLTA_A:
99 case PASCAL_A:
100 case MAXWELL_B:
101 valid = true;
102 break;
103
104 default:
105 break;
106 }
107 return valid;
108}
109
110bool gr_gv11b_is_valid_compute_class(struct gk20a *g, u32 class_num)
111{
112 bool valid = false;
113
114 switch (class_num) {
115 case VOLTA_COMPUTE_A:
116 case PASCAL_COMPUTE_A:
117 case MAXWELL_COMPUTE_B:
118 valid = true;
119 break;
120
121 default:
122 break;
123 }
124 return valid;
125}
126
127static u32 gv11b_gr_sm_offset(struct gk20a *g, u32 sm)
128{
129
130 u32 sm_pri_stride = nvgpu_get_litter_value(g, GPU_LIT_SM_PRI_STRIDE);
131 u32 sm_offset = sm_pri_stride * sm;
132
133 return sm_offset;
134}
135
136static int gr_gv11b_handle_l1_tag_exception(struct gk20a *g, u32 gpc, u32 tpc,
137 bool *post_event, struct channel_gk20a *fault_ch,
138 u32 *hww_global_esr)
139{
140 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
141 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE);
142 u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc;
143 u32 l1_tag_ecc_status, l1_tag_ecc_corrected_err_status = 0;
144 u32 l1_tag_ecc_uncorrected_err_status = 0;
145 u32 l1_tag_corrected_err_count_delta = 0;
146 u32 l1_tag_uncorrected_err_count_delta = 0;
147 bool is_l1_tag_ecc_corrected_total_err_overflow = 0;
148 bool is_l1_tag_ecc_uncorrected_total_err_overflow = 0;
149
150 /* Check for L1 tag ECC errors. */
151 l1_tag_ecc_status = gk20a_readl(g,
152 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_r() + offset);
153 l1_tag_ecc_corrected_err_status = l1_tag_ecc_status &
154 (gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_el1_0_m() |
155 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_el1_1_m() |
156 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_pixrpf_m() |
157 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_miss_fifo_m());
158 l1_tag_ecc_uncorrected_err_status = l1_tag_ecc_status &
159 (gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_el1_0_m() |
160 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_el1_1_m() |
161 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_pixrpf_m() |
162 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_miss_fifo_m());
163
164 if ((l1_tag_ecc_corrected_err_status == 0) && (l1_tag_ecc_uncorrected_err_status == 0))
165 return 0;
166
167 l1_tag_corrected_err_count_delta =
168 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_v(
169 gk20a_readl(g,
170 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_r() +
171 offset));
172 l1_tag_uncorrected_err_count_delta =
173 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_v(
174 gk20a_readl(g,
175 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_r() +
176 offset));
177 is_l1_tag_ecc_corrected_total_err_overflow =
178 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_total_counter_overflow_v(l1_tag_ecc_status);
179 is_l1_tag_ecc_uncorrected_total_err_overflow =
180 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_total_counter_overflow_v(l1_tag_ecc_status);
181
182 if ((l1_tag_corrected_err_count_delta > 0) || is_l1_tag_ecc_corrected_total_err_overflow) {
183 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
184 "corrected error (SBE) detected in SM L1 tag! err_mask [%08x] is_overf [%d]",
185 l1_tag_ecc_corrected_err_status, is_l1_tag_ecc_corrected_total_err_overflow);
186
187 /* HW uses 16-bits counter */
188 l1_tag_corrected_err_count_delta +=
189 (is_l1_tag_ecc_corrected_total_err_overflow <<
190 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_s());
191 g->ecc.gr.t19x.sm_l1_tag_corrected_err_count.counters[tpc] +=
192 l1_tag_corrected_err_count_delta;
193 gk20a_writel(g,
194 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_r() + offset,
195 0);
196 }
197 if ((l1_tag_uncorrected_err_count_delta > 0) || is_l1_tag_ecc_uncorrected_total_err_overflow) {
198 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
199 "Uncorrected error (DBE) detected in SM L1 tag! err_mask [%08x] is_overf [%d]",
200 l1_tag_ecc_uncorrected_err_status, is_l1_tag_ecc_uncorrected_total_err_overflow);
201
202 /* HW uses 16-bits counter */
203 l1_tag_uncorrected_err_count_delta +=
204 (is_l1_tag_ecc_uncorrected_total_err_overflow <<
205 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_s());
206 g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count.counters[tpc] +=
207 l1_tag_uncorrected_err_count_delta;
208 gk20a_writel(g,
209 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_r() + offset,
210 0);
211 }
212
213 gk20a_writel(g, gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_r() + offset,
214 gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_reset_task_f());
215
216 return 0;
217
218}
219
220static int gr_gv11b_handle_lrf_exception(struct gk20a *g, u32 gpc, u32 tpc,
221 bool *post_event, struct channel_gk20a *fault_ch,
222 u32 *hww_global_esr)
223{
224 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
225 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE);
226 u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc;
227 u32 lrf_ecc_status, lrf_ecc_corrected_err_status = 0;
228 u32 lrf_ecc_uncorrected_err_status = 0;
229 u32 lrf_corrected_err_count_delta = 0;
230 u32 lrf_uncorrected_err_count_delta = 0;
231 bool is_lrf_ecc_corrected_total_err_overflow = 0;
232 bool is_lrf_ecc_uncorrected_total_err_overflow = 0;
233
234 /* Check for LRF ECC errors. */
235 lrf_ecc_status = gk20a_readl(g,
236 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_r() + offset);
237 lrf_ecc_corrected_err_status = lrf_ecc_status &
238 (gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp0_m() |
239 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp1_m() |
240 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp2_m() |
241 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp3_m() |
242 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp4_m() |
243 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp5_m() |
244 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp6_m() |
245 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp7_m());
246 lrf_ecc_uncorrected_err_status = lrf_ecc_status &
247 (gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp0_m() |
248 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp1_m() |
249 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp2_m() |
250 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp3_m() |
251 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp4_m() |
252 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp5_m() |
253 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp6_m() |
254 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp7_m());
255
256 if ((lrf_ecc_corrected_err_status == 0) && (lrf_ecc_uncorrected_err_status == 0))
257 return 0;
258
259 lrf_corrected_err_count_delta =
260 gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_v(
261 gk20a_readl(g,
262 gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_r() +
263 offset));
264 lrf_uncorrected_err_count_delta =
265 gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_v(
266 gk20a_readl(g,
267 gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_r() +
268 offset));
269 is_lrf_ecc_corrected_total_err_overflow =
270 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_total_counter_overflow_v(lrf_ecc_status);
271 is_lrf_ecc_uncorrected_total_err_overflow =
272 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_total_counter_overflow_v(lrf_ecc_status);
273
274 if ((lrf_corrected_err_count_delta > 0) || is_lrf_ecc_corrected_total_err_overflow) {
275 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
276 "corrected error (SBE) detected in SM LRF! err_mask [%08x] is_overf [%d]",
277 lrf_ecc_corrected_err_status, is_lrf_ecc_corrected_total_err_overflow);
278
279 /* HW uses 16-bits counter */
280 lrf_corrected_err_count_delta +=
281 (is_lrf_ecc_corrected_total_err_overflow <<
282 gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_s());
283 g->ecc.gr.t18x.sm_lrf_single_err_count.counters[tpc] +=
284 lrf_corrected_err_count_delta;
285 gk20a_writel(g,
286 gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_r() + offset,
287 0);
288 }
289 if ((lrf_uncorrected_err_count_delta > 0) || is_lrf_ecc_uncorrected_total_err_overflow) {
290 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
291 "Uncorrected error (DBE) detected in SM LRF! err_mask [%08x] is_overf [%d]",
292 lrf_ecc_uncorrected_err_status, is_lrf_ecc_uncorrected_total_err_overflow);
293
294 /* HW uses 16-bits counter */
295 lrf_uncorrected_err_count_delta +=
296 (is_lrf_ecc_uncorrected_total_err_overflow <<
297 gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_s());
298 g->ecc.gr.t18x.sm_lrf_double_err_count.counters[tpc] +=
299 lrf_uncorrected_err_count_delta;
300 gk20a_writel(g,
301 gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_r() + offset,
302 0);
303 }
304
305 gk20a_writel(g, gr_pri_gpc0_tpc0_sm_lrf_ecc_status_r() + offset,
306 gr_pri_gpc0_tpc0_sm_lrf_ecc_status_reset_task_f());
307
308 return 0;
309
310}
311
312void gr_gv11b_enable_hww_exceptions(struct gk20a *g)
313{
314 /* enable exceptions */
315 gk20a_writel(g, gr_fe_hww_esr_r(),
316 gr_fe_hww_esr_en_enable_f() |
317 gr_fe_hww_esr_reset_active_f());
318 gk20a_writel(g, gr_memfmt_hww_esr_r(),
319 gr_memfmt_hww_esr_en_enable_f() |
320 gr_memfmt_hww_esr_reset_active_f());
321}
322
323void gr_gv11b_enable_exceptions(struct gk20a *g)
324{
325 struct gr_gk20a *gr = &g->gr;
326 u32 reg_val;
327
328 /*
329 * clear exceptions :
330 * other than SM : hww_esr are reset in *enable_hww_excetpions*
331 * SM : cleared in *set_hww_esr_report_mask*
332 */
333
334 /* enable exceptions */
335 gk20a_writel(g, gr_exception2_en_r(), 0x0); /* BE not enabled */
336 gk20a_writel(g, gr_exception1_en_r(), (1 << gr->gpc_count) - 1);
337
338 reg_val = gr_exception_en_fe_enabled_f() |
339 gr_exception_en_memfmt_enabled_f() |
340 gr_exception_en_ds_enabled_f() |
341 gr_exception_en_gpc_enabled_f();
342 gk20a_writel(g, gr_exception_en_r(), reg_val);
343
344}
345
346static int gr_gv11b_handle_cbu_exception(struct gk20a *g, u32 gpc, u32 tpc,
347 bool *post_event, struct channel_gk20a *fault_ch,
348 u32 *hww_global_esr)
349{
350 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
351 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE);
352 u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc;
353 u32 cbu_ecc_status, cbu_ecc_corrected_err_status = 0;
354 u32 cbu_ecc_uncorrected_err_status = 0;
355 u32 cbu_corrected_err_count_delta = 0;
356 u32 cbu_uncorrected_err_count_delta = 0;
357 bool is_cbu_ecc_corrected_total_err_overflow = 0;
358 bool is_cbu_ecc_uncorrected_total_err_overflow = 0;
359
360 /* Check for CBU ECC errors. */
361 cbu_ecc_status = gk20a_readl(g,
362 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_r() + offset);
363 cbu_ecc_corrected_err_status = cbu_ecc_status &
364 (gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_warp_sm0_m() |
365 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_warp_sm1_m() |
366 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_barrier_sm0_m() |
367 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_barrier_sm1_m());
368 cbu_ecc_uncorrected_err_status = cbu_ecc_status &
369 (gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_warp_sm0_m() |
370 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_warp_sm1_m() |
371 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_barrier_sm0_m() |
372 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_barrier_sm1_m());
373
374 if ((cbu_ecc_corrected_err_status == 0) && (cbu_ecc_uncorrected_err_status == 0))
375 return 0;
376
377 cbu_corrected_err_count_delta =
378 gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_v(
379 gk20a_readl(g,
380 gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_r() +
381 offset));
382 cbu_uncorrected_err_count_delta =
383 gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_v(
384 gk20a_readl(g,
385 gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_r() +
386 offset));
387 is_cbu_ecc_corrected_total_err_overflow =
388 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_total_counter_overflow_v(cbu_ecc_status);
389 is_cbu_ecc_uncorrected_total_err_overflow =
390 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_total_counter_overflow_v(cbu_ecc_status);
391
392 if ((cbu_corrected_err_count_delta > 0) || is_cbu_ecc_corrected_total_err_overflow) {
393 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
394 "corrected error (SBE) detected in SM CBU! err_mask [%08x] is_overf [%d]",
395 cbu_ecc_corrected_err_status, is_cbu_ecc_corrected_total_err_overflow);
396
397 /* HW uses 16-bits counter */
398 cbu_corrected_err_count_delta +=
399 (is_cbu_ecc_corrected_total_err_overflow <<
400 gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_s());
401 g->ecc.gr.t19x.sm_cbu_corrected_err_count.counters[tpc] +=
402 cbu_corrected_err_count_delta;
403 gk20a_writel(g,
404 gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_r() + offset,
405 0);
406 }
407 if ((cbu_uncorrected_err_count_delta > 0) || is_cbu_ecc_uncorrected_total_err_overflow) {
408 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
409 "Uncorrected error (DBE) detected in SM CBU! err_mask [%08x] is_overf [%d]",
410 cbu_ecc_uncorrected_err_status, is_cbu_ecc_uncorrected_total_err_overflow);
411
412 /* HW uses 16-bits counter */
413 cbu_uncorrected_err_count_delta +=
414 (is_cbu_ecc_uncorrected_total_err_overflow <<
415 gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_s());
416 g->ecc.gr.t19x.sm_cbu_uncorrected_err_count.counters[tpc] +=
417 cbu_uncorrected_err_count_delta;
418 gk20a_writel(g,
419 gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_r() + offset,
420 0);
421 }
422
423 gk20a_writel(g, gr_pri_gpc0_tpc0_sm_cbu_ecc_status_r() + offset,
424 gr_pri_gpc0_tpc0_sm_cbu_ecc_status_reset_task_f());
425
426 return 0;
427
428}
429
430static int gr_gv11b_handle_l1_data_exception(struct gk20a *g, u32 gpc, u32 tpc,
431 bool *post_event, struct channel_gk20a *fault_ch,
432 u32 *hww_global_esr)
433{
434 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
435 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE);
436 u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc;
437 u32 l1_data_ecc_status, l1_data_ecc_corrected_err_status = 0;
438 u32 l1_data_ecc_uncorrected_err_status = 0;
439 u32 l1_data_corrected_err_count_delta = 0;
440 u32 l1_data_uncorrected_err_count_delta = 0;
441 bool is_l1_data_ecc_corrected_total_err_overflow = 0;
442 bool is_l1_data_ecc_uncorrected_total_err_overflow = 0;
443
444 /* Check for L1 data ECC errors. */
445 l1_data_ecc_status = gk20a_readl(g,
446 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_r() + offset);
447 l1_data_ecc_corrected_err_status = l1_data_ecc_status &
448 (gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_el1_0_m() |
449 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_el1_1_m());
450 l1_data_ecc_uncorrected_err_status = l1_data_ecc_status &
451 (gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_el1_0_m() |
452 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_el1_1_m());
453
454 if ((l1_data_ecc_corrected_err_status == 0) && (l1_data_ecc_uncorrected_err_status == 0))
455 return 0;
456
457 l1_data_corrected_err_count_delta =
458 gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_v(
459 gk20a_readl(g,
460 gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_r() +
461 offset));
462 l1_data_uncorrected_err_count_delta =
463 gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_v(
464 gk20a_readl(g,
465 gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_r() +
466 offset));
467 is_l1_data_ecc_corrected_total_err_overflow =
468 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_total_counter_overflow_v(l1_data_ecc_status);
469 is_l1_data_ecc_uncorrected_total_err_overflow =
470 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_total_counter_overflow_v(l1_data_ecc_status);
471
472 if ((l1_data_corrected_err_count_delta > 0) || is_l1_data_ecc_corrected_total_err_overflow) {
473 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
474 "corrected error (SBE) detected in SM L1 data! err_mask [%08x] is_overf [%d]",
475 l1_data_ecc_corrected_err_status, is_l1_data_ecc_corrected_total_err_overflow);
476
477 /* HW uses 16-bits counter */
478 l1_data_corrected_err_count_delta +=
479 (is_l1_data_ecc_corrected_total_err_overflow <<
480 gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_s());
481 g->ecc.gr.t19x.sm_l1_data_corrected_err_count.counters[tpc] +=
482 l1_data_corrected_err_count_delta;
483 gk20a_writel(g,
484 gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_r() + offset,
485 0);
486 }
487 if ((l1_data_uncorrected_err_count_delta > 0) || is_l1_data_ecc_uncorrected_total_err_overflow) {
488 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
489 "Uncorrected error (DBE) detected in SM L1 data! err_mask [%08x] is_overf [%d]",
490 l1_data_ecc_uncorrected_err_status, is_l1_data_ecc_uncorrected_total_err_overflow);
491
492 /* HW uses 16-bits counter */
493 l1_data_uncorrected_err_count_delta +=
494 (is_l1_data_ecc_uncorrected_total_err_overflow <<
495 gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_s());
496 g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count.counters[tpc] +=
497 l1_data_uncorrected_err_count_delta;
498 gk20a_writel(g,
499 gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_r() + offset,
500 0);
501 }
502
503 gk20a_writel(g, gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_r() + offset,
504 gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_reset_task_f());
505
506 return 0;
507
508}
509
510static int gr_gv11b_handle_icache_exception(struct gk20a *g, u32 gpc, u32 tpc,
511 bool *post_event, struct channel_gk20a *fault_ch,
512 u32 *hww_global_esr)
513{
514 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
515 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE);
516 u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc;
517 u32 icache_ecc_status, icache_ecc_corrected_err_status = 0;
518 u32 icache_ecc_uncorrected_err_status = 0;
519 u32 icache_corrected_err_count_delta = 0;
520 u32 icache_uncorrected_err_count_delta = 0;
521 bool is_icache_ecc_corrected_total_err_overflow = 0;
522 bool is_icache_ecc_uncorrected_total_err_overflow = 0;
523
524 /* Check for L0 && L1 icache ECC errors. */
525 icache_ecc_status = gk20a_readl(g,
526 gr_pri_gpc0_tpc0_sm_icache_ecc_status_r() + offset);
527 icache_ecc_corrected_err_status = icache_ecc_status &
528 (gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l0_data_m() |
529 gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l0_predecode_m() |
530 gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l1_data_m() |
531 gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l1_predecode_m());
532 icache_ecc_uncorrected_err_status = icache_ecc_status &
533 (gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l0_data_m() |
534 gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l0_predecode_m() |
535 gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l1_data_m() |
536 gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l1_predecode_m());
537
538 if ((icache_ecc_corrected_err_status == 0) && (icache_ecc_uncorrected_err_status == 0))
539 return 0;
540
541 icache_corrected_err_count_delta =
542 gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_v(
543 gk20a_readl(g,
544 gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_r() +
545 offset));
546 icache_uncorrected_err_count_delta =
547 gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_v(
548 gk20a_readl(g,
549 gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_r() +
550 offset));
551 is_icache_ecc_corrected_total_err_overflow =
552 gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_total_counter_overflow_v(icache_ecc_status);
553 is_icache_ecc_uncorrected_total_err_overflow =
554 gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_total_counter_overflow_v(icache_ecc_status);
555
556 if ((icache_corrected_err_count_delta > 0) || is_icache_ecc_corrected_total_err_overflow) {
557 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
558 "corrected error (SBE) detected in SM L0 && L1 icache! err_mask [%08x] is_overf [%d]",
559 icache_ecc_corrected_err_status, is_icache_ecc_corrected_total_err_overflow);
560
561 /* HW uses 16-bits counter */
562 icache_corrected_err_count_delta +=
563 (is_icache_ecc_corrected_total_err_overflow <<
564 gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_s());
565 g->ecc.gr.t19x.sm_icache_corrected_err_count.counters[tpc] +=
566 icache_corrected_err_count_delta;
567 gk20a_writel(g,
568 gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_r() + offset,
569 0);
570 }
571 if ((icache_uncorrected_err_count_delta > 0) || is_icache_ecc_uncorrected_total_err_overflow) {
572 gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr,
573 "Uncorrected error (DBE) detected in SM L0 && L1 icache! err_mask [%08x] is_overf [%d]",
574 icache_ecc_uncorrected_err_status, is_icache_ecc_uncorrected_total_err_overflow);
575
576 /* HW uses 16-bits counter */
577 icache_uncorrected_err_count_delta +=
578 (is_icache_ecc_uncorrected_total_err_overflow <<
579 gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_s());
580 g->ecc.gr.t19x.sm_icache_uncorrected_err_count.counters[tpc] +=
581 icache_uncorrected_err_count_delta;
582 gk20a_writel(g,
583 gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_r() + offset,
584 0);
585 }
586
587 gk20a_writel(g, gr_pri_gpc0_tpc0_sm_icache_ecc_status_r() + offset,
588 gr_pri_gpc0_tpc0_sm_icache_ecc_status_reset_task_f());
589
590 return 0;
591
592}
593
594int gr_gv11b_handle_tpc_sm_ecc_exception(struct gk20a *g,
595 u32 gpc, u32 tpc,
596 bool *post_event, struct channel_gk20a *fault_ch,
597 u32 *hww_global_esr)
598{
599 int ret = 0;
600
601 /* Check for L1 tag ECC errors. */
602 gr_gv11b_handle_l1_tag_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr);
603
604 /* Check for LRF ECC errors. */
605 gr_gv11b_handle_lrf_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr);
606
607 /* Check for CBU ECC errors. */
608 gr_gv11b_handle_cbu_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr);
609
610 /* Check for L1 data ECC errors. */
611 gr_gv11b_handle_l1_data_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr);
612
613 /* Check for L0 && L1 icache ECC errors. */
614 gr_gv11b_handle_icache_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr);
615
616 return ret;
617}
618
619int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc,
620 bool *post_event, struct channel_gk20a *fault_ch,
621 u32 *hww_global_esr)
622{
623 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
624 u32 offset = gpc_stride * gpc;
625 u32 gcc_l15_ecc_status, gcc_l15_ecc_corrected_err_status = 0;
626 u32 gcc_l15_ecc_uncorrected_err_status = 0;
627 u32 gcc_l15_corrected_err_count_delta = 0;
628 u32 gcc_l15_uncorrected_err_count_delta = 0;
629 bool is_gcc_l15_ecc_corrected_total_err_overflow = 0;
630 bool is_gcc_l15_ecc_uncorrected_total_err_overflow = 0;
631
632 /* Check for gcc l15 ECC errors. */
633 gcc_l15_ecc_status = gk20a_readl(g,
634 gr_pri_gpc0_gcc_l15_ecc_status_r() + offset);
635 gcc_l15_ecc_corrected_err_status = gcc_l15_ecc_status &
636 (gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_bank0_m() |
637 gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_bank1_m());
638 gcc_l15_ecc_uncorrected_err_status = gcc_l15_ecc_status &
639 (gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_bank0_m() |
640 gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_bank1_m());
641
642 if ((gcc_l15_ecc_corrected_err_status == 0) && (gcc_l15_ecc_uncorrected_err_status == 0))
643 return 0;
644
645 gcc_l15_corrected_err_count_delta =
646 gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_v(
647 gk20a_readl(g,
648 gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_r() +
649 offset));
650 gcc_l15_uncorrected_err_count_delta =
651 gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_v(
652 gk20a_readl(g,
653 gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_r() +
654 offset));
655 is_gcc_l15_ecc_corrected_total_err_overflow =
656 gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_total_counter_overflow_v(gcc_l15_ecc_status);
657 is_gcc_l15_ecc_uncorrected_total_err_overflow =
658 gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_total_counter_overflow_v(gcc_l15_ecc_status);
659
660 if ((gcc_l15_corrected_err_count_delta > 0) || is_gcc_l15_ecc_corrected_total_err_overflow) {
661 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr,
662 "corrected error (SBE) detected in GCC L1.5! err_mask [%08x] is_overf [%d]",
663 gcc_l15_ecc_corrected_err_status, is_gcc_l15_ecc_corrected_total_err_overflow);
664
665 /* HW uses 16-bits counter */
666 gcc_l15_corrected_err_count_delta +=
667 (is_gcc_l15_ecc_corrected_total_err_overflow <<
668 gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_s());
669 g->ecc.gr.t19x.gcc_l15_corrected_err_count.counters[gpc] +=
670 gcc_l15_corrected_err_count_delta;
671 gk20a_writel(g,
672 gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_r() + offset,
673 0);
674 }
675 if ((gcc_l15_uncorrected_err_count_delta > 0) || is_gcc_l15_ecc_uncorrected_total_err_overflow) {
676 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr,
677 "Uncorrected error (DBE) detected in GCC L1.5! err_mask [%08x] is_overf [%d]",
678 gcc_l15_ecc_uncorrected_err_status, is_gcc_l15_ecc_uncorrected_total_err_overflow);
679
680 /* HW uses 16-bits counter */
681 gcc_l15_uncorrected_err_count_delta +=
682 (is_gcc_l15_ecc_uncorrected_total_err_overflow <<
683 gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_s());
684 g->ecc.gr.t19x.gcc_l15_uncorrected_err_count.counters[gpc] +=
685 gcc_l15_uncorrected_err_count_delta;
686 gk20a_writel(g,
687 gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_r() + offset,
688 0);
689 }
690
691 gk20a_writel(g, gr_pri_gpc0_gcc_l15_ecc_status_r() + offset,
692 gr_pri_gpc0_gcc_l15_ecc_status_reset_task_f());
693
694 return 0;
695}
696
697static int gr_gv11b_handle_gpcmmu_ecc_exception(struct gk20a *g, u32 gpc,
698 u32 exception)
699{
700 int ret = 0;
701 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
702 u32 offset = gpc_stride * gpc;
703 u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt;
704 u32 corrected_delta, uncorrected_delta;
705 u32 corrected_overflow, uncorrected_overflow;
706 int hww_esr;
707
708 hww_esr = gk20a_readl(g, gr_gpc0_mmu_gpcmmu_global_esr_r() + offset);
709
710 if (!(hww_esr & (gr_gpc0_mmu_gpcmmu_global_esr_ecc_corrected_m() |
711 gr_gpc0_mmu_gpcmmu_global_esr_ecc_uncorrected_m())))
712 return ret;
713
714 ecc_status = gk20a_readl(g,
715 gr_gpc0_mmu_l1tlb_ecc_status_r() + offset);
716 ecc_addr = gk20a_readl(g,
717 gr_gpc0_mmu_l1tlb_ecc_address_r() + offset);
718 corrected_cnt = gk20a_readl(g,
719 gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_r() + offset);
720 uncorrected_cnt = gk20a_readl(g,
721 gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_r() + offset);
722
723 corrected_delta = gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_total_v(
724 corrected_cnt);
725 uncorrected_delta = gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_total_v(
726 uncorrected_cnt);
727 corrected_overflow = ecc_status &
728 gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_total_counter_overflow_m();
729
730 uncorrected_overflow = ecc_status &
731 gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_total_counter_overflow_m();
732
733
734 /* clear the interrupt */
735 if ((corrected_delta > 0) || corrected_overflow)
736 gk20a_writel(g,
737 gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_r() +
738 offset, 0);
739 if ((uncorrected_delta > 0) || uncorrected_overflow)
740 gk20a_writel(g,
741 gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_r() +
742 offset, 0);
743
744 gk20a_writel(g, gr_gpc0_mmu_l1tlb_ecc_status_r() + offset,
745 gr_gpc0_mmu_l1tlb_ecc_status_reset_task_f());
746
747 /* Handle overflow */
748 if (corrected_overflow)
749 corrected_delta += (0x1UL << gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_total_s());
750 if (uncorrected_overflow)
751 uncorrected_delta += (0x1UL << gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_total_s());
752
753
754 g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count.counters[gpc] +=
755 corrected_delta;
756 g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count.counters[gpc] +=
757 uncorrected_delta;
758 nvgpu_log(g, gpu_dbg_intr,
759 "mmu l1tlb gpc:%d ecc interrupt intr: 0x%x", gpc, hww_esr);
760
761 if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_l1tlb_sa_data_m())
762 nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
763 if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_l1tlb_sa_data_m())
764 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
765 if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_l1tlb_fa_data_m())
766 nvgpu_log(g, gpu_dbg_intr, "corrected ecc fa data error");
767 if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_l1tlb_fa_data_m())
768 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc fa data error");
769 if (corrected_overflow || uncorrected_overflow)
770 nvgpu_info(g, "mmu l1tlb ecc counter overflow!");
771
772 nvgpu_log(g, gpu_dbg_intr,
773 "ecc error address: 0x%x", ecc_addr);
774 nvgpu_log(g, gpu_dbg_intr,
775 "ecc error count corrected: %d, uncorrected %d",
776 g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count.counters[gpc],
777 g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count.counters[gpc]);
778
779 return ret;
780}
781
782static int gr_gv11b_handle_gpccs_ecc_exception(struct gk20a *g, u32 gpc,
783 u32 exception)
784{
785 int ret = 0;
786 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
787 u32 offset = gpc_stride * gpc;
788 u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt;
789 u32 corrected_delta, uncorrected_delta;
790 u32 corrected_overflow, uncorrected_overflow;
791 int hww_esr;
792
793 hww_esr = gk20a_readl(g, gr_gpc0_gpccs_hww_esr_r() + offset);
794
795 if (!(hww_esr & (gr_gpc0_gpccs_hww_esr_ecc_uncorrected_m() |
796 gr_gpc0_gpccs_hww_esr_ecc_corrected_m())))
797 return ret;
798
799 ecc_status = gk20a_readl(g,
800 gr_gpc0_gpccs_falcon_ecc_status_r() + offset);
801 ecc_addr = gk20a_readl(g,
802 gr_gpc0_gpccs_falcon_ecc_address_r() + offset);
803 corrected_cnt = gk20a_readl(g,
804 gr_gpc0_gpccs_falcon_ecc_corrected_err_count_r() + offset);
805 uncorrected_cnt = gk20a_readl(g,
806 gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_r() + offset);
807
808 corrected_delta = gr_gpc0_gpccs_falcon_ecc_corrected_err_count_total_v(
809 corrected_cnt);
810 uncorrected_delta = gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_total_v(
811 uncorrected_cnt);
812 corrected_overflow = ecc_status &
813 gr_gpc0_gpccs_falcon_ecc_status_corrected_err_total_counter_overflow_m();
814
815 uncorrected_overflow = ecc_status &
816 gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_total_counter_overflow_m();
817
818
819 /* clear the interrupt */
820 if ((corrected_delta > 0) || corrected_overflow)
821 gk20a_writel(g,
822 gr_gpc0_gpccs_falcon_ecc_corrected_err_count_r() +
823 offset, 0);
824 if ((uncorrected_delta > 0) || uncorrected_overflow)
825 gk20a_writel(g,
826 gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_r() +
827 offset, 0);
828
829 gk20a_writel(g, gr_gpc0_gpccs_falcon_ecc_status_r() + offset,
830 gr_gpc0_gpccs_falcon_ecc_status_reset_task_f());
831
832 g->ecc.gr.t19x.gpccs_corrected_err_count.counters[gpc] +=
833 corrected_delta;
834 g->ecc.gr.t19x.gpccs_uncorrected_err_count.counters[gpc] +=
835 uncorrected_delta;
836 nvgpu_log(g, gpu_dbg_intr,
837 "gppcs gpc:%d ecc interrupt intr: 0x%x", gpc, hww_esr);
838
839 if (ecc_status & gr_gpc0_gpccs_falcon_ecc_status_corrected_err_imem_m())
840 nvgpu_log(g, gpu_dbg_intr, "imem ecc error corrected");
841 if (ecc_status &
842 gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_imem_m())
843 nvgpu_log(g, gpu_dbg_intr, "imem ecc error uncorrected");
844 if (ecc_status &
845 gr_gpc0_gpccs_falcon_ecc_status_corrected_err_dmem_m())
846 nvgpu_log(g, gpu_dbg_intr, "dmem ecc error corrected");
847 if (ecc_status &
848 gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_dmem_m())
849 nvgpu_log(g, gpu_dbg_intr, "dmem ecc error uncorrected");
850 if (corrected_overflow || uncorrected_overflow)
851 nvgpu_info(g, "gpccs ecc counter overflow!");
852
853 nvgpu_log(g, gpu_dbg_intr,
854 "ecc error row address: 0x%x",
855 gr_gpc0_gpccs_falcon_ecc_address_row_address_v(ecc_addr));
856
857 nvgpu_log(g, gpu_dbg_intr,
858 "ecc error count corrected: %d, uncorrected %d",
859 g->ecc.gr.t19x.gpccs_corrected_err_count.counters[gpc],
860 g->ecc.gr.t19x.gpccs_uncorrected_err_count.counters[gpc]);
861
862 return ret;
863}
864
865int gr_gv11b_handle_gpc_gpcmmu_exception(struct gk20a *g, u32 gpc,
866 u32 gpc_exception)
867{
868 if (gpc_exception & gr_gpc0_gpccs_gpc_exception_gpcmmu_m())
869 return gr_gv11b_handle_gpcmmu_ecc_exception(g, gpc,
870 gpc_exception);
871 return 0;
872}
873
874int gr_gv11b_handle_gpc_gpccs_exception(struct gk20a *g, u32 gpc,
875 u32 gpc_exception)
876{
877 if (gpc_exception & gr_gpc0_gpccs_gpc_exception_gpccs_m())
878 return gr_gv11b_handle_gpccs_ecc_exception(g, gpc,
879 gpc_exception);
880
881 return 0;
882}
883
884void gr_gv11b_enable_gpc_exceptions(struct gk20a *g)
885{
886 struct gr_gk20a *gr = &g->gr;
887 u32 tpc_mask;
888
889 gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(),
890 gr_gpcs_tpcs_tpccs_tpc_exception_en_sm_enabled_f() |
891 gr_gpcs_tpcs_tpccs_tpc_exception_en_mpc_enabled_f());
892
893 tpc_mask =
894 gr_gpcs_gpccs_gpc_exception_en_tpc_f((1 << gr->tpc_count) - 1);
895
896 gk20a_writel(g, gr_gpcs_gpccs_gpc_exception_en_r(),
897 (tpc_mask | gr_gpcs_gpccs_gpc_exception_en_gcc_f(1) |
898 gr_gpcs_gpccs_gpc_exception_en_gpccs_f(1) |
899 gr_gpcs_gpccs_gpc_exception_en_gpcmmu_f(1)));
900}
901
902int gr_gv11b_handle_tex_exception(struct gk20a *g, u32 gpc, u32 tpc,
903 bool *post_event)
904{
905 return 0;
906}
907
908int gr_gv11b_zbc_s_query_table(struct gk20a *g, struct gr_gk20a *gr,
909 struct zbc_query_params *query_params)
910{
911 u32 index = query_params->index_size;
912
913 if (index >= GK20A_ZBC_TABLE_SIZE) {
914 nvgpu_err(g, "invalid zbc stencil table index");
915 return -EINVAL;
916 }
917 query_params->depth = gr->zbc_s_tbl[index].stencil;
918 query_params->format = gr->zbc_s_tbl[index].format;
919 query_params->ref_cnt = gr->zbc_s_tbl[index].ref_cnt;
920
921 return 0;
922}
923
924bool gr_gv11b_add_zbc_type_s(struct gk20a *g, struct gr_gk20a *gr,
925 struct zbc_entry *zbc_val, int *ret_val)
926{
927 struct zbc_s_table *s_tbl;
928 u32 i;
929 bool added = false;
930
931 *ret_val = -ENOMEM;
932
933 /* search existing tables */
934 for (i = 0; i < gr->max_used_s_index; i++) {
935
936 s_tbl = &gr->zbc_s_tbl[i];
937
938 if (s_tbl->ref_cnt &&
939 s_tbl->stencil == zbc_val->depth &&
940 s_tbl->format == zbc_val->format) {
941 added = true;
942 s_tbl->ref_cnt++;
943 *ret_val = 0;
944 break;
945 }
946 }
947 /* add new table */
948 if (!added &&
949 gr->max_used_s_index < GK20A_ZBC_TABLE_SIZE) {
950
951 s_tbl = &gr->zbc_s_tbl[gr->max_used_s_index];
952 WARN_ON(s_tbl->ref_cnt != 0);
953
954 *ret_val = g->ops.gr.add_zbc_s(g, gr,
955 zbc_val, gr->max_used_s_index);
956
957 if (!(*ret_val))
958 gr->max_used_s_index++;
959 }
960 return added;
961}
962
963int gr_gv11b_add_zbc_stencil(struct gk20a *g, struct gr_gk20a *gr,
964 struct zbc_entry *stencil_val, u32 index)
965{
966 u32 zbc_s;
967
968 /* update l2 table */
969 g->ops.ltc.set_zbc_s_entry(g, stencil_val, index);
970
971 /* update local copy */
972 gr->zbc_s_tbl[index].stencil = stencil_val->depth;
973 gr->zbc_s_tbl[index].format = stencil_val->format;
974 gr->zbc_s_tbl[index].ref_cnt++;
975
976 gk20a_writel(g, gr_gpcs_swdx_dss_zbc_s_r(index), stencil_val->depth);
977 zbc_s = gk20a_readl(g, gr_gpcs_swdx_dss_zbc_s_01_to_04_format_r() +
978 (index & ~3));
979 zbc_s &= ~(0x7f << (index % 4) * 7);
980 zbc_s |= stencil_val->format << (index % 4) * 7;
981 gk20a_writel(g, gr_gpcs_swdx_dss_zbc_s_01_to_04_format_r() +
982 (index & ~3), zbc_s);
983
984 return 0;
985}
986
987int gr_gv11b_load_stencil_default_tbl(struct gk20a *g,
988 struct gr_gk20a *gr)
989{
990 struct zbc_entry zbc_val;
991 u32 err;
992
993 /* load default stencil table */
994 zbc_val.type = GV11B_ZBC_TYPE_STENCIL;
995
996 zbc_val.depth = 0x0;
997 zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8;
998 err = gr_gk20a_add_zbc(g, gr, &zbc_val);
999
1000 zbc_val.depth = 0x1;
1001 zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8;
1002 err |= gr_gk20a_add_zbc(g, gr, &zbc_val);
1003
1004 zbc_val.depth = 0xff;
1005 zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8;
1006 err |= gr_gk20a_add_zbc(g, gr, &zbc_val);
1007
1008 if (!err) {
1009 gr->max_default_s_index = 3;
1010 } else {
1011 nvgpu_err(g, "fail to load default zbc stencil table");
1012 return err;
1013 }
1014
1015 return 0;
1016}
1017
1018int gr_gv11b_load_stencil_tbl(struct gk20a *g, struct gr_gk20a *gr)
1019{
1020 int ret;
1021 u32 i;
1022
1023 for (i = 0; i < gr->max_used_s_index; i++) {
1024 struct zbc_s_table *s_tbl = &gr->zbc_s_tbl[i];
1025 struct zbc_entry zbc_val;
1026
1027 zbc_val.type = GV11B_ZBC_TYPE_STENCIL;
1028 zbc_val.depth = s_tbl->stencil;
1029 zbc_val.format = s_tbl->format;
1030
1031 ret = g->ops.gr.add_zbc_s(g, gr, &zbc_val, i);
1032 if (ret)
1033 return ret;
1034 }
1035 return 0;
1036}
1037
1038u32 gr_gv11b_pagepool_default_size(struct gk20a *g)
1039{
1040 return gr_scc_pagepool_total_pages_hwmax_value_v();
1041}
1042
1043int gr_gv11b_calc_global_ctx_buffer_size(struct gk20a *g)
1044{
1045 struct gr_gk20a *gr = &g->gr;
1046 int size;
1047
1048 gr->attrib_cb_size = gr->attrib_cb_default_size;
1049 gr->alpha_cb_size = gr->alpha_cb_default_size;
1050
1051 gr->attrib_cb_size = min(gr->attrib_cb_size,
1052 gr_gpc0_ppc0_cbm_beta_cb_size_v_f(~0) / g->gr.tpc_count);
1053 gr->alpha_cb_size = min(gr->alpha_cb_size,
1054 gr_gpc0_ppc0_cbm_alpha_cb_size_v_f(~0) / g->gr.tpc_count);
1055
1056 size = gr->attrib_cb_size *
1057 gr_gpc0_ppc0_cbm_beta_cb_size_v_granularity_v() *
1058 gr->max_tpc_count;
1059
1060 size += gr->alpha_cb_size *
1061 gr_gpc0_ppc0_cbm_alpha_cb_size_v_granularity_v() *
1062 gr->max_tpc_count;
1063
1064 size = ALIGN(size, 128);
1065
1066 return size;
1067}
1068
1069static void gr_gv11b_set_go_idle_timeout(struct gk20a *g, u32 data)
1070{
1071 gk20a_writel(g, gr_fe_go_idle_timeout_r(), data);
1072}
1073
1074static void gr_gv11b_set_coalesce_buffer_size(struct gk20a *g, u32 data)
1075{
1076 u32 val;
1077
1078 gk20a_dbg_fn("");
1079
1080 val = gk20a_readl(g, gr_gpcs_tc_debug0_r());
1081 val = set_field(val, gr_gpcs_tc_debug0_limit_coalesce_buffer_size_m(),
1082 gr_gpcs_tc_debug0_limit_coalesce_buffer_size_f(data));
1083 gk20a_writel(g, gr_gpcs_tc_debug0_r(), val);
1084
1085 gk20a_dbg_fn("done");
1086}
1087
1088static void gr_gv11b_set_tex_in_dbg(struct gk20a *g, u32 data)
1089{
1090 u32 val;
1091 bool flag;
1092
1093 gk20a_dbg_fn("");
1094
1095 val = gk20a_readl(g, gr_gpcs_tpcs_tex_in_dbg_r());
1096 flag = (data & NVC397_SET_TEX_IN_DBG_TSL1_RVCH_INVALIDATE) ? 1 : 0;
1097 val = set_field(val, gr_gpcs_tpcs_tex_in_dbg_tsl1_rvch_invalidate_m(),
1098 gr_gpcs_tpcs_tex_in_dbg_tsl1_rvch_invalidate_f(flag));
1099 gk20a_writel(g, gr_gpcs_tpcs_tex_in_dbg_r(), val);
1100
1101 val = gk20a_readl(g, gr_gpcs_tpcs_sm_l1tag_ctrl_r());
1102 flag = (data &
1103 NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_LD) ? 1 : 0;
1104 val = set_field(val, gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_ld_m(),
1105 gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_ld_f(flag));
1106 flag = (data &
1107 NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_ST) ? 1 : 0;
1108 val = set_field(val, gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_st_m(),
1109 gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_st_f(flag));
1110 gk20a_writel(g, gr_gpcs_tpcs_sm_l1tag_ctrl_r(), val);
1111}
1112
1113static void gr_gv11b_set_skedcheck(struct gk20a *g, u32 data)
1114{
1115 u32 reg_val;
1116
1117 reg_val = gk20a_readl(g, gr_sked_hww_esr_en_r());
1118
1119 if ((data & NVC397_SET_SKEDCHECK_18_MASK) ==
1120 NVC397_SET_SKEDCHECK_18_DISABLE) {
1121 reg_val = set_field(reg_val,
1122 gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_m(),
1123 gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_disabled_f()
1124 );
1125 } else if ((data & NVC397_SET_SKEDCHECK_18_MASK) ==
1126 NVC397_SET_SKEDCHECK_18_ENABLE) {
1127 reg_val = set_field(reg_val,
1128 gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_m(),
1129 gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_enabled_f()
1130 );
1131 }
1132 nvgpu_log_info(g, "sked_hww_esr_en = 0x%x", reg_val);
1133 gk20a_writel(g, gr_sked_hww_esr_en_r(), reg_val);
1134
1135}
1136
1137static void gv11b_gr_set_shader_exceptions(struct gk20a *g, u32 data)
1138{
1139 gk20a_dbg_fn("");
1140
1141 if (data == NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE) {
1142 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r(),
1143 0);
1144 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r(),
1145 0);
1146 } else {
1147 g->ops.gr.set_hww_esr_report_mask(g);
1148 }
1149}
1150
1151int gr_gv11b_handle_sw_method(struct gk20a *g, u32 addr,
1152 u32 class_num, u32 offset, u32 data)
1153{
1154 gk20a_dbg_fn("");
1155
1156 if (class_num == VOLTA_COMPUTE_A) {
1157 switch (offset << 2) {
1158 case NVC0C0_SET_SHADER_EXCEPTIONS:
1159 gv11b_gr_set_shader_exceptions(g, data);
1160 break;
1161 case NVC3C0_SET_SKEDCHECK:
1162 gr_gv11b_set_skedcheck(g, data);
1163 break;
1164 default:
1165 goto fail;
1166 }
1167 }
1168
1169 if (class_num == VOLTA_A) {
1170 switch (offset << 2) {
1171 case NVC397_SET_SHADER_EXCEPTIONS:
1172 gv11b_gr_set_shader_exceptions(g, data);
1173 break;
1174 case NVC397_SET_CIRCULAR_BUFFER_SIZE:
1175 g->ops.gr.set_circular_buffer_size(g, data);
1176 break;
1177 case NVC397_SET_ALPHA_CIRCULAR_BUFFER_SIZE:
1178 g->ops.gr.set_alpha_circular_buffer_size(g, data);
1179 break;
1180 case NVC397_SET_GO_IDLE_TIMEOUT:
1181 gr_gv11b_set_go_idle_timeout(g, data);
1182 break;
1183 case NVC097_SET_COALESCE_BUFFER_SIZE:
1184 gr_gv11b_set_coalesce_buffer_size(g, data);
1185 break;
1186 case NVC397_SET_TEX_IN_DBG:
1187 gr_gv11b_set_tex_in_dbg(g, data);
1188 break;
1189 case NVC397_SET_SKEDCHECK:
1190 gr_gv11b_set_skedcheck(g, data);
1191 break;
1192 case NVC397_SET_BES_CROP_DEBUG3:
1193 g->ops.gr.set_bes_crop_debug3(g, data);
1194 break;
1195 default:
1196 goto fail;
1197 }
1198 }
1199 return 0;
1200
1201fail:
1202 return -EINVAL;
1203}
1204
1205void gr_gv11b_bundle_cb_defaults(struct gk20a *g)
1206{
1207 struct gr_gk20a *gr = &g->gr;
1208
1209 gr->bundle_cb_default_size =
1210 gr_scc_bundle_cb_size_div_256b__prod_v();
1211 gr->min_gpm_fifo_depth =
1212 gr_pd_ab_dist_cfg2_state_limit_min_gpm_fifo_depths_v();
1213 gr->bundle_cb_token_limit =
1214 gr_pd_ab_dist_cfg2_token_limit_init_v();
1215}
1216
1217void gr_gv11b_cb_size_default(struct gk20a *g)
1218{
1219 struct gr_gk20a *gr = &g->gr;
1220
1221 if (!gr->attrib_cb_default_size)
1222 gr->attrib_cb_default_size =
1223 gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v();
1224 gr->alpha_cb_default_size =
1225 gr_gpc0_ppc0_cbm_alpha_cb_size_v_default_v();
1226}
1227
1228void gr_gv11b_set_alpha_circular_buffer_size(struct gk20a *g, u32 data)
1229{
1230 struct gr_gk20a *gr = &g->gr;
1231 u32 gpc_index, ppc_index, stride, val;
1232 u32 pd_ab_max_output;
1233 u32 alpha_cb_size = data * 4;
1234
1235 gk20a_dbg_fn("");
1236
1237 if (alpha_cb_size > gr->alpha_cb_size)
1238 alpha_cb_size = gr->alpha_cb_size;
1239
1240 gk20a_writel(g, gr_ds_tga_constraintlogic_alpha_r(),
1241 (gk20a_readl(g, gr_ds_tga_constraintlogic_alpha_r()) &
1242 ~gr_ds_tga_constraintlogic_alpha_cbsize_f(~0)) |
1243 gr_ds_tga_constraintlogic_alpha_cbsize_f(alpha_cb_size));
1244
1245 pd_ab_max_output = alpha_cb_size *
1246 gr_gpc0_ppc0_cbm_alpha_cb_size_v_granularity_v() /
1247 gr_pd_ab_dist_cfg1_max_output_granularity_v();
1248
1249 gk20a_writel(g, gr_pd_ab_dist_cfg1_r(),
1250 gr_pd_ab_dist_cfg1_max_output_f(pd_ab_max_output) |
1251 gr_pd_ab_dist_cfg1_max_batches_init_f());
1252
1253 for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) {
1254 stride = proj_gpc_stride_v() * gpc_index;
1255
1256 for (ppc_index = 0; ppc_index < gr->gpc_ppc_count[gpc_index];
1257 ppc_index++) {
1258
1259 val = gk20a_readl(g, gr_gpc0_ppc0_cbm_alpha_cb_size_r() +
1260 stride +
1261 proj_ppc_in_gpc_stride_v() * ppc_index);
1262
1263 val = set_field(val, gr_gpc0_ppc0_cbm_alpha_cb_size_v_m(),
1264 gr_gpc0_ppc0_cbm_alpha_cb_size_v_f(alpha_cb_size *
1265 gr->pes_tpc_count[ppc_index][gpc_index]));
1266
1267 gk20a_writel(g, gr_gpc0_ppc0_cbm_alpha_cb_size_r() +
1268 stride +
1269 proj_ppc_in_gpc_stride_v() * ppc_index, val);
1270 }
1271 }
1272}
1273
1274void gr_gv11b_set_circular_buffer_size(struct gk20a *g, u32 data)
1275{
1276 struct gr_gk20a *gr = &g->gr;
1277 u32 gpc_index, ppc_index, stride, val;
1278 u32 cb_size_steady = data * 4, cb_size;
1279
1280 gk20a_dbg_fn("");
1281
1282 if (cb_size_steady > gr->attrib_cb_size)
1283 cb_size_steady = gr->attrib_cb_size;
1284 if (gk20a_readl(g, gr_gpc0_ppc0_cbm_beta_cb_size_r()) !=
1285 gk20a_readl(g,
1286 gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_r())) {
1287 cb_size = cb_size_steady +
1288 (gr_gpc0_ppc0_cbm_beta_cb_size_v_gfxp_v() -
1289 gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v());
1290 } else {
1291 cb_size = cb_size_steady;
1292 }
1293
1294 gk20a_writel(g, gr_ds_tga_constraintlogic_beta_r(),
1295 (gk20a_readl(g, gr_ds_tga_constraintlogic_beta_r()) &
1296 ~gr_ds_tga_constraintlogic_beta_cbsize_f(~0)) |
1297 gr_ds_tga_constraintlogic_beta_cbsize_f(cb_size_steady));
1298
1299 for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) {
1300 stride = proj_gpc_stride_v() * gpc_index;
1301
1302 for (ppc_index = 0; ppc_index < gr->gpc_ppc_count[gpc_index];
1303 ppc_index++) {
1304
1305 val = gk20a_readl(g, gr_gpc0_ppc0_cbm_beta_cb_size_r() +
1306 stride +
1307 proj_ppc_in_gpc_stride_v() * ppc_index);
1308
1309 val = set_field(val,
1310 gr_gpc0_ppc0_cbm_beta_cb_size_v_m(),
1311 gr_gpc0_ppc0_cbm_beta_cb_size_v_f(cb_size *
1312 gr->pes_tpc_count[ppc_index][gpc_index]));
1313
1314 gk20a_writel(g, gr_gpc0_ppc0_cbm_beta_cb_size_r() +
1315 stride +
1316 proj_ppc_in_gpc_stride_v() * ppc_index, val);
1317
1318 gk20a_writel(g, proj_ppc_in_gpc_stride_v() * ppc_index +
1319 gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_r() +
1320 stride,
1321 gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_v_f(
1322 cb_size_steady));
1323
1324 val = gk20a_readl(g, gr_gpcs_swdx_tc_beta_cb_size_r(
1325 ppc_index + gpc_index));
1326
1327 val = set_field(val,
1328 gr_gpcs_swdx_tc_beta_cb_size_v_m(),
1329 gr_gpcs_swdx_tc_beta_cb_size_v_f(
1330 cb_size_steady *
1331 gr->gpc_ppc_count[gpc_index]));
1332
1333 gk20a_writel(g, gr_gpcs_swdx_tc_beta_cb_size_r(
1334 ppc_index + gpc_index), val);
1335 }
1336 }
1337}
1338
1339int gr_gv11b_alloc_buffer(struct vm_gk20a *vm, size_t size,
1340 struct nvgpu_mem *mem)
1341{
1342 int err;
1343
1344 gk20a_dbg_fn("");
1345
1346 err = nvgpu_dma_alloc_sys(vm->mm->g, size, mem);
1347 if (err)
1348 return err;
1349
1350 mem->gpu_va = nvgpu_gmmu_map(vm,
1351 mem,
1352 size,
1353 NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE,
1354 gk20a_mem_flag_none,
1355 false,
1356 mem->aperture);
1357
1358 if (!mem->gpu_va) {
1359 err = -ENOMEM;
1360 goto fail_free;
1361 }
1362
1363 return 0;
1364
1365fail_free:
1366 nvgpu_dma_free(vm->mm->g, mem);
1367 return err;
1368}
1369
1370static void gr_gv11b_dump_gr_per_sm_regs(struct gk20a *g,
1371 struct gk20a_debug_output *o,
1372 u32 gpc, u32 tpc, u32 sm, u32 offset)
1373{
1374
1375 gk20a_debug_output(o,
1376 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_WARP_ESR: 0x%x\n",
1377 gpc, tpc, sm, gk20a_readl(g,
1378 gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset));
1379
1380 gk20a_debug_output(o,
1381 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_WARP_ESR_REPORT_MASK: 0x%x\n",
1382 gpc, tpc, sm, gk20a_readl(g,
1383 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset));
1384
1385 gk20a_debug_output(o,
1386 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_GLOBAL_ESR: 0x%x\n",
1387 gpc, tpc, sm, gk20a_readl(g,
1388 gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset));
1389
1390 gk20a_debug_output(o,
1391 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_GLOBAL_ESR_REPORT_MASK: 0x%x\n",
1392 gpc, tpc, sm, gk20a_readl(g,
1393 gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset));
1394
1395 gk20a_debug_output(o,
1396 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_DBGR_CONTROL0: 0x%x\n",
1397 gpc, tpc, sm, gk20a_readl(g,
1398 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset));
1399
1400 gk20a_debug_output(o,
1401 "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_DBGR_STATUS0: 0x%x\n",
1402 gpc, tpc, sm, gk20a_readl(g,
1403 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset));
1404}
1405
1406static int gr_gv11b_dump_gr_sm_regs(struct gk20a *g,
1407 struct gk20a_debug_output *o)
1408{
1409 u32 gpc, tpc, sm, sm_per_tpc;
1410 u32 gpc_offset, tpc_offset, offset;
1411
1412 gk20a_debug_output(o,
1413 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x%x\n",
1414 gk20a_readl(g,
1415 gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r()));
1416 gk20a_debug_output(o,
1417 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0x%x\n",
1418 gk20a_readl(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r()));
1419 gk20a_debug_output(o,
1420 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x%x\n",
1421 gk20a_readl(g, gr_gpcs_tpcs_sms_hww_global_esr_r()));
1422 gk20a_debug_output(o,
1423 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x%x\n",
1424 gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_control0_r()));
1425 gk20a_debug_output(o,
1426 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x%x\n",
1427 gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_status0_r()));
1428 gk20a_debug_output(o,
1429 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x%x\n",
1430 gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_0_r()));
1431 gk20a_debug_output(o,
1432 "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x%x\n",
1433 gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_1_r()));
1434
1435 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC);
1436 for (gpc = 0; gpc < g->gr.gpc_count; gpc++) {
1437 gpc_offset = gk20a_gr_gpc_offset(g, gpc);
1438
1439 for (tpc = 0; tpc < g->gr.tpc_count; tpc++) {
1440 tpc_offset = gk20a_gr_tpc_offset(g, tpc);
1441
1442 for (sm = 0; sm < sm_per_tpc; sm++) {
1443 offset = gpc_offset + tpc_offset +
1444 gv11b_gr_sm_offset(g, sm);
1445
1446 gr_gv11b_dump_gr_per_sm_regs(g, o,
1447 gpc, tpc, sm, offset);
1448 }
1449 }
1450 }
1451
1452 return 0;
1453}
1454
1455int gr_gv11b_dump_gr_status_regs(struct gk20a *g,
1456 struct gk20a_debug_output *o)
1457{
1458 struct gr_gk20a *gr = &g->gr;
1459 u32 gr_engine_id;
1460
1461 gr_engine_id = gk20a_fifo_get_gr_engine_id(g);
1462
1463 gk20a_debug_output(o, "NV_PGRAPH_STATUS: 0x%x\n",
1464 gk20a_readl(g, gr_status_r()));
1465 gk20a_debug_output(o, "NV_PGRAPH_STATUS1: 0x%x\n",
1466 gk20a_readl(g, gr_status_1_r()));
1467 gk20a_debug_output(o, "NV_PGRAPH_STATUS2: 0x%x\n",
1468 gk20a_readl(g, gr_status_2_r()));
1469 gk20a_debug_output(o, "NV_PGRAPH_ENGINE_STATUS: 0x%x\n",
1470 gk20a_readl(g, gr_engine_status_r()));
1471 gk20a_debug_output(o, "NV_PGRAPH_GRFIFO_STATUS : 0x%x\n",
1472 gk20a_readl(g, gr_gpfifo_status_r()));
1473 gk20a_debug_output(o, "NV_PGRAPH_GRFIFO_CONTROL : 0x%x\n",
1474 gk20a_readl(g, gr_gpfifo_ctl_r()));
1475 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x%x\n",
1476 gk20a_readl(g, gr_fecs_host_int_status_r()));
1477 gk20a_debug_output(o, "NV_PGRAPH_EXCEPTION : 0x%x\n",
1478 gk20a_readl(g, gr_exception_r()));
1479 gk20a_debug_output(o, "NV_PGRAPH_FECS_INTR : 0x%x\n",
1480 gk20a_readl(g, gr_fecs_intr_r()));
1481 gk20a_debug_output(o, "NV_PFIFO_ENGINE_STATUS(GR) : 0x%x\n",
1482 gk20a_readl(g, fifo_engine_status_r(gr_engine_id)));
1483 gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY0: 0x%x\n",
1484 gk20a_readl(g, gr_activity_0_r()));
1485 gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY1: 0x%x\n",
1486 gk20a_readl(g, gr_activity_1_r()));
1487 gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY2: 0x%x\n",
1488 gk20a_readl(g, gr_activity_2_r()));
1489 gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY4: 0x%x\n",
1490 gk20a_readl(g, gr_activity_4_r()));
1491 gk20a_debug_output(o, "NV_PGRAPH_PRI_SKED_ACTIVITY: 0x%x\n",
1492 gk20a_readl(g, gr_pri_sked_activity_r()));
1493 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x%x\n",
1494 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity0_r()));
1495 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x%x\n",
1496 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity1_r()));
1497 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x%x\n",
1498 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity2_r()));
1499 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x%x\n",
1500 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity3_r()));
1501 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1502 gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_activity_0_r()));
1503 if (gr->gpc_tpc_count[0] == 2)
1504 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1505 gk20a_readl(g, gr_pri_gpc0_tpc1_tpccs_tpc_activity_0_r()));
1506 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPCS_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1507 gk20a_readl(g, gr_pri_gpc0_tpcs_tpccs_tpc_activity_0_r()));
1508 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x%x\n",
1509 gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_0_r()));
1510 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x%x\n",
1511 gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_1_r()));
1512 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x%x\n",
1513 gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_2_r()));
1514 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x%x\n",
1515 gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_3_r()));
1516 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPC0_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1517 gk20a_readl(g, gr_pri_gpcs_tpc0_tpccs_tpc_activity_0_r()));
1518 if (gr->gpc_tpc_count[0] == 2)
1519 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPC1_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1520 gk20a_readl(g, gr_pri_gpcs_tpc1_tpccs_tpc_activity_0_r()));
1521 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x%x\n",
1522 gk20a_readl(g, gr_pri_gpcs_tpcs_tpccs_tpc_activity_0_r()));
1523 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_ACTIVITY0: 0x%x\n",
1524 gk20a_readl(g, gr_pri_be0_becs_be_activity0_r()));
1525 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE1_BECS_BE_ACTIVITY0: 0x%x\n",
1526 gk20a_readl(g, gr_pri_be1_becs_be_activity0_r()));
1527 gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_BECS_BE_ACTIVITY0: 0x%x\n",
1528 gk20a_readl(g, gr_pri_bes_becs_be_activity0_r()));
1529 gk20a_debug_output(o, "NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x%x\n",
1530 gk20a_readl(g, gr_pri_ds_mpipe_status_r()));
1531 gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x%x\n",
1532 gk20a_readl(g, gr_fe_go_idle_timeout_r()));
1533 gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x%x\n",
1534 gk20a_readl(g, gr_pri_fe_go_idle_info_r()));
1535 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x%x\n",
1536 gk20a_readl(g, gr_pri_gpc0_tpc0_tex_m_tex_subunits_status_r()));
1537 gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_FS: 0x%x\n",
1538 gk20a_readl(g, gr_cwd_fs_r()));
1539 gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_TPC_FS(0): 0x%x\n",
1540 gk20a_readl(g, gr_fe_tpc_fs_r(0)));
1541 gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x%x\n",
1542 gk20a_readl(g, gr_cwd_gpc_tpc_id_r(0)));
1543 gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_SM_ID(0): 0x%x\n",
1544 gk20a_readl(g, gr_cwd_sm_id_r(0)));
1545 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x%x\n",
1546 gk20a_readl(g, gr_fecs_ctxsw_status_fe_0_r()));
1547 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x%x\n",
1548 gk20a_readl(g, gr_fecs_ctxsw_status_1_r()));
1549 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x%x\n",
1550 gk20a_readl(g, gr_gpc0_gpccs_ctxsw_status_gpc_0_r()));
1551 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x%x\n",
1552 gk20a_readl(g, gr_gpc0_gpccs_ctxsw_status_1_r()));
1553 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0x%x\n",
1554 gk20a_readl(g, gr_fecs_ctxsw_idlestate_r()));
1555 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0x%x\n",
1556 gk20a_readl(g, gr_gpc0_gpccs_ctxsw_idlestate_r()));
1557 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0x%x\n",
1558 gk20a_readl(g, gr_fecs_current_ctx_r()));
1559 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_NEW_CTX : 0x%x\n",
1560 gk20a_readl(g, gr_fecs_new_ctx_r()));
1561 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x%x\n",
1562 gk20a_readl(g, gr_fecs_host_int_enable_r()));
1563 gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x%x\n",
1564 gk20a_readl(g, gr_fecs_host_int_status_r()));
1565 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_CROP_STATUS1 : 0x%x\n",
1566 gk20a_readl(g, gr_pri_be0_crop_status1_r()));
1567 gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_CROP_STATUS1 : 0x%x\n",
1568 gk20a_readl(g, gr_pri_bes_crop_status1_r()));
1569 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_ZROP_STATUS : 0x%x\n",
1570 gk20a_readl(g, gr_pri_be0_zrop_status_r()));
1571 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_ZROP_STATUS2 : 0x%x\n",
1572 gk20a_readl(g, gr_pri_be0_zrop_status2_r()));
1573 gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_ZROP_STATUS : 0x%x\n",
1574 gk20a_readl(g, gr_pri_bes_zrop_status_r()));
1575 gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_ZROP_STATUS2 : 0x%x\n",
1576 gk20a_readl(g, gr_pri_bes_zrop_status2_r()));
1577 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_EXCEPTION: 0x%x\n",
1578 gk20a_readl(g, gr_pri_be0_becs_be_exception_r()));
1579 gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_EXCEPTION_EN: 0x%x\n",
1580 gk20a_readl(g, gr_pri_be0_becs_be_exception_en_r()));
1581 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x%x\n",
1582 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_exception_r()));
1583 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0x%x\n",
1584 gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_exception_en_r()));
1585 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x%x\n",
1586 gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_exception_r()));
1587 gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x%x\n",
1588 gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_exception_en_r()));
1589
1590 gr_gv11b_dump_gr_sm_regs(g, o);
1591
1592 return 0;
1593}
1594
1595static bool gr_activity_empty_or_preempted(u32 val)
1596{
1597 while (val) {
1598 u32 v = val & 7;
1599 if (v != gr_activity_4_gpc0_empty_v() &&
1600 v != gr_activity_4_gpc0_preempted_v())
1601 return false;
1602 val >>= 3;
1603 }
1604
1605 return true;
1606}
1607
1608int gr_gv11b_wait_empty(struct gk20a *g, unsigned long duration_ms,
1609 u32 expect_delay)
1610{
1611 u32 delay = expect_delay;
1612 bool gr_enabled;
1613 bool ctxsw_active;
1614 bool gr_busy;
1615 u32 gr_status;
1616 u32 activity0, activity1, activity2, activity4;
1617 struct nvgpu_timeout timeout;
1618
1619 gk20a_dbg_fn("");
1620
1621 nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER);
1622
1623 do {
1624 /* fmodel: host gets fifo_engine_status(gr) from gr
1625 only when gr_status is read */
1626 gr_status = gk20a_readl(g, gr_status_r());
1627
1628 gr_enabled = gk20a_readl(g, mc_enable_r()) &
1629 mc_enable_pgraph_enabled_f();
1630
1631 ctxsw_active = gr_status & 1<<7;
1632
1633 activity0 = gk20a_readl(g, gr_activity_0_r());
1634 activity1 = gk20a_readl(g, gr_activity_1_r());
1635 activity2 = gk20a_readl(g, gr_activity_2_r());
1636 activity4 = gk20a_readl(g, gr_activity_4_r());
1637
1638 gr_busy = !(gr_activity_empty_or_preempted(activity0) &&
1639 gr_activity_empty_or_preempted(activity1) &&
1640 activity2 == 0 &&
1641 gr_activity_empty_or_preempted(activity4));
1642
1643 if (!gr_enabled || (!gr_busy && !ctxsw_active)) {
1644 gk20a_dbg_fn("done");
1645 return 0;
1646 }
1647
1648 usleep_range(delay, delay * 2);
1649 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
1650
1651 } while (!nvgpu_timeout_expired(&timeout));
1652
1653 nvgpu_err(g,
1654 "timeout, ctxsw busy : %d, gr busy : %d, %08x, %08x, %08x, %08x",
1655 ctxsw_active, gr_busy, activity0, activity1, activity2, activity4);
1656
1657 return -EAGAIN;
1658}
1659
1660void gr_gv11b_commit_global_attrib_cb(struct gk20a *g,
1661 struct channel_ctx_gk20a *ch_ctx,
1662 u64 addr, bool patch)
1663{
1664 struct gr_ctx_desc *gr_ctx = ch_ctx->gr_ctx;
1665 int attrBufferSize;
1666
1667 if (gr_ctx->t18x.preempt_ctxsw_buffer.gpu_va)
1668 attrBufferSize = gr_ctx->t18x.betacb_ctxsw_buffer.size;
1669 else
1670 attrBufferSize = g->ops.gr.calc_global_ctx_buffer_size(g);
1671
1672 attrBufferSize /= gr_gpcs_tpcs_tex_rm_cb_1_size_div_128b_granularity_f();
1673
1674 gr_gm20b_commit_global_attrib_cb(g, ch_ctx, addr, patch);
1675
1676 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_r(),
1677 gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_v_f(addr) |
1678 gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_valid_true_f(), patch);
1679
1680 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_tex_rm_cb_0_r(),
1681 gr_gpcs_tpcs_tex_rm_cb_0_base_addr_43_12_f(addr), patch);
1682
1683 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_tex_rm_cb_1_r(),
1684 gr_gpcs_tpcs_tex_rm_cb_1_size_div_128b_f(attrBufferSize) |
1685 gr_gpcs_tpcs_tex_rm_cb_1_valid_true_f(), patch);
1686}
1687
1688void gr_gv11b_set_gpc_tpc_mask(struct gk20a *g, u32 gpc_index)
1689{
1690#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
1691 tegra_fuse_writel(0x1, FUSE_FUSEBYPASS_0);
1692 tegra_fuse_writel(0x0, FUSE_WRITE_ACCESS_SW_0);
1693#else
1694 tegra_fuse_control_write(0x1, FUSE_FUSEBYPASS_0);
1695 tegra_fuse_control_write(0x0, FUSE_WRITE_ACCESS_SW_0);
1696#endif
1697
1698 if (g->gr.gpc_tpc_mask[gpc_index] == 0x1)
1699 tegra_fuse_writel(0x2, FUSE_OPT_GPU_TPC0_DISABLE_0);
1700 else if (g->gr.gpc_tpc_mask[gpc_index] == 0x2)
1701 tegra_fuse_writel(0x1, FUSE_OPT_GPU_TPC0_DISABLE_0);
1702 else
1703 tegra_fuse_writel(0x0, FUSE_OPT_GPU_TPC0_DISABLE_0);
1704}
1705
1706void gr_gv11b_get_access_map(struct gk20a *g,
1707 u32 **whitelist, int *num_entries)
1708{
1709 static u32 wl_addr_gv11b[] = {
1710 /* this list must be sorted (low to high) */
1711 0x404468, /* gr_pri_mme_max_instructions */
1712 0x418300, /* gr_pri_gpcs_rasterarb_line_class */
1713 0x418800, /* gr_pri_gpcs_setup_debug */
1714 0x418e00, /* gr_pri_gpcs_swdx_config */
1715 0x418e40, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */
1716 0x418e44, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */
1717 0x418e48, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */
1718 0x418e4c, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */
1719 0x418e50, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */
1720 0x418e58, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1721 0x418e5c, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1722 0x418e60, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1723 0x418e64, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1724 0x418e68, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1725 0x418e6c, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1726 0x418e70, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1727 0x418e74, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1728 0x418e78, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1729 0x418e7c, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1730 0x418e80, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1731 0x418e84, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1732 0x418e88, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1733 0x418e8c, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1734 0x418e90, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1735 0x418e94, /* gr_pri_gpcs_swdx_tc_bundle_addr */
1736 0x419864, /* gr_pri_gpcs_tpcs_pe_l2_evict_policy */
1737 0x419a04, /* gr_pri_gpcs_tpcs_tex_lod_dbg */
1738 0x419a08, /* gr_pri_gpcs_tpcs_tex_samp_dbg */
1739 0x419e84, /* gr_pri_gpcs_tpcs_sms_dbgr_control0 */
1740 0x419ba4, /* gr_pri_gpcs_tpcs_sm_disp_ctrl */
1741 };
1742
1743 *whitelist = wl_addr_gv11b;
1744 *num_entries = ARRAY_SIZE(wl_addr_gv11b);
1745}
1746
1747/* @brief pre-process work on the SM exceptions to determine if we clear them or not.
1748 *
1749 * On Pascal, if we are in CILP preemtion mode, preempt the channel and handle errors with special processing
1750 */
1751int gr_gv11b_pre_process_sm_exception(struct gk20a *g,
1752 u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr,
1753 bool sm_debugger_attached, struct channel_gk20a *fault_ch,
1754 bool *early_exit, bool *ignore_debugger)
1755{
1756 int ret;
1757 bool cilp_enabled = false;
1758 u32 global_mask = 0, dbgr_control0, global_esr_copy;
1759 u32 offset = gk20a_gr_gpc_offset(g, gpc) +
1760 gk20a_gr_tpc_offset(g, tpc) +
1761 gv11b_gr_sm_offset(g, sm);
1762
1763 *early_exit = false;
1764 *ignore_debugger = false;
1765
1766 if (fault_ch)
1767 cilp_enabled = (fault_ch->ch_ctx.gr_ctx->compute_preempt_mode ==
1768 NVGPU_PREEMPTION_MODE_COMPUTE_CILP);
1769
1770 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1771 "SM Exception received on gpc %d tpc %d sm %d = 0x%08x",
1772 gpc, tpc, sm, global_esr);
1773
1774 if (cilp_enabled && sm_debugger_attached) {
1775 if (global_esr & gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f())
1776 gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset,
1777 gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f());
1778
1779 if (global_esr & gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f())
1780 gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset,
1781 gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f());
1782
1783 global_mask = gr_gpc0_tpc0_sm0_hww_global_esr_multiple_warp_errors_pending_f() |
1784 gr_gpc0_tpc0_sm0_hww_global_esr_bpt_pause_pending_f();
1785
1786 if (warp_esr != 0 || (global_esr & global_mask) != 0) {
1787 *ignore_debugger = true;
1788
1789 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1790 "CILP: starting wait for LOCKED_DOWN on "
1791 "gpc %d tpc %d sm %d",
1792 gpc, tpc, sm);
1793
1794 if (gk20a_dbg_gpu_broadcast_stop_trigger(fault_ch)) {
1795 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1796 "CILP: Broadcasting STOP_TRIGGER from "
1797 "gpc %d tpc %d sm %d",
1798 gpc, tpc, sm);
1799 g->ops.gr.suspend_all_sms(g,
1800 global_mask, false);
1801
1802 gk20a_dbg_gpu_clear_broadcast_stop_trigger(fault_ch);
1803 } else {
1804 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1805 "CILP: STOP_TRIGGER from "
1806 "gpc %d tpc %d sm %d",
1807 gpc, tpc, sm);
1808 g->ops.gr.suspend_single_sm(g,
1809 gpc, tpc, sm, global_mask, true);
1810 }
1811
1812 /* reset the HWW errors after locking down */
1813 global_esr_copy = g->ops.gr.get_sm_hww_global_esr(g,
1814 gpc, tpc, sm);
1815 g->ops.gr.clear_sm_hww(g,
1816 gpc, tpc, sm, global_esr_copy);
1817 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1818 "CILP: HWWs cleared for "
1819 "gpc %d tpc %d sm %d",
1820 gpc, tpc, sm);
1821
1822 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, "CILP: Setting CILP preempt pending\n");
1823 ret = gr_gp10b_set_cilp_preempt_pending(g, fault_ch);
1824 if (ret) {
1825 nvgpu_err(g, "CILP: error while setting CILP preempt pending!");
1826 return ret;
1827 }
1828
1829 dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
1830 if (dbgr_control0 & gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_enable_f()) {
1831 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1832 "CILP: clearing SINGLE_STEP_MODE "
1833 "before resume for gpc %d tpc %d sm %d",
1834 gpc, tpc, sm);
1835 dbgr_control0 = set_field(dbgr_control0,
1836 gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_m(),
1837 gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_disable_f());
1838 gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset, dbgr_control0);
1839 }
1840
1841 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1842 "CILP: resume for gpc %d tpc %d sm %d",
1843 gpc, tpc, sm);
1844 g->ops.gr.resume_single_sm(g, gpc, tpc, sm);
1845
1846 *ignore_debugger = true;
1847 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
1848 "CILP: All done on gpc %d, tpc %d sm %d",
1849 gpc, tpc, sm);
1850 }
1851
1852 *early_exit = true;
1853 }
1854 return 0;
1855}
1856
1857static void gr_gv11b_handle_fecs_ecc_error(struct gk20a *g, u32 intr)
1858{
1859 u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt;
1860 u32 corrected_delta, uncorrected_delta;
1861 u32 corrected_overflow, uncorrected_overflow;
1862
1863 if (intr & (gr_fecs_host_int_status_ecc_uncorrected_m() |
1864 gr_fecs_host_int_status_ecc_corrected_m())) {
1865 ecc_status = gk20a_readl(g, gr_fecs_falcon_ecc_status_r());
1866 ecc_addr = gk20a_readl(g,
1867 gr_fecs_falcon_ecc_address_r());
1868 corrected_cnt = gk20a_readl(g,
1869 gr_fecs_falcon_ecc_corrected_err_count_r());
1870 uncorrected_cnt = gk20a_readl(g,
1871 gr_fecs_falcon_ecc_uncorrected_err_count_r());
1872
1873 corrected_delta =
1874 gr_fecs_falcon_ecc_corrected_err_count_total_v(
1875 corrected_cnt);
1876 uncorrected_delta =
1877 gr_fecs_falcon_ecc_uncorrected_err_count_total_v(
1878 uncorrected_cnt);
1879
1880 corrected_overflow = ecc_status &
1881 gr_fecs_falcon_ecc_status_corrected_err_total_counter_overflow_m();
1882 uncorrected_overflow = ecc_status &
1883 gr_fecs_falcon_ecc_status_uncorrected_err_total_counter_overflow_m();
1884
1885 /* clear the interrupt */
1886 if ((corrected_delta > 0) || corrected_overflow)
1887 gk20a_writel(g,
1888 gr_fecs_falcon_ecc_corrected_err_count_r(), 0);
1889 if ((uncorrected_delta > 0) || uncorrected_overflow)
1890 gk20a_writel(g,
1891 gr_fecs_falcon_ecc_uncorrected_err_count_r(),
1892 0);
1893
1894
1895 /* clear the interrupt */
1896 gk20a_writel(g, gr_fecs_falcon_ecc_uncorrected_err_count_r(),
1897 0);
1898 gk20a_writel(g, gr_fecs_falcon_ecc_corrected_err_count_r(), 0);
1899
1900 /* clear the interrupt */
1901 gk20a_writel(g, gr_fecs_falcon_ecc_status_r(),
1902 gr_fecs_falcon_ecc_status_reset_task_f());
1903
1904 g->ecc.gr.t19x.fecs_corrected_err_count.counters[0] +=
1905 corrected_delta;
1906 g->ecc.gr.t19x.fecs_uncorrected_err_count.counters[0] +=
1907 uncorrected_delta;
1908
1909 nvgpu_log(g, gpu_dbg_intr,
1910 "fecs ecc interrupt intr: 0x%x", intr);
1911
1912 if (ecc_status &
1913 gr_fecs_falcon_ecc_status_corrected_err_imem_m())
1914 nvgpu_log(g, gpu_dbg_intr, "imem ecc error corrected");
1915 if (ecc_status &
1916 gr_fecs_falcon_ecc_status_uncorrected_err_imem_m())
1917 nvgpu_log(g, gpu_dbg_intr,
1918 "imem ecc error uncorrected");
1919 if (ecc_status &
1920 gr_fecs_falcon_ecc_status_corrected_err_dmem_m())
1921 nvgpu_log(g, gpu_dbg_intr, "dmem ecc error corrected");
1922 if (ecc_status &
1923 gr_fecs_falcon_ecc_status_uncorrected_err_dmem_m())
1924 nvgpu_log(g, gpu_dbg_intr,
1925 "dmem ecc error uncorrected");
1926 if (corrected_overflow || uncorrected_overflow)
1927 nvgpu_info(g, "fecs ecc counter overflow!");
1928
1929 nvgpu_log(g, gpu_dbg_intr,
1930 "ecc error row address: 0x%x",
1931 gr_fecs_falcon_ecc_address_row_address_v(ecc_addr));
1932
1933 nvgpu_log(g, gpu_dbg_intr,
1934 "ecc error count corrected: %d, uncorrected %d",
1935 g->ecc.gr.t19x.fecs_corrected_err_count.counters[0],
1936 g->ecc.gr.t19x.fecs_uncorrected_err_count.counters[0]);
1937 }
1938}
1939
1940int gr_gv11b_handle_fecs_error(struct gk20a *g,
1941 struct channel_gk20a *__ch,
1942 struct gr_gk20a_isr_data *isr_data)
1943{
1944 u32 gr_fecs_intr = gk20a_readl(g, gr_fecs_host_int_status_r());
1945 int ret;
1946
1947 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, "");
1948
1949 ret = gr_gp10b_handle_fecs_error(g, __ch, isr_data);
1950
1951 /* Handle ECC errors */
1952 gr_gv11b_handle_fecs_ecc_error(g, gr_fecs_intr);
1953
1954 return ret;
1955}
1956
1957int gr_gv11b_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr)
1958{
1959 u32 map;
1960 u32 i, j, mapregs;
1961 u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS);
1962 u32 num_tpc_per_gpc = nvgpu_get_litter_value(g,
1963 GPU_LIT_NUM_TPC_PER_GPC);
1964
1965 gk20a_dbg_fn("");
1966
1967 if (!gr->map_tiles)
1968 return -1;
1969
1970 gk20a_writel(g, gr_crstr_map_table_cfg_r(),
1971 gr_crstr_map_table_cfg_row_offset_f(gr->map_row_offset) |
1972 gr_crstr_map_table_cfg_num_entries_f(gr->tpc_count));
1973
1974 /* 6 tpc can be stored in one map register */
1975 mapregs = (num_gpcs * num_tpc_per_gpc + 5) / 6;
1976
1977 for (i = 0, j = 0; i < mapregs; i++, j = j + 6) {
1978 map = gr_crstr_gpc_map_tile0_f(gr->map_tiles[j]) |
1979 gr_crstr_gpc_map_tile1_f(gr->map_tiles[j + 1]) |
1980 gr_crstr_gpc_map_tile2_f(gr->map_tiles[j + 2]) |
1981 gr_crstr_gpc_map_tile3_f(gr->map_tiles[j + 3]) |
1982 gr_crstr_gpc_map_tile4_f(gr->map_tiles[j + 4]) |
1983 gr_crstr_gpc_map_tile5_f(gr->map_tiles[j + 5]);
1984
1985 gk20a_writel(g, gr_crstr_gpc_map_r(i), map);
1986 gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map_r(i), map);
1987 gk20a_writel(g, gr_rstr2d_gpc_map_r(i), map);
1988 }
1989
1990 gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_r(),
1991 gr_ppcs_wwdx_map_table_cfg_row_offset_f(gr->map_row_offset) |
1992 gr_ppcs_wwdx_map_table_cfg_num_entries_f(gr->tpc_count));
1993
1994 for (i = 0, j = 1; i < gr_ppcs_wwdx_map_table_cfg_coeff__size_1_v();
1995 i++, j = j + 4) {
1996 gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_coeff_r(i),
1997 gr_ppcs_wwdx_map_table_cfg_coeff_0_mod_value_f(
1998 ((1 << j) % gr->tpc_count)) |
1999 gr_ppcs_wwdx_map_table_cfg_coeff_1_mod_value_f(
2000 ((1 << (j + 1)) % gr->tpc_count)) |
2001 gr_ppcs_wwdx_map_table_cfg_coeff_2_mod_value_f(
2002 ((1 << (j + 2)) % gr->tpc_count)) |
2003 gr_ppcs_wwdx_map_table_cfg_coeff_3_mod_value_f(
2004 ((1 << (j + 3)) % gr->tpc_count)));
2005 }
2006
2007 gk20a_writel(g, gr_rstr2d_map_table_cfg_r(),
2008 gr_rstr2d_map_table_cfg_row_offset_f(gr->map_row_offset) |
2009 gr_rstr2d_map_table_cfg_num_entries_f(gr->tpc_count));
2010
2011 return 0;
2012}
2013
2014static int gv11b_write_bundle_veid_state(struct gk20a *g, u32 index)
2015{
2016 struct av_list_gk20a *sw_veid_bundle_init =
2017 &g->gr.ctx_vars.sw_veid_bundle_init;
2018 u32 j;
2019 u32 num_subctx, err = 0;
2020
2021 num_subctx = g->fifo.t19x.max_subctx_count;
2022
2023 for (j = 0; j < num_subctx; j++) {
2024 nvgpu_log_fn(g, "write bundle_address_r for subctx: %d", j);
2025 gk20a_writel(g, gr_pipe_bundle_address_r(),
2026 sw_veid_bundle_init->l[index].addr |
2027 gr_pipe_bundle_address_veid_f(j));
2028
2029 err = gr_gk20a_wait_fe_idle(g, gk20a_get_gr_idle_timeout(g),
2030 GR_IDLE_CHECK_DEFAULT);
2031 }
2032 return err;
2033}
2034
2035int gr_gv11b_init_sw_veid_bundle(struct gk20a *g)
2036{
2037 struct av_list_gk20a *sw_veid_bundle_init =
2038 &g->gr.ctx_vars.sw_veid_bundle_init;
2039 u32 i;
2040 u32 last_bundle_data = 0;
2041 u32 err = 0;
2042
2043 for (i = 0; i < sw_veid_bundle_init->count; i++) {
2044 nvgpu_log_fn(g, "veid bundle count: %d", i);
2045
2046 if (i == 0 || last_bundle_data !=
2047 sw_veid_bundle_init->l[i].value) {
2048 gk20a_writel(g, gr_pipe_bundle_data_r(),
2049 sw_veid_bundle_init->l[i].value);
2050 last_bundle_data = sw_veid_bundle_init->l[i].value;
2051 nvgpu_log_fn(g, "last_bundle_data : 0x%08x",
2052 last_bundle_data);
2053 }
2054
2055 if (gr_pipe_bundle_address_value_v(
2056 sw_veid_bundle_init->l[i].addr) == GR_GO_IDLE_BUNDLE) {
2057 nvgpu_log_fn(g, "go idle bundle");
2058 gk20a_writel(g, gr_pipe_bundle_address_r(),
2059 sw_veid_bundle_init->l[i].addr);
2060 err |= gr_gk20a_wait_idle(g,
2061 gk20a_get_gr_idle_timeout(g),
2062 GR_IDLE_CHECK_DEFAULT);
2063 } else
2064 err = gv11b_write_bundle_veid_state(g, i);
2065
2066 if (err) {
2067 nvgpu_err(g, "failed to init sw veid bundle");
2068 break;
2069 }
2070 }
2071 return err;
2072}
2073
2074void gr_gv11b_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries,
2075 u32 *zcull_map_tiles)
2076{
2077 u32 val, i, j;
2078
2079 gk20a_dbg_fn("");
2080
2081 for (i = 0, j = 0; i < (zcull_num_entries / 8); i++, j += 8) {
2082 val =
2083 gr_gpcs_zcull_sm_in_gpc_number_map_tile_0_f(
2084 zcull_map_tiles[j+0]) |
2085 gr_gpcs_zcull_sm_in_gpc_number_map_tile_1_f(
2086 zcull_map_tiles[j+1]) |
2087 gr_gpcs_zcull_sm_in_gpc_number_map_tile_2_f(
2088 zcull_map_tiles[j+2]) |
2089 gr_gpcs_zcull_sm_in_gpc_number_map_tile_3_f(
2090 zcull_map_tiles[j+3]) |
2091 gr_gpcs_zcull_sm_in_gpc_number_map_tile_4_f(
2092 zcull_map_tiles[j+4]) |
2093 gr_gpcs_zcull_sm_in_gpc_number_map_tile_5_f(
2094 zcull_map_tiles[j+5]) |
2095 gr_gpcs_zcull_sm_in_gpc_number_map_tile_6_f(
2096 zcull_map_tiles[j+6]) |
2097 gr_gpcs_zcull_sm_in_gpc_number_map_tile_7_f(
2098 zcull_map_tiles[j+7]);
2099
2100 gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map_r(i), val);
2101 }
2102}
2103
2104void gr_gv11b_detect_sm_arch(struct gk20a *g)
2105{
2106 u32 v = gk20a_readl(g, gr_gpc0_tpc0_sm_arch_r());
2107
2108 g->params.sm_arch_spa_version =
2109 gr_gpc0_tpc0_sm_arch_spa_version_v(v);
2110 g->params.sm_arch_sm_version =
2111 gr_gpc0_tpc0_sm_arch_sm_version_v(v);
2112 g->params.sm_arch_warp_count =
2113 gr_gpc0_tpc0_sm_arch_warp_count_v(v);
2114}
2115
2116void gr_gv11b_program_sm_id_numbering(struct gk20a *g,
2117 u32 gpc, u32 tpc, u32 smid)
2118{
2119 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
2120 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g,
2121 GPU_LIT_TPC_IN_GPC_STRIDE);
2122 u32 gpc_offset = gpc_stride * gpc;
2123 u32 tpc_offset = tpc_in_gpc_stride * tpc;
2124 u32 global_tpc_index = g->gr.sm_to_cluster[smid].global_tpc_index;
2125
2126 gk20a_writel(g, gr_gpc0_tpc0_sm_cfg_r() + gpc_offset + tpc_offset,
2127 gr_gpc0_tpc0_sm_cfg_tpc_id_f(global_tpc_index));
2128 gk20a_writel(g, gr_gpc0_gpm_pd_sm_id_r(tpc) + gpc_offset,
2129 gr_gpc0_gpm_pd_sm_id_id_f(global_tpc_index));
2130 gk20a_writel(g, gr_gpc0_tpc0_pe_cfg_smid_r() + gpc_offset + tpc_offset,
2131 gr_gpc0_tpc0_pe_cfg_smid_value_f(global_tpc_index));
2132}
2133
2134int gr_gv11b_load_smid_config(struct gk20a *g)
2135{
2136 u32 *tpc_sm_id;
2137 u32 i, j;
2138 u32 tpc_index, gpc_index, tpc_id;
2139 u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC);
2140 int num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS);
2141
2142 tpc_sm_id = nvgpu_kcalloc(g, gr_cwd_sm_id__size_1_v(), sizeof(u32));
2143 if (!tpc_sm_id)
2144 return -ENOMEM;
2145
2146 /* Each NV_PGRAPH_PRI_CWD_GPC_TPC_ID can store 4 TPCs.*/
2147 for (i = 0; i <= ((g->gr.tpc_count-1) / 4); i++) {
2148 u32 reg = 0;
2149 u32 bit_stride = gr_cwd_gpc_tpc_id_gpc0_s() +
2150 gr_cwd_gpc_tpc_id_tpc0_s();
2151
2152 for (j = 0; j < 4; j++) {
2153 u32 sm_id;
2154 u32 bits;
2155
2156 tpc_id = (i << 2) + j;
2157 sm_id = tpc_id * sm_per_tpc;
2158
2159 if (sm_id >= g->gr.no_of_sm)
2160 break;
2161
2162 gpc_index = g->gr.sm_to_cluster[sm_id].gpc_index;
2163 tpc_index = g->gr.sm_to_cluster[sm_id].tpc_index;
2164
2165 bits = gr_cwd_gpc_tpc_id_gpc0_f(gpc_index) |
2166 gr_cwd_gpc_tpc_id_tpc0_f(tpc_index);
2167 reg |= bits << (j * bit_stride);
2168
2169 tpc_sm_id[gpc_index + (num_gpcs * ((tpc_index & 4)
2170 >> 2))] |= tpc_id << tpc_index * bit_stride;
2171 }
2172 gk20a_writel(g, gr_cwd_gpc_tpc_id_r(i), reg);
2173 }
2174
2175 for (i = 0; i < gr_cwd_sm_id__size_1_v(); i++)
2176 gk20a_writel(g, gr_cwd_sm_id_r(i), tpc_sm_id[i]);
2177 nvgpu_kfree(g, tpc_sm_id);
2178
2179 return 0;
2180}
2181
2182int gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va)
2183{
2184 u32 addr_lo;
2185 u32 addr_hi;
2186 struct ctx_header_desc *ctx;
2187 int err;
2188
2189 gk20a_dbg_fn("");
2190
2191 err = gv11b_alloc_subctx_header(c);
2192 if (err)
2193 return err;
2194
2195 err = gv11b_update_subctx_header(c, gpu_va);
2196 if (err)
2197 return err;
2198
2199 ctx = &c->ch_ctx.ctx_header;
2200 addr_lo = u64_lo32(ctx->mem.gpu_va) >> ram_in_base_shift_v();
2201 addr_hi = u64_hi32(ctx->mem.gpu_va);
2202
2203 /* point this address to engine_wfi_ptr */
2204 nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_engine_wfi_target_w(),
2205 ram_in_engine_cs_wfi_v() |
2206 ram_in_engine_wfi_mode_f(ram_in_engine_wfi_mode_virtual_v()) |
2207 ram_in_engine_wfi_ptr_lo_f(addr_lo));
2208
2209 nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_engine_wfi_ptr_hi_w(),
2210 ram_in_engine_wfi_ptr_hi_f(addr_hi));
2211
2212 return 0;
2213}
2214
2215
2216
2217int gr_gv11b_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c)
2218{
2219 struct channel_ctx_gk20a *ch_ctx = NULL;
2220 u32 pd_ab_dist_cfg0;
2221 u32 ds_debug;
2222 u32 mpc_vtg_debug;
2223 u32 pe_vaf;
2224 u32 pe_vsc_vpc;
2225
2226 gk20a_dbg_fn("");
2227
2228 pd_ab_dist_cfg0 = gk20a_readl(g, gr_pd_ab_dist_cfg0_r());
2229 ds_debug = gk20a_readl(g, gr_ds_debug_r());
2230 mpc_vtg_debug = gk20a_readl(g, gr_gpcs_tpcs_mpc_vtg_debug_r());
2231
2232 pe_vaf = gk20a_readl(g, gr_gpcs_tpcs_pe_vaf_r());
2233 pe_vsc_vpc = gk20a_readl(g, gr_gpcs_tpcs_pes_vsc_vpc_r());
2234
2235 pe_vaf = gr_gpcs_tpcs_pe_vaf_fast_mode_switch_true_f() | pe_vaf;
2236 pe_vsc_vpc = gr_gpcs_tpcs_pes_vsc_vpc_fast_mode_switch_true_f() |
2237 pe_vsc_vpc;
2238 pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_en_f() |
2239 pd_ab_dist_cfg0;
2240 ds_debug = gr_ds_debug_timeslice_mode_enable_f() | ds_debug;
2241 mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_enabled_f() |
2242 mpc_vtg_debug;
2243
2244 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_pe_vaf_r(), pe_vaf,
2245 false);
2246 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_pes_vsc_vpc_r(),
2247 pe_vsc_vpc, false);
2248 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_pd_ab_dist_cfg0_r(),
2249 pd_ab_dist_cfg0, false);
2250 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_ds_debug_r(), ds_debug, false);
2251 gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(),
2252 mpc_vtg_debug, false);
2253
2254 return 0;
2255}
2256
2257void gr_gv11b_write_zcull_ptr(struct gk20a *g,
2258 struct nvgpu_mem *mem, u64 gpu_va)
2259{
2260 u32 va_lo, va_hi;
2261
2262 gpu_va = gpu_va >> 8;
2263 va_lo = u64_lo32(gpu_va);
2264 va_hi = u64_hi32(gpu_va);
2265 nvgpu_mem_wr(g, mem,
2266 ctxsw_prog_main_image_zcull_ptr_o(), va_lo);
2267 nvgpu_mem_wr(g, mem,
2268 ctxsw_prog_main_image_zcull_ptr_hi_o(), va_hi);
2269}
2270
2271
2272void gr_gv11b_write_pm_ptr(struct gk20a *g,
2273 struct nvgpu_mem *mem, u64 gpu_va)
2274{
2275 u32 va_lo, va_hi;
2276
2277 gpu_va = gpu_va >> 8;
2278 va_lo = u64_lo32(gpu_va);
2279 va_hi = u64_hi32(gpu_va);
2280 nvgpu_mem_wr(g, mem,
2281 ctxsw_prog_main_image_pm_ptr_o(), va_lo);
2282 nvgpu_mem_wr(g, mem,
2283 ctxsw_prog_main_image_pm_ptr_hi_o(), va_hi);
2284}
2285
2286void gr_gv11b_init_elcg_mode(struct gk20a *g, u32 mode, u32 engine)
2287{
2288 u32 gate_ctrl;
2289
2290 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_ELCG))
2291 return;
2292
2293 gate_ctrl = gk20a_readl(g, therm_gate_ctrl_r(engine));
2294
2295 switch (mode) {
2296 case ELCG_RUN:
2297 gate_ctrl = set_field(gate_ctrl,
2298 therm_gate_ctrl_eng_clk_m(),
2299 therm_gate_ctrl_eng_clk_run_f());
2300 gate_ctrl = set_field(gate_ctrl,
2301 therm_gate_ctrl_idle_holdoff_m(),
2302 therm_gate_ctrl_idle_holdoff_on_f());
2303 break;
2304 case ELCG_STOP:
2305 gate_ctrl = set_field(gate_ctrl,
2306 therm_gate_ctrl_eng_clk_m(),
2307 therm_gate_ctrl_eng_clk_stop_f());
2308 break;
2309 case ELCG_AUTO:
2310 gate_ctrl = set_field(gate_ctrl,
2311 therm_gate_ctrl_eng_clk_m(),
2312 therm_gate_ctrl_eng_clk_auto_f());
2313 break;
2314 default:
2315 nvgpu_err(g, "invalid elcg mode %d", mode);
2316 }
2317
2318 gk20a_writel(g, therm_gate_ctrl_r(engine), gate_ctrl);
2319}
2320
2321void gr_gv11b_load_tpc_mask(struct gk20a *g)
2322{
2323 u32 pes_tpc_mask = 0, fuse_tpc_mask;
2324 u32 gpc, pes, val;
2325 u32 num_tpc_per_gpc = nvgpu_get_litter_value(g,
2326 GPU_LIT_NUM_TPC_PER_GPC);
2327
2328 /* gv11b has 1 GPC and 4 TPC/GPC, so mask will not overflow u32 */
2329 for (gpc = 0; gpc < g->gr.gpc_count; gpc++) {
2330 for (pes = 0; pes < g->gr.pe_count_per_gpc; pes++) {
2331 pes_tpc_mask |= g->gr.pes_tpc_mask[pes][gpc] <<
2332 num_tpc_per_gpc * gpc;
2333 }
2334 }
2335
2336 gk20a_dbg_info("pes_tpc_mask %u\n", pes_tpc_mask);
2337 fuse_tpc_mask = g->ops.gr.get_gpc_tpc_mask(g, gpc);
2338 if (g->tpc_fs_mask_user &&
2339 g->tpc_fs_mask_user != fuse_tpc_mask &&
2340 fuse_tpc_mask == (0x1U << g->gr.max_tpc_count) - 1U) {
2341 val = g->tpc_fs_mask_user;
2342 val &= (0x1U << g->gr.max_tpc_count) - 1U;
2343 val = (0x1U << hweight32(val)) - 1U;
2344 gk20a_writel(g, gr_fe_tpc_fs_r(0), val);
2345 } else {
2346 gk20a_writel(g, gr_fe_tpc_fs_r(0), pes_tpc_mask);
2347 }
2348
2349}
2350
2351void gr_gv11b_set_preemption_buffer_va(struct gk20a *g,
2352 struct nvgpu_mem *mem, u64 gpu_va)
2353{
2354 u32 addr_lo, addr_hi;
2355
2356 addr_lo = u64_lo32(gpu_va);
2357 addr_hi = u64_hi32(gpu_va);
2358
2359 nvgpu_mem_wr(g, mem,
2360 ctxsw_prog_main_image_full_preemption_ptr_o(), addr_lo);
2361 nvgpu_mem_wr(g, mem,
2362 ctxsw_prog_main_image_full_preemption_ptr_hi_o(), addr_hi);
2363
2364 nvgpu_mem_wr(g, mem,
2365 ctxsw_prog_main_image_full_preemption_ptr_veid0_o(), addr_lo);
2366 nvgpu_mem_wr(g, mem,
2367 ctxsw_prog_main_image_full_preemption_ptr_veid0_hi_o(),
2368 addr_hi);
2369
2370}
2371
2372int gr_gv11b_init_fs_state(struct gk20a *g)
2373{
2374 u32 data;
2375
2376 gk20a_dbg_fn("");
2377
2378 data = gk20a_readl(g, gr_gpcs_tpcs_sm_texio_control_r());
2379 data = set_field(data, gr_gpcs_tpcs_sm_texio_control_oor_addr_check_mode_m(),
2380 gr_gpcs_tpcs_sm_texio_control_oor_addr_check_mode_arm_63_48_match_f());
2381 gk20a_writel(g, gr_gpcs_tpcs_sm_texio_control_r(), data);
2382
2383 data = gk20a_readl(g, gr_gpcs_tpcs_sm_disp_ctrl_r());
2384 data = set_field(data, gr_gpcs_tpcs_sm_disp_ctrl_re_suppress_m(),
2385 gr_gpcs_tpcs_sm_disp_ctrl_re_suppress_disable_f());
2386 gk20a_writel(g, gr_gpcs_tpcs_sm_disp_ctrl_r(), data);
2387
2388 if (g->gr.t18x.fecs_feature_override_ecc_val != 0) {
2389 gk20a_writel(g,
2390 gr_fecs_feature_override_ecc_r(),
2391 g->gr.t18x.fecs_feature_override_ecc_val);
2392 }
2393
2394 return gr_gm20b_init_fs_state(g);
2395}
2396
2397void gv11b_gr_get_esr_sm_sel(struct gk20a *g, u32 gpc, u32 tpc,
2398 u32 *esr_sm_sel)
2399{
2400 u32 reg_val;
2401 u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc);
2402
2403 reg_val = gk20a_readl(g, gr_gpc0_tpc0_sm_tpc_esr_sm_sel_r() + offset);
2404 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
2405 "sm tpc esr sm sel reg val: 0x%x", reg_val);
2406 *esr_sm_sel = 0;
2407 if (gr_gpc0_tpc0_sm_tpc_esr_sm_sel_sm0_error_v(reg_val))
2408 *esr_sm_sel = 1;
2409 if (gr_gpc0_tpc0_sm_tpc_esr_sm_sel_sm1_error_v(reg_val))
2410 *esr_sm_sel |= 1 << 1;
2411 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
2412 "esr_sm_sel bitmask: 0x%x", *esr_sm_sel);
2413}
2414
2415int gv11b_gr_sm_trigger_suspend(struct gk20a *g)
2416{
2417 u32 dbgr_control0;
2418
2419 /* assert stop trigger. uniformity assumption: all SMs will have
2420 * the same state in dbg_control0.
2421 */
2422 dbgr_control0 =
2423 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2424 dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f();
2425
2426 /* broadcast write */
2427 gk20a_writel(g,
2428 gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0);
2429
2430 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
2431 "stop trigger enable: broadcast dbgr_control0: 0x%x ",
2432 dbgr_control0);
2433
2434 return 0;
2435}
2436
2437void gv11b_gr_bpt_reg_info(struct gk20a *g, struct nvgpu_warpstate *w_state)
2438{
2439 /* Check if we have at least one valid warp
2440 * get paused state on maxwell
2441 */
2442 struct gr_gk20a *gr = &g->gr;
2443 u32 gpc, tpc, sm, sm_id;
2444 u32 offset;
2445 u64 warps_valid = 0, warps_paused = 0, warps_trapped = 0;
2446
2447 for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) {
2448 gpc = g->gr.sm_to_cluster[sm_id].gpc_index;
2449 tpc = g->gr.sm_to_cluster[sm_id].tpc_index;
2450 sm = g->gr.sm_to_cluster[sm_id].sm_index;
2451
2452 offset = gk20a_gr_gpc_offset(g, gpc) +
2453 gk20a_gr_tpc_offset(g, tpc) +
2454 gv11b_gr_sm_offset(g, sm);
2455
2456 /* 64 bit read */
2457 warps_valid = (u64)gk20a_readl(g,
2458 gr_gpc0_tpc0_sm0_warp_valid_mask_1_r() +
2459 offset) << 32;
2460 warps_valid |= gk20a_readl(g,
2461 gr_gpc0_tpc0_sm0_warp_valid_mask_0_r() +
2462 offset);
2463
2464 /* 64 bit read */
2465 warps_paused = (u64)gk20a_readl(g,
2466 gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_1_r() +
2467 offset) << 32;
2468 warps_paused |= gk20a_readl(g,
2469 gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_0_r() +
2470 offset);
2471
2472 /* 64 bit read */
2473 warps_trapped = (u64)gk20a_readl(g,
2474 gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_1_r() +
2475 offset) << 32;
2476 warps_trapped |= gk20a_readl(g,
2477 gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_0_r() +
2478 offset);
2479
2480 w_state[sm_id].valid_warps[0] = warps_valid;
2481 w_state[sm_id].trapped_warps[0] = warps_trapped;
2482 w_state[sm_id].paused_warps[0] = warps_paused;
2483 }
2484
2485
2486 /* Only for debug purpose */
2487 for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) {
2488 gk20a_dbg_fn("w_state[%d].valid_warps[0]: %llx\n",
2489 sm_id, w_state[sm_id].valid_warps[0]);
2490 gk20a_dbg_fn("w_state[%d].valid_warps[1]: %llx\n",
2491 sm_id, w_state[sm_id].valid_warps[1]);
2492
2493 gk20a_dbg_fn("w_state[%d].trapped_warps[0]: %llx\n",
2494 sm_id, w_state[sm_id].trapped_warps[0]);
2495 gk20a_dbg_fn("w_state[%d].trapped_warps[1]: %llx\n",
2496 sm_id, w_state[sm_id].trapped_warps[1]);
2497
2498 gk20a_dbg_fn("w_state[%d].paused_warps[0]: %llx\n",
2499 sm_id, w_state[sm_id].paused_warps[0]);
2500 gk20a_dbg_fn("w_state[%d].paused_warps[1]: %llx\n",
2501 sm_id, w_state[sm_id].paused_warps[1]);
2502 }
2503}
2504
2505int gv11b_gr_update_sm_error_state(struct gk20a *g,
2506 struct channel_gk20a *ch, u32 sm_id,
2507 struct nvgpu_gr_sm_error_state *sm_error_state)
2508{
2509 u32 gpc, tpc, sm, offset;
2510 struct gr_gk20a *gr = &g->gr;
2511 struct channel_ctx_gk20a *ch_ctx = &ch->ch_ctx;
2512 int err = 0;
2513
2514 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
2515
2516 gr->sm_error_states[sm_id].hww_global_esr =
2517 sm_error_state->hww_global_esr;
2518 gr->sm_error_states[sm_id].hww_warp_esr =
2519 sm_error_state->hww_warp_esr;
2520 gr->sm_error_states[sm_id].hww_warp_esr_pc =
2521 sm_error_state->hww_warp_esr_pc;
2522 gr->sm_error_states[sm_id].hww_global_esr_report_mask =
2523 sm_error_state->hww_global_esr_report_mask;
2524 gr->sm_error_states[sm_id].hww_warp_esr_report_mask =
2525 sm_error_state->hww_warp_esr_report_mask;
2526
2527 err = gr_gk20a_disable_ctxsw(g);
2528 if (err) {
2529 nvgpu_err(g, "unable to stop gr ctxsw");
2530 goto fail;
2531 }
2532
2533 gpc = g->gr.sm_to_cluster[sm_id].gpc_index;
2534 tpc = g->gr.sm_to_cluster[sm_id].tpc_index;
2535 sm = g->gr.sm_to_cluster[sm_id].sm_index;
2536
2537 offset = gk20a_gr_gpc_offset(g, gpc) +
2538 gk20a_gr_tpc_offset(g, tpc) +
2539 gv11b_gr_sm_offset(g, sm);
2540
2541 if (gk20a_is_channel_ctx_resident(ch)) {
2542 gk20a_writel(g,
2543 gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset,
2544 gr->sm_error_states[sm_id].hww_global_esr);
2545 gk20a_writel(g,
2546 gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset,
2547 gr->sm_error_states[sm_id].hww_warp_esr);
2548 gk20a_writel(g,
2549 gr_gpc0_tpc0_sm0_hww_warp_esr_pc_r() + offset,
2550 gr->sm_error_states[sm_id].hww_warp_esr_pc);
2551 gk20a_writel(g,
2552 gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset,
2553 gr->sm_error_states[sm_id].hww_global_esr_report_mask);
2554 gk20a_writel(g,
2555 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset,
2556 gr->sm_error_states[sm_id].hww_warp_esr_report_mask);
2557 } else {
2558 err = gr_gk20a_ctx_patch_write_begin(g, ch_ctx, false);
2559 if (err)
2560 goto enable_ctxsw;
2561
2562 gr_gk20a_ctx_patch_write(g, ch_ctx,
2563 gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r() +
2564 offset,
2565 gr->sm_error_states[sm_id].hww_global_esr_report_mask,
2566 true);
2567 gr_gk20a_ctx_patch_write(g, ch_ctx,
2568 gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r() +
2569 offset,
2570 gr->sm_error_states[sm_id].hww_warp_esr_report_mask,
2571 true);
2572
2573 gr_gk20a_ctx_patch_write_end(g, ch_ctx, false);
2574 }
2575
2576enable_ctxsw:
2577 err = gr_gk20a_enable_ctxsw(g);
2578
2579fail:
2580 nvgpu_mutex_release(&g->dbg_sessions_lock);
2581 return err;
2582}
2583
2584int gv11b_gr_set_sm_debug_mode(struct gk20a *g,
2585 struct channel_gk20a *ch, u64 sms, bool enable)
2586{
2587 struct nvgpu_dbg_gpu_reg_op *ops;
2588 unsigned int i = 0, sm_id;
2589 int err;
2590
2591 ops = nvgpu_kcalloc(g, g->gr.no_of_sm, sizeof(*ops));
2592 if (!ops)
2593 return -ENOMEM;
2594 for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) {
2595 u32 gpc, tpc, sm;
2596 u32 reg_offset, reg_mask, reg_val;
2597
2598 if (!(sms & (1 << sm_id)))
2599 continue;
2600
2601 gpc = g->gr.sm_to_cluster[sm_id].gpc_index;
2602 tpc = g->gr.sm_to_cluster[sm_id].tpc_index;
2603 sm = g->gr.sm_to_cluster[sm_id].sm_index;
2604
2605 reg_offset = gk20a_gr_gpc_offset(g, gpc) +
2606 gk20a_gr_tpc_offset(g, tpc) +
2607 gv11b_gr_sm_offset(g, sm);
2608
2609 ops[i].op = REGOP(WRITE_32);
2610 ops[i].type = REGOP(TYPE_GR_CTX);
2611 ops[i].offset = gr_gpc0_tpc0_sm0_dbgr_control0_r() + reg_offset;
2612
2613 reg_mask = 0;
2614 reg_val = 0;
2615 if (enable) {
2616 nvgpu_log(g, gpu_dbg_gpu_dbg,
2617 "SM:%d debuggger mode ON", sm);
2618 reg_mask |=
2619 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_m();
2620 reg_val |=
2621 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_on_f();
2622 } else {
2623 nvgpu_log(g, gpu_dbg_gpu_dbg,
2624 "SM:%d debuggger mode Off", sm);
2625 reg_mask |=
2626 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_m();
2627 reg_val |=
2628 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_off_f();
2629 }
2630
2631 ops[i].and_n_mask_lo = reg_mask;
2632 ops[i].value_lo = reg_val;
2633 i++;
2634 }
2635
2636 err = gr_gk20a_exec_ctx_ops(ch, ops, i, i, 0);
2637 if (err)
2638 nvgpu_err(g, "Failed to access register\n");
2639 nvgpu_kfree(g, ops);
2640 return err;
2641}
2642
2643int gv11b_gr_record_sm_error_state(struct gk20a *g, u32 gpc, u32 tpc)
2644{
2645 int sm_id;
2646 struct gr_gk20a *gr = &g->gr;
2647 u32 offset, sm, sm_per_tpc;
2648 u32 gpc_tpc_offset;
2649
2650 nvgpu_mutex_acquire(&g->dbg_sessions_lock);
2651
2652 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC);
2653 gpc_tpc_offset = gk20a_gr_gpc_offset(g, gpc) +
2654 gk20a_gr_tpc_offset(g, tpc);
2655
2656 sm_id = gr_gpc0_tpc0_sm_cfg_tpc_id_v(gk20a_readl(g,
2657 gr_gpc0_tpc0_sm_cfg_r() + gpc_tpc_offset));
2658
2659 sm = sm_id % sm_per_tpc;
2660
2661 offset = gpc_tpc_offset + gv11b_gr_sm_offset(g, sm);
2662
2663 gr->sm_error_states[sm_id].hww_global_esr = gk20a_readl(g,
2664 gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset);
2665
2666 gr->sm_error_states[sm_id].hww_warp_esr = gk20a_readl(g,
2667 gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset);
2668
2669 gr->sm_error_states[sm_id].hww_warp_esr_pc = gk20a_readl(g,
2670 gr_gpc0_tpc0_sm0_hww_warp_esr_pc_r() + offset);
2671
2672 gr->sm_error_states[sm_id].hww_global_esr_report_mask = gk20a_readl(g,
2673 gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset);
2674
2675 gr->sm_error_states[sm_id].hww_warp_esr_report_mask = gk20a_readl(g,
2676 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset);
2677
2678 nvgpu_mutex_release(&g->dbg_sessions_lock);
2679
2680 return 0;
2681}
2682
2683void gv11b_gr_set_hww_esr_report_mask(struct gk20a *g)
2684{
2685
2686 /* clear hww */
2687 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_r(), 0xffffffff);
2688 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_r(), 0xffffffff);
2689
2690 /* setup sm warp esr report masks */
2691 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r(),
2692 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_stack_error_report_f() |
2693 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_api_stack_error_report_f() |
2694 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_pc_wrap_report_f() |
2695 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_pc_report_f() |
2696 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_pc_overflow_report_f() |
2697 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_reg_report_f() |
2698 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_illegal_instr_encoding_report_f() |
2699 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_illegal_instr_param_report_f() |
2700 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_oor_reg_report_f() |
2701 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_oor_addr_report_f() |
2702 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_addr_report_f() |
2703 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_invalid_addr_space_report_f() |
2704 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_invalid_const_addr_ldc_report_f() |
2705 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_stack_overflow_report_f() |
2706 gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_mmu_fault_report_f());
2707
2708 /* setup sm global esr report mask. vat_alarm_report is not enabled */
2709 gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r(),
2710 gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_multiple_warp_errors_report_f());
2711}
2712
2713bool gv11b_gr_sm_debugger_attached(struct gk20a *g)
2714{
2715 u32 debugger_mode;
2716 u32 dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2717
2718 /* check if sm debugger is attached.
2719 * assumption: all SMs will have debug mode enabled/disabled
2720 * uniformly.
2721 */
2722 debugger_mode =
2723 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_v(dbgr_control0);
2724 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
2725 "SM Debugger Mode: %d", debugger_mode);
2726 if (debugger_mode ==
2727 gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_on_v())
2728 return true;
2729
2730 return false;
2731}
2732
2733void gv11b_gr_suspend_single_sm(struct gk20a *g,
2734 u32 gpc, u32 tpc, u32 sm,
2735 u32 global_esr_mask, bool check_errors)
2736{
2737 int err;
2738 u32 dbgr_control0;
2739 u32 offset = gk20a_gr_gpc_offset(g, gpc) +
2740 gk20a_gr_tpc_offset(g, tpc) +
2741 gv11b_gr_sm_offset(g, sm);
2742
2743 /* if an SM debugger isn't attached, skip suspend */
2744 if (!g->ops.gr.sm_debugger_attached(g)) {
2745 nvgpu_err(g,
2746 "SM debugger not attached, skipping suspend!");
2747 return;
2748 }
2749
2750 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2751 "suspending gpc:%d, tpc:%d, sm%d", gpc, tpc, sm);
2752
2753 /* assert stop trigger. */
2754 dbgr_control0 = gk20a_readl(g,
2755 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
2756 dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f();
2757 gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset,
2758 dbgr_control0);
2759
2760 err = g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm,
2761 global_esr_mask, check_errors);
2762 if (err) {
2763 nvgpu_err(g,
2764 "SuspendSm failed");
2765 return;
2766 }
2767}
2768
2769void gv11b_gr_suspend_all_sms(struct gk20a *g,
2770 u32 global_esr_mask, bool check_errors)
2771{
2772 struct gr_gk20a *gr = &g->gr;
2773 u32 gpc, tpc, sm;
2774 int err;
2775 u32 dbgr_control0;
2776 u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC);
2777
2778 /* if an SM debugger isn't attached, skip suspend */
2779 if (!g->ops.gr.sm_debugger_attached(g)) {
2780 nvgpu_err(g,
2781 "SM debugger not attached, skipping suspend!");
2782 return;
2783 }
2784
2785 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "suspending all sms");
2786
2787 /* assert stop trigger. uniformity assumption: all SMs will have
2788 * the same state in dbg_control0.
2789 */
2790 dbgr_control0 =
2791 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2792 dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f();
2793
2794 /* broadcast write */
2795 gk20a_writel(g,
2796 gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0);
2797
2798 for (gpc = 0; gpc < gr->gpc_count; gpc++) {
2799 for (tpc = 0; tpc < gr_gk20a_get_tpc_count(gr, gpc); tpc++) {
2800 for (sm = 0; sm < sm_per_tpc; sm++) {
2801 err = g->ops.gr.wait_for_sm_lock_down(g,
2802 gpc, tpc, sm,
2803 global_esr_mask, check_errors);
2804 if (err) {
2805 nvgpu_err(g,
2806 "SuspendAllSms failed");
2807 return;
2808 }
2809 }
2810 }
2811 }
2812}
2813
2814void gv11b_gr_resume_single_sm(struct gk20a *g,
2815 u32 gpc, u32 tpc, u32 sm)
2816{
2817 u32 dbgr_control0, dbgr_status0;
2818 u32 offset;
2819 /*
2820 * The following requires some clarification. Despite the fact that both
2821 * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their
2822 * names, only one is actually a trigger, and that is the STOP_TRIGGER.
2823 * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to
2824 * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0
2825 * (_DISABLE) as well.
2826
2827 * Advice from the arch group: Disable the stop trigger first, as a
2828 * separate operation, in order to ensure that the trigger has taken
2829 * effect, before enabling the run trigger.
2830 */
2831
2832 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) +
2833 gv11b_gr_sm_offset(g, sm);
2834
2835 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2836 "resuming gpc:%d, tpc:%d, sm%d", gpc, tpc, sm);
2837 dbgr_control0 = gk20a_readl(g,
2838 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
2839 dbgr_status0 = gk20a_readl(g,
2840 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset);
2841
2842 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2843 "before stop trigger disable: "
2844 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2845 dbgr_control0, dbgr_status0);
2846
2847 /*De-assert stop trigger */
2848 dbgr_control0 = set_field(dbgr_control0,
2849 gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_m(),
2850 gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_disable_f());
2851 gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() +
2852 offset, dbgr_control0);
2853
2854 dbgr_control0 = gk20a_readl(g,
2855 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
2856 dbgr_status0 = gk20a_readl(g,
2857 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset);
2858
2859 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2860 "before run trigger: "
2861 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2862 dbgr_control0, dbgr_status0);
2863 /* Run trigger */
2864 dbgr_control0 |=
2865 gr_gpc0_tpc0_sm0_dbgr_control0_run_trigger_task_f();
2866 gk20a_writel(g,
2867 gr_gpc0_tpc0_sm0_dbgr_control0_r() +
2868 offset, dbgr_control0);
2869
2870 dbgr_control0 = gk20a_readl(g,
2871 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
2872 dbgr_status0 = gk20a_readl(g,
2873 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset);
2874 /* run trigger is not sticky bit. SM clears it immediately */
2875 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2876 "after run trigger: "
2877 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2878 dbgr_control0, dbgr_status0);
2879
2880}
2881
2882void gv11b_gr_resume_all_sms(struct gk20a *g)
2883{
2884 u32 dbgr_control0, dbgr_status0;
2885 /*
2886 * The following requires some clarification. Despite the fact that both
2887 * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their
2888 * names, only one is actually a trigger, and that is the STOP_TRIGGER.
2889 * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to
2890 * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0
2891 * (_DISABLE) as well.
2892
2893 * Advice from the arch group: Disable the stop trigger first, as a
2894 * separate operation, in order to ensure that the trigger has taken
2895 * effect, before enabling the run trigger.
2896 */
2897
2898 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "resuming all sms");
2899
2900 /* Read from unicast registers */
2901 dbgr_control0 =
2902 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2903 dbgr_status0 =
2904 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r());
2905
2906 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2907 "before stop trigger disable: "
2908 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2909 dbgr_control0, dbgr_status0);
2910
2911 dbgr_control0 = set_field(dbgr_control0,
2912 gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_m(),
2913 gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_disable_f());
2914 /* Write to broadcast registers */
2915 gk20a_writel(g,
2916 gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0);
2917
2918 /* Read from unicast registers */
2919 dbgr_control0 =
2920 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2921 dbgr_status0 =
2922 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r());
2923
2924 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2925 "before run trigger: "
2926 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2927 dbgr_control0, dbgr_status0);
2928 /* Run trigger */
2929 dbgr_control0 |=
2930 gr_gpc0_tpc0_sm0_dbgr_control0_run_trigger_task_f();
2931 /* Write to broadcast registers */
2932 gk20a_writel(g,
2933 gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0);
2934
2935 /* Read from unicast registers */
2936 dbgr_control0 =
2937 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r());
2938 dbgr_status0 =
2939 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r());
2940 /* run trigger is not sticky bit. SM clears it immediately */
2941 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
2942 "after run trigger: "
2943 "dbgr_control0 = 0x%x dbgr_status0: 0x%x",
2944 dbgr_control0, dbgr_status0);
2945}
2946
2947int gv11b_gr_resume_from_pause(struct gk20a *g)
2948{
2949 int err = 0;
2950 u32 reg_val;
2951
2952 /* Clear the pause mask to tell the GPU we want to resume everyone */
2953 gk20a_writel(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_0_r(), 0);
2954
2955 /* explicitly re-enable forwarding of SM interrupts upon any resume */
2956 reg_val = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r());
2957 reg_val |= gr_gpc0_tpc0_tpccs_tpc_exception_en_sm_enabled_f();
2958
2959 gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), reg_val);
2960
2961 g->ops.gr.resume_all_sms(g);
2962
2963 return err;
2964}
2965
2966u32 gv11b_gr_get_sm_hww_warp_esr(struct gk20a *g,
2967 u32 gpc, u32 tpc, u32 sm)
2968{
2969 u32 offset = gk20a_gr_gpc_offset(g, gpc) +
2970 gk20a_gr_tpc_offset(g, tpc) +
2971 gv11b_gr_sm_offset(g, sm);
2972
2973 u32 hww_warp_esr = gk20a_readl(g,
2974 gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset);
2975 return hww_warp_esr;
2976}
2977
2978u32 gv11b_gr_get_sm_hww_global_esr(struct gk20a *g,
2979 u32 gpc, u32 tpc, u32 sm)
2980{
2981 u32 offset = gk20a_gr_gpc_offset(g, gpc) +
2982 gk20a_gr_tpc_offset(g, tpc) +
2983 gv11b_gr_sm_offset(g, sm);
2984
2985 u32 hww_global_esr = gk20a_readl(g,
2986 gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset);
2987
2988 return hww_global_esr;
2989}
2990
2991u32 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g)
2992{
2993 /*
2994 * These three interrupts don't require locking down the SM. They can
2995 * be handled by usermode clients as they aren't fatal. Additionally,
2996 * usermode clients may wish to allow some warps to execute while others
2997 * are at breakpoints, as opposed to fatal errors where all warps should
2998 * halt.
2999 */
3000 u32 global_esr_mask =
3001 gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f() |
3002 gr_gpc0_tpc0_sm0_hww_global_esr_bpt_pause_pending_f() |
3003 gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f();
3004
3005 return global_esr_mask;
3006}
3007
3008static void gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(struct gk20a *g,
3009 u32 offset, bool timeout)
3010{
3011 u64 warps_valid = 0, warps_paused = 0, warps_trapped = 0;
3012 u32 dbgr_control0 = gk20a_readl(g,
3013 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
3014 u32 dbgr_status0 = gk20a_readl(g,
3015 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset);
3016 /* 64 bit read */
3017 warps_valid =
3018 (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_warp_valid_mask_1_r() +
3019 offset) << 32;
3020 warps_valid |= gk20a_readl(g,
3021 gr_gpc0_tpc0_sm0_warp_valid_mask_0_r() + offset);
3022
3023 /* 64 bit read */
3024 warps_paused =
3025 (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_1_r() +
3026 offset) << 32;
3027 warps_paused |= gk20a_readl(g,
3028 gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_0_r() + offset);
3029
3030 /* 64 bit read */
3031 warps_trapped =
3032 (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_1_r() +
3033 offset) << 32;
3034 warps_trapped |= gk20a_readl(g,
3035 gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_0_r() + offset);
3036 if (timeout)
3037 nvgpu_err(g,
3038 "STATUS0=0x%x CONTROL0=0x%x VALID_MASK=0x%llx "
3039 "PAUSE_MASK=0x%llx TRAP_MASK=0x%llx\n",
3040 dbgr_status0, dbgr_control0, warps_valid,
3041 warps_paused, warps_trapped);
3042 else
3043 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
3044 "STATUS0=0x%x CONTROL0=0x%x VALID_MASK=0x%llx "
3045 "PAUSE_MASK=0x%llx TRAP_MASK=0x%llx\n",
3046 dbgr_status0, dbgr_control0, warps_valid,
3047 warps_paused, warps_trapped);
3048}
3049
3050int gv11b_gr_wait_for_sm_lock_down(struct gk20a *g,
3051 u32 gpc, u32 tpc, u32 sm,
3052 u32 global_esr_mask, bool check_errors)
3053{
3054 bool locked_down;
3055 bool no_error_pending;
3056 u32 delay = GR_IDLE_CHECK_DEFAULT;
3057 bool mmu_debug_mode_enabled = g->ops.fb.is_debug_mode_enabled(g);
3058 u32 dbgr_status0 = 0;
3059 u32 warp_esr, global_esr;
3060 struct nvgpu_timeout timeout;
3061 u32 offset = gk20a_gr_gpc_offset(g, gpc) +
3062 gk20a_gr_tpc_offset(g, tpc) +
3063 gv11b_gr_sm_offset(g, sm);
3064
3065 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
3066 "GPC%d TPC%d: locking down SM%d", gpc, tpc, sm);
3067
3068 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
3069 NVGPU_TIMER_CPU_TIMER);
3070
3071 /* wait for the sm to lock down */
3072 do {
3073 global_esr = g->ops.gr.get_sm_hww_global_esr(g, gpc, tpc, sm);
3074 dbgr_status0 = gk20a_readl(g,
3075 gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset);
3076
3077 warp_esr = g->ops.gr.get_sm_hww_warp_esr(g, gpc, tpc, sm);
3078
3079 locked_down =
3080 (gr_gpc0_tpc0_sm0_dbgr_status0_locked_down_v(dbgr_status0) ==
3081 gr_gpc0_tpc0_sm0_dbgr_status0_locked_down_true_v());
3082 no_error_pending =
3083 check_errors &&
3084 (gr_gpc0_tpc0_sm0_hww_warp_esr_error_v(warp_esr) ==
3085 gr_gpc0_tpc0_sm0_hww_warp_esr_error_none_v()) &&
3086 ((global_esr & global_esr_mask) == 0);
3087
3088 if (locked_down) {
3089 /*
3090 * if SM reports locked down, it means that SM is idle and
3091 * trapped and also that one of the these conditions are true
3092 * 1) sm is nonempty and all valid warps are paused
3093 * 2) sm is empty and held in trapped state due to stop trigger
3094 * 3) sm is nonempty and some warps are not paused, but are
3095 * instead held at RTT due to an "active" stop trigger
3096 * Check for Paused warp mask != Valid
3097 * warp mask after SM reports it is locked down in order to
3098 * distinguish case 1 from case 3. When case 3 is detected,
3099 * it implies a misprogrammed trap handler code, as all warps
3100 * in the handler must promise to BPT.PAUSE instead of RTT
3101 * whenever SR64 read in trap mode indicates stop trigger
3102 * is asserted.
3103 */
3104 gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(g,
3105 offset, false);
3106 }
3107
3108 if (locked_down || no_error_pending) {
3109 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
3110 "GPC%d TPC%d: locked down SM%d", gpc, tpc, sm);
3111 return 0;
3112 }
3113
3114 /* if an mmu fault is pending and mmu debug mode is not
3115 * enabled, the sm will never lock down.
3116 */
3117 if (!mmu_debug_mode_enabled &&
3118 (g->ops.mm.mmu_fault_pending(g))) {
3119 nvgpu_err(g,
3120 "GPC%d TPC%d: mmu fault pending,"
3121 " SM%d will never lock down!", gpc, tpc, sm);
3122 return -EFAULT;
3123 }
3124
3125 nvgpu_usleep_range(delay, delay * 2);
3126 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
3127 } while (!nvgpu_timeout_expired(&timeout));
3128
3129 nvgpu_err(g, "GPC%d TPC%d: timed out while trying to "
3130 "lock down SM%d", gpc, tpc, sm);
3131 gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(g, offset, true);
3132
3133 return -ETIMEDOUT;
3134}
3135
3136int gv11b_gr_lock_down_sm(struct gk20a *g,
3137 u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask,
3138 bool check_errors)
3139{
3140 u32 dbgr_control0;
3141 u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) +
3142 gv11b_gr_sm_offset(g, sm);
3143
3144 gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg,
3145 "GPC%d TPC%d SM%d: assert stop trigger", gpc, tpc, sm);
3146
3147 /* assert stop trigger */
3148 dbgr_control0 =
3149 gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset);
3150 dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f();
3151 gk20a_writel(g,
3152 gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset, dbgr_control0);
3153
3154 return g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, global_esr_mask,
3155 check_errors);
3156}
3157
3158void gv11b_gr_clear_sm_hww(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
3159 u32 global_esr)
3160{
3161 u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) +
3162 gv11b_gr_sm_offset(g, sm);
3163
3164 gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset,
3165 global_esr);
3166 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
3167 "Cleared HWW global esr, current reg val: 0x%x",
3168 gk20a_readl(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() +
3169 offset));
3170
3171 gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset, 0);
3172 gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg,
3173 "Cleared HWW warp esr, current reg val: 0x%x",
3174 gk20a_readl(g, gr_gpc0_tpc0_sm0_hww_warp_esr_r() +
3175 offset));
3176}
3177
3178int gr_gv11b_handle_tpc_mpc_exception(struct gk20a *g,
3179 u32 gpc, u32 tpc, bool *post_event)
3180{
3181 u32 esr;
3182 u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc);
3183 u32 tpc_exception = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_r()
3184 + offset);
3185
3186 if (!(tpc_exception & gr_gpc0_tpc0_tpccs_tpc_exception_mpc_m()))
3187 return 0;
3188
3189 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg,
3190 "GPC%d TPC%d MPC exception", gpc, tpc);
3191
3192 esr = gk20a_readl(g, gr_gpc0_tpc0_mpc_hww_esr_r() + offset);
3193 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "mpc hww esr 0x%08x", esr);
3194
3195 esr = gk20a_readl(g, gr_gpc0_tpc0_mpc_hww_esr_info_r() + offset);
3196 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg,
3197 "mpc hww esr info: veid 0x%08x",
3198 gr_gpc0_tpc0_mpc_hww_esr_info_veid_v(esr));
3199
3200 gk20a_writel(g, gr_gpc0_tpc0_mpc_hww_esr_r() + offset,
3201 gr_gpc0_tpc0_mpc_hww_esr_reset_trigger_f());
3202
3203 return 0;
3204}
3205
3206static const u32 _num_ovr_perf_regs = 20;
3207static u32 _ovr_perf_regs[20] = { 0, };
3208
3209void gv11b_gr_init_ovr_sm_dsm_perf(void)
3210{
3211 if (_ovr_perf_regs[0] != 0)
3212 return;
3213
3214 _ovr_perf_regs[0] = gr_egpc0_etpc0_sm_dsm_perf_counter_control_sel0_r();
3215 _ovr_perf_regs[1] = gr_egpc0_etpc0_sm_dsm_perf_counter_control_sel1_r();
3216 _ovr_perf_regs[2] = gr_egpc0_etpc0_sm_dsm_perf_counter_control0_r();
3217 _ovr_perf_regs[3] = gr_egpc0_etpc0_sm_dsm_perf_counter_control1_r();
3218 _ovr_perf_regs[4] = gr_egpc0_etpc0_sm_dsm_perf_counter_control2_r();
3219 _ovr_perf_regs[5] = gr_egpc0_etpc0_sm_dsm_perf_counter_control3_r();
3220 _ovr_perf_regs[6] = gr_egpc0_etpc0_sm_dsm_perf_counter_control4_r();
3221 _ovr_perf_regs[7] = gr_egpc0_etpc0_sm_dsm_perf_counter_control5_r();
3222 _ovr_perf_regs[8] = gr_egpc0_etpc0_sm_dsm_perf_counter0_control_r();
3223 _ovr_perf_regs[9] = gr_egpc0_etpc0_sm_dsm_perf_counter1_control_r();
3224 _ovr_perf_regs[10] = gr_egpc0_etpc0_sm_dsm_perf_counter2_control_r();
3225 _ovr_perf_regs[11] = gr_egpc0_etpc0_sm_dsm_perf_counter3_control_r();
3226 _ovr_perf_regs[12] = gr_egpc0_etpc0_sm_dsm_perf_counter4_control_r();
3227 _ovr_perf_regs[13] = gr_egpc0_etpc0_sm_dsm_perf_counter5_control_r();
3228 _ovr_perf_regs[14] = gr_egpc0_etpc0_sm_dsm_perf_counter6_control_r();
3229 _ovr_perf_regs[15] = gr_egpc0_etpc0_sm_dsm_perf_counter7_control_r();
3230
3231 _ovr_perf_regs[16] = gr_egpc0_etpc0_sm0_dsm_perf_counter4_r();
3232 _ovr_perf_regs[17] = gr_egpc0_etpc0_sm0_dsm_perf_counter5_r();
3233 _ovr_perf_regs[18] = gr_egpc0_etpc0_sm0_dsm_perf_counter6_r();
3234 _ovr_perf_regs[19] = gr_egpc0_etpc0_sm0_dsm_perf_counter7_r();
3235}
3236
3237/* Following are the blocks of registers that the ucode
3238 * stores in the extended region.
3239 */
3240/* == ctxsw_extended_sm_dsm_perf_counter_register_stride_v() ? */
3241static const u32 _num_sm_dsm_perf_regs;
3242/* == ctxsw_extended_sm_dsm_perf_counter_control_register_stride_v() ?*/
3243static const u32 _num_sm_dsm_perf_ctrl_regs = 2;
3244static u32 *_sm_dsm_perf_regs;
3245static u32 _sm_dsm_perf_ctrl_regs[2];
3246
3247void gv11b_gr_init_sm_dsm_reg_info(void)
3248{
3249 if (_sm_dsm_perf_ctrl_regs[0] != 0)
3250 return;
3251
3252 _sm_dsm_perf_ctrl_regs[0] =
3253 gr_egpc0_etpc0_sm_dsm_perf_counter_control0_r();
3254 _sm_dsm_perf_ctrl_regs[1] =
3255 gr_egpc0_etpc0_sm_dsm_perf_counter_control5_r();
3256}
3257
3258void gv11b_gr_get_sm_dsm_perf_regs(struct gk20a *g,
3259 u32 *num_sm_dsm_perf_regs,
3260 u32 **sm_dsm_perf_regs,
3261 u32 *perf_register_stride)
3262{
3263 *num_sm_dsm_perf_regs = _num_sm_dsm_perf_regs;
3264 *sm_dsm_perf_regs = _sm_dsm_perf_regs;
3265 *perf_register_stride =
3266 ctxsw_prog_extended_sm_dsm_perf_counter_register_stride_v();
3267}
3268
3269void gv11b_gr_get_sm_dsm_perf_ctrl_regs(struct gk20a *g,
3270 u32 *num_sm_dsm_perf_ctrl_regs,
3271 u32 **sm_dsm_perf_ctrl_regs,
3272 u32 *ctrl_register_stride)
3273{
3274 *num_sm_dsm_perf_ctrl_regs = _num_sm_dsm_perf_ctrl_regs;
3275 *sm_dsm_perf_ctrl_regs = _sm_dsm_perf_ctrl_regs;
3276 *ctrl_register_stride =
3277 ctxsw_prog_extended_sm_dsm_perf_counter_control_register_stride_v();
3278}
3279
3280void gv11b_gr_get_ovr_perf_regs(struct gk20a *g, u32 *num_ovr_perf_regs,
3281 u32 **ovr_perf_regs)
3282{
3283 *num_ovr_perf_regs = _num_ovr_perf_regs;
3284 *ovr_perf_regs = _ovr_perf_regs;
3285}
3286
3287void gv11b_gr_access_smpc_reg(struct gk20a *g, u32 quad, u32 offset)
3288{
3289 u32 reg_val;
3290 u32 quad_ctrl;
3291 u32 half_ctrl;
3292 u32 tpc, gpc;
3293 u32 gpc_tpc_addr;
3294 u32 gpc_tpc_stride;
3295 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3296 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g,
3297 GPU_LIT_TPC_IN_GPC_STRIDE);
3298
3299 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "offset=0x%x", offset);
3300
3301 gpc = pri_get_gpc_num(g, offset);
3302 gpc_tpc_addr = pri_gpccs_addr_mask(offset);
3303 tpc = g->ops.gr.get_tpc_num(g, gpc_tpc_addr);
3304
3305 quad_ctrl = quad & 0x1; /* first bit tells us quad */
3306 half_ctrl = (quad >> 1) & 0x1; /* second bit tells us half */
3307
3308 gpc_tpc_stride = gpc * gpc_stride + tpc * tpc_in_gpc_stride;
3309 gpc_tpc_addr = gr_gpc0_tpc0_sm_halfctl_ctrl_r() + gpc_tpc_stride;
3310
3311 /* read from unicast reg */
3312 reg_val = gk20a_readl(g, gpc_tpc_addr);
3313 reg_val = set_field(reg_val,
3314 gr_gpcs_tpcs_sm_halfctl_ctrl_sctl_read_quad_ctl_m(),
3315 gr_gpcs_tpcs_sm_halfctl_ctrl_sctl_read_quad_ctl_f(quad_ctrl));
3316
3317 /* write to broadcast reg */
3318 gk20a_writel(g, gr_gpcs_tpcs_sm_halfctl_ctrl_r(), reg_val);
3319
3320 gpc_tpc_addr = gr_gpc0_tpc0_sm_debug_sfe_control_r() + gpc_tpc_stride;
3321 reg_val = gk20a_readl(g, gpc_tpc_addr);
3322 reg_val = set_field(reg_val,
3323 gr_gpcs_tpcs_sm_debug_sfe_control_read_half_ctl_m(),
3324 gr_gpcs_tpcs_sm_debug_sfe_control_read_half_ctl_f(half_ctrl));
3325
3326 /* write to broadcast reg */
3327 gk20a_writel(g, gr_gpcs_tpcs_sm_debug_sfe_control_r(), reg_val);
3328}
3329
3330static bool pri_is_egpc_addr_shared(struct gk20a *g, u32 addr)
3331{
3332 u32 egpc_shared_base = EGPC_PRI_SHARED_BASE;
3333 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3334
3335 return (addr >= egpc_shared_base) &&
3336 (addr < egpc_shared_base + gpc_stride);
3337}
3338
3339bool gv11b_gr_pri_is_egpc_addr(struct gk20a *g, u32 addr)
3340{
3341 u32 egpc_base = g->ops.gr.get_egpc_base(g);
3342 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3343 u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS);
3344
3345 return ((addr >= egpc_base) &&
3346 (addr < egpc_base + num_gpcs * gpc_stride)) ||
3347 pri_is_egpc_addr_shared(g, addr);
3348}
3349
3350static inline u32 pri_smpc_in_etpc_addr_mask(struct gk20a *g, u32 addr)
3351{
3352 u32 smpc_stride = nvgpu_get_litter_value(g,
3353 GPU_LIT_SMPC_PRI_STRIDE);
3354
3355 return (addr & (smpc_stride - 1));
3356}
3357
3358static u32 pri_smpc_ext_addr(struct gk20a *g, u32 sm_offset, u32 gpc_num,
3359 u32 tpc_num, u32 sm_num)
3360{
3361 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3362 u32 tpc_in_gpc_base = nvgpu_get_litter_value(g,
3363 GPU_LIT_TPC_IN_GPC_BASE);
3364 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g,
3365 GPU_LIT_TPC_IN_GPC_STRIDE);
3366 u32 egpc_base = g->ops.gr.get_egpc_base(g);
3367 u32 smpc_unique_base = nvgpu_get_litter_value(g,
3368 GPU_LIT_SMPC_PRI_UNIQUE_BASE);
3369 u32 smpc_stride = nvgpu_get_litter_value(g,
3370 GPU_LIT_SMPC_PRI_STRIDE);
3371
3372 return (egpc_base + (gpc_num * gpc_stride) + tpc_in_gpc_base +
3373 (tpc_num * tpc_in_gpc_stride) +
3374 (sm_num * smpc_stride) +
3375 (smpc_unique_base + sm_offset));
3376}
3377
3378static bool pri_is_smpc_addr_in_etpc_shared(struct gk20a *g, u32 addr)
3379{
3380 u32 smpc_shared_base = nvgpu_get_litter_value(g,
3381 GPU_LIT_SMPC_PRI_SHARED_BASE);
3382 u32 smpc_stride = nvgpu_get_litter_value(g,
3383 GPU_LIT_SMPC_PRI_STRIDE);
3384
3385 return (addr >= smpc_shared_base) &&
3386 (addr < smpc_shared_base + smpc_stride);
3387}
3388
3389bool gv11b_gr_pri_is_etpc_addr(struct gk20a *g, u32 addr)
3390{
3391 u32 egpc_addr = 0;
3392
3393 if (g->ops.gr.is_egpc_addr(g, addr)) {
3394 egpc_addr = pri_gpccs_addr_mask(addr);
3395 if (g->ops.gr.is_tpc_addr(g, egpc_addr))
3396 return true;
3397 }
3398
3399 return false;
3400}
3401
3402static u32 pri_get_egpc_num(struct gk20a *g, u32 addr)
3403{
3404 u32 i, start;
3405 u32 egpc_base = g->ops.gr.get_egpc_base(g);
3406 u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS);
3407 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3408
3409 for (i = 0; i < num_gpcs; i++) {
3410 start = egpc_base + (i * gpc_stride);
3411 if ((addr >= start) && (addr < (start + gpc_stride)))
3412 return i;
3413 }
3414 return 0;
3415}
3416
3417static u32 pri_egpc_addr(struct gk20a *g, u32 addr, u32 gpc)
3418{
3419 u32 egpc_base = g->ops.gr.get_egpc_base(g);
3420 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3421
3422 return egpc_base + (gpc * gpc_stride) + addr;
3423}
3424
3425static u32 pri_etpc_addr(struct gk20a *g, u32 addr, u32 gpc, u32 tpc)
3426{
3427 u32 egpc_base = g->ops.gr.get_egpc_base(g);
3428 u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE);
3429 u32 tpc_in_gpc_base = nvgpu_get_litter_value(g,
3430 GPU_LIT_TPC_IN_GPC_BASE);
3431 u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g,
3432 GPU_LIT_TPC_IN_GPC_STRIDE);
3433
3434 return egpc_base + (gpc * gpc_stride) +
3435 tpc_in_gpc_base + (tpc * tpc_in_gpc_stride) +
3436 addr;
3437}
3438
3439void gv11b_gr_get_egpc_etpc_num(struct gk20a *g, u32 addr,
3440 u32 *egpc_num, u32 *etpc_num)
3441{
3442 u32 egpc_addr = 0;
3443
3444 *egpc_num = pri_get_egpc_num(g, addr);
3445 egpc_addr = pri_gpccs_addr_mask(addr);
3446 *etpc_num = g->ops.gr.get_tpc_num(g, egpc_addr);
3447
3448 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
3449 "egpc_num = %d etpc_num = %d", *egpc_num, *etpc_num);
3450}
3451
3452int gv11b_gr_decode_egpc_addr(struct gk20a *g, u32 addr, int *addr_type,
3453 u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags)
3454{
3455 u32 gpc_addr;
3456 u32 tpc_addr;
3457
3458 if (g->ops.gr.is_egpc_addr(g, addr)) {
3459 nvgpu_log_info(g, "addr=0x%x is egpc", addr);
3460
3461 *addr_type = CTXSW_ADDR_TYPE_EGPC;
3462 gpc_addr = pri_gpccs_addr_mask(addr);
3463 if (pri_is_egpc_addr_shared(g, addr)) {
3464 *broadcast_flags |= PRI_BROADCAST_FLAGS_EGPC;
3465 *gpc_num = 0;
3466 nvgpu_log_info(g, "shared egpc");
3467 } else {
3468 *gpc_num = pri_get_egpc_num(g, addr);
3469 nvgpu_log_info(g, "gpc=0x%x", *gpc_num);
3470 }
3471 if (g->ops.gr.is_tpc_addr(g, gpc_addr)) {
3472 nvgpu_log_info(g, "addr=0x%x is etpc", addr);
3473 *addr_type = CTXSW_ADDR_TYPE_ETPC;
3474 if (pri_is_tpc_addr_shared(g, gpc_addr)) {
3475 *broadcast_flags |= PRI_BROADCAST_FLAGS_ETPC;
3476 *tpc_num = 0;
3477 nvgpu_log_info(g, "shared etpc");
3478 } else {
3479 *tpc_num = g->ops.gr.get_tpc_num(g, gpc_addr);
3480 nvgpu_log_info(g, "tpc=0x%x", *tpc_num);
3481 }
3482 tpc_addr = pri_tpccs_addr_mask(addr);
3483 if (pri_is_smpc_addr_in_etpc_shared(g, tpc_addr))
3484 *broadcast_flags |= PRI_BROADCAST_FLAGS_SMPC;
3485 }
3486
3487 nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg,
3488 "addr_type = %d, broadcast_flags = %#08x",
3489 *addr_type, *broadcast_flags);
3490 return 0;
3491 }
3492 return -EINVAL;
3493}
3494
3495static void gv11b_gr_update_priv_addr_table_smpc(struct gk20a *g, u32 gpc_num,
3496 u32 tpc_num, u32 addr,
3497 u32 *priv_addr_table, u32 *t)
3498{
3499 u32 sm_per_tpc, sm_num;
3500
3501 nvgpu_log_info(g, "broadcast flags smpc");
3502
3503 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC);
3504 for (sm_num = 0; sm_num < sm_per_tpc; sm_num++) {
3505 priv_addr_table[*t] = pri_smpc_ext_addr(g,
3506 pri_smpc_in_etpc_addr_mask(g, addr),
3507 gpc_num, tpc_num, sm_num);
3508 nvgpu_log_info(g, "priv_addr_table[%d]:%#08x",
3509 *t, priv_addr_table[*t]);
3510 (*t)++;
3511 }
3512}
3513
3514void gv11b_gr_egpc_etpc_priv_addr_table(struct gk20a *g, u32 addr,
3515 u32 gpc, u32 broadcast_flags, u32 *priv_addr_table, u32 *t)
3516{
3517 u32 gpc_num, tpc_num;
3518
3519 nvgpu_log_info(g, "addr=0x%x", addr);
3520
3521 /* The GPC/TPC unicast registers are included in the compressed PRI
3522 * tables. Convert a GPC/TPC broadcast address to unicast addresses so
3523 * that we can look up the offsets.
3524 */
3525 if (broadcast_flags & PRI_BROADCAST_FLAGS_EGPC) {
3526 nvgpu_log_info(g, "broadcast flags egpc");
3527 for (gpc_num = 0; gpc_num < g->gr.gpc_count; gpc_num++) {
3528
3529 if (broadcast_flags & PRI_BROADCAST_FLAGS_ETPC) {
3530 nvgpu_log_info(g, "broadcast flags etpc");
3531 for (tpc_num = 0;
3532 tpc_num < g->gr.gpc_tpc_count[gpc_num];
3533 tpc_num++) {
3534 if (broadcast_flags &
3535 PRI_BROADCAST_FLAGS_SMPC) {
3536 gv11b_gr_update_priv_addr_table_smpc(
3537 g, gpc_num, tpc_num, addr,
3538 priv_addr_table, t);
3539 } else {
3540 priv_addr_table[*t] =
3541 pri_etpc_addr(g,
3542 pri_tpccs_addr_mask(addr),
3543 gpc_num, tpc_num);
3544 nvgpu_log_info(g,
3545 "priv_addr_table[%d]:%#08x",
3546 *t, priv_addr_table[*t]);
3547 (*t)++;
3548 }
3549 }
3550 } else if (broadcast_flags & PRI_BROADCAST_FLAGS_SMPC) {
3551 tpc_num = 0;
3552 gv11b_gr_update_priv_addr_table_smpc(
3553 g, gpc_num, tpc_num, addr,
3554 priv_addr_table, t);
3555 } else {
3556 priv_addr_table[*t] =
3557 pri_egpc_addr(g,
3558 pri_gpccs_addr_mask(addr),
3559 gpc_num);
3560 nvgpu_log_info(g, "priv_addr_table[%d]:%#08x",
3561 *t, priv_addr_table[*t]);
3562 (*t)++;
3563 }
3564 }
3565 } else if (!(broadcast_flags & PRI_BROADCAST_FLAGS_EGPC)) {
3566 if (broadcast_flags & PRI_BROADCAST_FLAGS_ETPC) {
3567 nvgpu_log_info(g, "broadcast flags etpc but not egpc");
3568 gpc_num = 0;
3569 for (tpc_num = 0;
3570 tpc_num < g->gr.gpc_tpc_count[gpc];
3571 tpc_num++) {
3572 if (broadcast_flags &
3573 PRI_BROADCAST_FLAGS_SMPC)
3574 gv11b_gr_update_priv_addr_table_smpc(
3575 g, gpc_num, tpc_num, addr,
3576 priv_addr_table, t);
3577 else {
3578 priv_addr_table[*t] =
3579 pri_etpc_addr(g,
3580 pri_tpccs_addr_mask(addr),
3581 gpc, tpc_num);
3582 nvgpu_log_info(g,
3583 "priv_addr_table[%d]:%#08x",
3584 *t, priv_addr_table[*t]);
3585 (*t)++;
3586 }
3587 }
3588 } else if (broadcast_flags & PRI_BROADCAST_FLAGS_SMPC) {
3589 tpc_num = 0;
3590 gpc_num = 0;
3591 gv11b_gr_update_priv_addr_table_smpc(
3592 g, gpc_num, tpc_num, addr,
3593 priv_addr_table, t);
3594 } else {
3595 priv_addr_table[*t] = addr;
3596 nvgpu_log_info(g, "priv_addr_table[%d]:%#08x",
3597 *t, priv_addr_table[*t]);
3598 (*t)++;
3599 }
3600 }
3601}
3602
3603u32 gv11b_gr_get_egpc_base(struct gk20a *g)
3604{
3605 return EGPC_PRI_BASE;
3606}
3607
3608void gr_gv11b_init_gpc_mmu(struct gk20a *g)
3609{
3610 u32 temp;
3611
3612 nvgpu_log_info(g, "initialize gpc mmu");
3613
3614 if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
3615 /* Bypass MMU check for non-secure boot. For
3616 * secure-boot,this register write has no-effect */
3617 gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffff);
3618 }
3619 temp = gk20a_readl(g, fb_mmu_ctrl_r());
3620 temp &= gr_gpcs_pri_mmu_ctrl_vm_pg_size_m() |
3621 gr_gpcs_pri_mmu_ctrl_use_pdb_big_page_size_m() |
3622 gr_gpcs_pri_mmu_ctrl_vol_fault_m() |
3623 gr_gpcs_pri_mmu_ctrl_comp_fault_m() |
3624 gr_gpcs_pri_mmu_ctrl_miss_gran_m() |
3625 gr_gpcs_pri_mmu_ctrl_cache_mode_m() |
3626 gr_gpcs_pri_mmu_ctrl_mmu_aperture_m() |
3627 gr_gpcs_pri_mmu_ctrl_mmu_vol_m() |
3628 gr_gpcs_pri_mmu_ctrl_mmu_disable_m();
3629 gk20a_writel(g, gr_gpcs_pri_mmu_ctrl_r(), temp);
3630 gk20a_writel(g, gr_gpcs_pri_mmu_pm_unit_mask_r(), 0);
3631 gk20a_writel(g, gr_gpcs_pri_mmu_pm_req_mask_r(), 0);
3632
3633 gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(),
3634 gk20a_readl(g, fb_mmu_debug_ctrl_r()));
3635 gk20a_writel(g, gr_gpcs_pri_mmu_debug_wr_r(),
3636 gk20a_readl(g, fb_mmu_debug_wr_r()));
3637 gk20a_writel(g, gr_gpcs_pri_mmu_debug_rd_r(),
3638 gk20a_readl(g, fb_mmu_debug_rd_r()));
3639}
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.h b/drivers/gpu/nvgpu/gv11b/gr_gv11b.h
new file mode 100644
index 00000000..b6ba231e
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.h
@@ -0,0 +1,215 @@
1/*
2 * GV11B GPU GR
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef _NVGPU_GR_GV11B_H_
26#define _NVGPU_GR_GV11B_H_
27
28#define EGPC_PRI_BASE 0x580000
29#define EGPC_PRI_SHARED_BASE 0x480000
30
31#define PRI_BROADCAST_FLAGS_SMPC BIT(17)
32
33#define GV11B_ZBC_TYPE_STENCIL T19X_ZBC
34#define ZBC_STENCIL_CLEAR_FMT_INVAILD 0
35#define ZBC_STENCIL_CLEAR_FMT_U8 1
36
37struct zbc_s_table {
38 u32 stencil;
39 u32 format;
40 u32 ref_cnt;
41};
42
43struct gk20a;
44struct zbc_entry;
45struct zbc_query_params;
46struct channel_ctx_gk20a;
47struct nvgpu_warpstate;
48struct nvgpu_gr_sm_error_state;
49
50enum {
51 VOLTA_CHANNEL_GPFIFO_A = 0xC36F,
52 VOLTA_A = 0xC397,
53 VOLTA_COMPUTE_A = 0xC3C0,
54 VOLTA_DMA_COPY_A = 0xC3B5,
55};
56
57#define NVC397_SET_SHADER_EXCEPTIONS 0x1528
58#define NVC397_SET_CIRCULAR_BUFFER_SIZE 0x1280
59#define NVC397_SET_ALPHA_CIRCULAR_BUFFER_SIZE 0x02dc
60#define NVC397_SET_GO_IDLE_TIMEOUT 0x022c
61#define NVC397_SET_TEX_IN_DBG 0x10bc
62#define NVC397_SET_SKEDCHECK 0x10c0
63#define NVC397_SET_BES_CROP_DEBUG3 0x10c4
64
65#define NVC397_SET_TEX_IN_DBG_TSL1_RVCH_INVALIDATE 0x1
66#define NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_LD 0x2
67#define NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_ST 0x4
68
69#define NVC397_SET_SKEDCHECK_18_MASK 0x3
70#define NVC397_SET_SKEDCHECK_18_DEFAULT 0x0
71#define NVC397_SET_SKEDCHECK_18_DISABLE 0x1
72#define NVC397_SET_SKEDCHECK_18_ENABLE 0x2
73
74#define NVC3C0_SET_SKEDCHECK 0x23c
75
76#define NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE 0
77
78int gr_gv11b_alloc_buffer(struct vm_gk20a *vm, size_t size,
79 struct nvgpu_mem *mem);
80/*zcull*/
81void gr_gv11b_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries,
82 u32 *zcull_map_tiles);
83void gr_gv11b_create_sysfs(struct gk20a *g);
84
85bool gr_gv11b_is_valid_class(struct gk20a *g, u32 class_num);
86bool gr_gv11b_is_valid_gfx_class(struct gk20a *g, u32 class_num);
87bool gr_gv11b_is_valid_compute_class(struct gk20a *g, u32 class_num);
88void gr_gv11b_enable_hww_exceptions(struct gk20a *g);
89void gr_gv11b_enable_exceptions(struct gk20a *g);
90int gr_gv11b_handle_tpc_sm_ecc_exception(struct gk20a *g,
91 u32 gpc, u32 tpc,
92 bool *post_event, struct channel_gk20a *fault_ch,
93 u32 *hww_global_esr);
94int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc,
95 bool *post_event, struct channel_gk20a *fault_ch,
96 u32 *hww_global_esr);
97int gr_gv11b_handle_gpc_gpcmmu_exception(struct gk20a *g, u32 gpc,
98 u32 gpc_exception);
99int gr_gv11b_handle_gpc_gpccs_exception(struct gk20a *g, u32 gpc,
100 u32 gpc_exception);
101void gr_gv11b_enable_gpc_exceptions(struct gk20a *g);
102int gr_gv11b_handle_tex_exception(struct gk20a *g, u32 gpc, u32 tpc,
103 bool *post_event);
104int gr_gv11b_zbc_s_query_table(struct gk20a *g, struct gr_gk20a *gr,
105 struct zbc_query_params *query_params);
106bool gr_gv11b_add_zbc_type_s(struct gk20a *g, struct gr_gk20a *gr,
107 struct zbc_entry *zbc_val, int *ret_val);
108int gr_gv11b_add_zbc_stencil(struct gk20a *g, struct gr_gk20a *gr,
109 struct zbc_entry *stencil_val, u32 index);
110int gr_gv11b_load_stencil_default_tbl(struct gk20a *g,
111 struct gr_gk20a *gr);
112int gr_gv11b_load_stencil_tbl(struct gk20a *g, struct gr_gk20a *gr);
113u32 gr_gv11b_pagepool_default_size(struct gk20a *g);
114int gr_gv11b_calc_global_ctx_buffer_size(struct gk20a *g);
115int gr_gv11b_handle_sw_method(struct gk20a *g, u32 addr,
116 u32 class_num, u32 offset, u32 data);
117void gr_gv11b_bundle_cb_defaults(struct gk20a *g);
118void gr_gv11b_cb_size_default(struct gk20a *g);
119void gr_gv11b_set_alpha_circular_buffer_size(struct gk20a *g, u32 data);
120void gr_gv11b_set_circular_buffer_size(struct gk20a *g, u32 data);
121int gr_gv11b_dump_gr_status_regs(struct gk20a *g,
122 struct gk20a_debug_output *o);
123int gr_gv11b_wait_empty(struct gk20a *g, unsigned long duration_ms,
124 u32 expect_delay);
125void gr_gv11b_commit_global_attrib_cb(struct gk20a *g,
126 struct channel_ctx_gk20a *ch_ctx,
127 u64 addr, bool patch);
128void gr_gv11b_set_gpc_tpc_mask(struct gk20a *g, u32 gpc_index);
129void gr_gv11b_get_access_map(struct gk20a *g,
130 u32 **whitelist, int *num_entries);
131int gr_gv11b_pre_process_sm_exception(struct gk20a *g,
132 u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr,
133 bool sm_debugger_attached, struct channel_gk20a *fault_ch,
134 bool *early_exit, bool *ignore_debugger);
135int gr_gv11b_handle_fecs_error(struct gk20a *g,
136 struct channel_gk20a *__ch,
137 struct gr_gk20a_isr_data *isr_data);
138int gr_gv11b_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr);
139int gr_gv11b_init_sw_veid_bundle(struct gk20a *g);
140void gr_gv11b_detect_sm_arch(struct gk20a *g);
141void gr_gv11b_program_sm_id_numbering(struct gk20a *g,
142 u32 gpc, u32 tpc, u32 smid);
143int gr_gv11b_load_smid_config(struct gk20a *g);
144int gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va);
145int gr_gv11b_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c);
146void gr_gv11b_write_zcull_ptr(struct gk20a *g,
147 struct nvgpu_mem *mem, u64 gpu_va);
148void gr_gv11b_write_pm_ptr(struct gk20a *g,
149 struct nvgpu_mem *mem, u64 gpu_va);
150void gr_gv11b_init_elcg_mode(struct gk20a *g, u32 mode, u32 engine);
151void gr_gv11b_load_tpc_mask(struct gk20a *g);
152void gr_gv11b_set_preemption_buffer_va(struct gk20a *g,
153 struct nvgpu_mem *mem, u64 gpu_va);
154int gr_gv11b_init_fs_state(struct gk20a *g);
155void gv11b_gr_get_esr_sm_sel(struct gk20a *g, u32 gpc, u32 tpc,
156 u32 *esr_sm_sel);
157int gv11b_gr_sm_trigger_suspend(struct gk20a *g);
158void gv11b_gr_bpt_reg_info(struct gk20a *g, struct nvgpu_warpstate *w_state);
159int gv11b_gr_update_sm_error_state(struct gk20a *g,
160 struct channel_gk20a *ch, u32 sm_id,
161 struct nvgpu_gr_sm_error_state *sm_error_state);
162int gv11b_gr_set_sm_debug_mode(struct gk20a *g,
163 struct channel_gk20a *ch, u64 sms, bool enable);
164int gv11b_gr_record_sm_error_state(struct gk20a *g, u32 gpc, u32 tpc);
165void gv11b_gr_set_hww_esr_report_mask(struct gk20a *g);
166bool gv11b_gr_sm_debugger_attached(struct gk20a *g);
167void gv11b_gr_suspend_single_sm(struct gk20a *g,
168 u32 gpc, u32 tpc, u32 sm,
169 u32 global_esr_mask, bool check_errors);
170void gv11b_gr_suspend_all_sms(struct gk20a *g,
171 u32 global_esr_mask, bool check_errors);
172void gv11b_gr_resume_single_sm(struct gk20a *g,
173 u32 gpc, u32 tpc, u32 sm);
174void gv11b_gr_resume_all_sms(struct gk20a *g);
175int gv11b_gr_resume_from_pause(struct gk20a *g);
176u32 gv11b_gr_get_sm_hww_warp_esr(struct gk20a *g,
177 u32 gpc, u32 tpc, u32 sm);
178u32 gv11b_gr_get_sm_hww_global_esr(struct gk20a *g,
179 u32 gpc, u32 tpc, u32 sm);
180u32 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g);
181int gv11b_gr_wait_for_sm_lock_down(struct gk20a *g,
182 u32 gpc, u32 tpc, u32 sm,
183 u32 global_esr_mask, bool check_errors);
184int gv11b_gr_lock_down_sm(struct gk20a *g,
185 u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask,
186 bool check_errors);
187void gv11b_gr_clear_sm_hww(struct gk20a *g, u32 gpc, u32 tpc, u32 sm,
188 u32 global_esr);
189int gr_gv11b_handle_tpc_mpc_exception(struct gk20a *g,
190 u32 gpc, u32 tpc, bool *post_event);
191void gv11b_gr_init_ovr_sm_dsm_perf(void);
192void gv11b_gr_init_sm_dsm_reg_info(void);
193void gv11b_gr_get_sm_dsm_perf_regs(struct gk20a *g,
194 u32 *num_sm_dsm_perf_regs,
195 u32 **sm_dsm_perf_regs,
196 u32 *perf_register_stride);
197void gv11b_gr_get_sm_dsm_perf_ctrl_regs(struct gk20a *g,
198 u32 *num_sm_dsm_perf_ctrl_regs,
199 u32 **sm_dsm_perf_ctrl_regs,
200 u32 *ctrl_register_stride);
201void gv11b_gr_get_ovr_perf_regs(struct gk20a *g, u32 *num_ovr_perf_regs,
202 u32 **ovr_perf_regs);
203void gv11b_gr_access_smpc_reg(struct gk20a *g, u32 quad, u32 offset);
204bool gv11b_gr_pri_is_egpc_addr(struct gk20a *g, u32 addr);
205bool gv11b_gr_pri_is_etpc_addr(struct gk20a *g, u32 addr);
206void gv11b_gr_get_egpc_etpc_num(struct gk20a *g, u32 addr,
207 u32 *egpc_num, u32 *etpc_num);
208int gv11b_gr_decode_egpc_addr(struct gk20a *g, u32 addr, int *addr_type,
209 u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags);
210void gv11b_gr_egpc_etpc_priv_addr_table(struct gk20a *g, u32 addr,
211 u32 gpc, u32 broadcast_flags, u32 *priv_addr_table, u32 *t);
212u32 gv11b_gr_get_egpc_base(struct gk20a *g);
213void gr_gv11b_init_gpc_mmu(struct gk20a *g);
214
215#endif
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b.c b/drivers/gpu/nvgpu/gv11b/gv11b.c
new file mode 100644
index 00000000..211755e5
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gv11b.c
@@ -0,0 +1,38 @@
1/*
2 * GV11B Graphics
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <nvgpu/enabled.h>
26#include <nvgpu/enabled_t19x.h>
27
28#include "gk20a/gk20a.h"
29
30#include "gv11b/gv11b.h"
31
32int gv11b_init_gpu_characteristics(struct gk20a *g)
33{
34 gk20a_init_gpu_characteristics(g);
35 __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG_SUBCONTEXTS, true);
36 __nvgpu_set_enabled(g, NVGPU_SUPPORT_IO_COHERENCE, true);
37 return 0;
38}
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b.h b/drivers/gpu/nvgpu/gv11b/gv11b.h
new file mode 100644
index 00000000..3d5490e6
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gv11b.h
@@ -0,0 +1,32 @@
1/*
2 * GV11B Graphics
3 *
4 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef GV11B_H
26#define GV11B_H
27
28#include "gk20a/gk20a.h"
29
30int gv11b_init_gpu_characteristics(struct gk20a *g);
31
32#endif /* GV11B_H */
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c
new file mode 100644
index 00000000..9f6057ae
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c
@@ -0,0 +1,748 @@
1/*
2 * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 *
22 * This file is autogenerated. Do not edit.
23 */
24
25#ifndef __gv11b_gating_reglist_h__
26#define __gv11b_gating_reglist_h__
27
28#include <linux/types.h>
29#include "gv11b_gating_reglist.h"
30#include <nvgpu/enabled.h>
31
32struct gating_desc {
33 u32 addr;
34 u32 prod;
35 u32 disable;
36};
37/* slcg bus */
38static const struct gating_desc gv11b_slcg_bus[] = {
39 {.addr = 0x00001c04, .prod = 0x00000000, .disable = 0x000003fe},
40};
41
42/* slcg ce2 */
43static const struct gating_desc gv11b_slcg_ce2[] = {
44 {.addr = 0x00104204, .prod = 0x00000040, .disable = 0x000007fe},
45};
46
47/* slcg chiplet */
48static const struct gating_desc gv11b_slcg_chiplet[] = {
49 {.addr = 0x0010c07c, .prod = 0x00000000, .disable = 0x00000007},
50 {.addr = 0x0010e07c, .prod = 0x00000000, .disable = 0x00000007},
51 {.addr = 0x0010d07c, .prod = 0x00000000, .disable = 0x00000007},
52 {.addr = 0x0010e17c, .prod = 0x00000000, .disable = 0x00000007},
53};
54
55/* slcg fb */
56static const struct gating_desc gv11b_slcg_fb[] = {
57 {.addr = 0x00100d14, .prod = 0x00000000, .disable = 0xfffffffe},
58 {.addr = 0x00100c9c, .prod = 0x00000000, .disable = 0x000001fe},
59};
60
61/* slcg fifo */
62static const struct gating_desc gv11b_slcg_fifo[] = {
63 {.addr = 0x000026ec, .prod = 0x00000000, .disable = 0x0001fffe},
64};
65
66/* slcg gr */
67static const struct gating_desc gv11b_slcg_gr[] = {
68 {.addr = 0x004041f4, .prod = 0x00000000, .disable = 0x07fffffe},
69 {.addr = 0x00409134, .prod = 0x00020008, .disable = 0x0003fffe},
70 {.addr = 0x00409894, .prod = 0x00000000, .disable = 0x0000fffe},
71 {.addr = 0x004078c4, .prod = 0x00000000, .disable = 0x000001fe},
72 {.addr = 0x00406004, .prod = 0x00000200, .disable = 0x0001fffe},
73 {.addr = 0x00405864, .prod = 0x00000000, .disable = 0x000001fe},
74 {.addr = 0x00405910, .prod = 0xfffffff0, .disable = 0xfffffffe},
75 {.addr = 0x00408044, .prod = 0x00000000, .disable = 0x000007fe},
76 {.addr = 0x00407004, .prod = 0x00000000, .disable = 0x000001fe},
77 {.addr = 0x00405bf4, .prod = 0x00000000, .disable = 0x00000002},
78 {.addr = 0x0041a134, .prod = 0x00020008, .disable = 0x0003fffe},
79 {.addr = 0x0041a894, .prod = 0x00000000, .disable = 0x0000fffe},
80 {.addr = 0x00418504, .prod = 0x00000000, .disable = 0x0007fffe},
81 {.addr = 0x0041860c, .prod = 0x00000000, .disable = 0x000001fe},
82 {.addr = 0x0041868c, .prod = 0x00000000, .disable = 0x0000001e},
83 {.addr = 0x0041871c, .prod = 0x00000000, .disable = 0x000003fe},
84 {.addr = 0x00418388, .prod = 0x00000000, .disable = 0x00000001},
85 {.addr = 0x0041882c, .prod = 0x00000000, .disable = 0x0001fffe},
86 {.addr = 0x00418bc0, .prod = 0x00000000, .disable = 0x000001fe},
87 {.addr = 0x00418974, .prod = 0x00000000, .disable = 0x0001fffe},
88 {.addr = 0x00418c74, .prod = 0xffffff80, .disable = 0xfffffffe},
89 {.addr = 0x00418cf4, .prod = 0xfffffff8, .disable = 0xfffffffe},
90 {.addr = 0x00418d74, .prod = 0xffffffe0, .disable = 0xfffffffe},
91 {.addr = 0x00418f10, .prod = 0xffffffe0, .disable = 0xfffffffe},
92 {.addr = 0x00418e10, .prod = 0xfffffffe, .disable = 0xfffffffe},
93 {.addr = 0x00419024, .prod = 0x000001fe, .disable = 0x000001fe},
94 {.addr = 0x0041889c, .prod = 0x00000000, .disable = 0x000001fe},
95 {.addr = 0x00419d24, .prod = 0x00000000, .disable = 0x000000ff},
96 {.addr = 0x0041986c, .prod = 0x00000104, .disable = 0x00fffffe},
97 {.addr = 0x00419c74, .prod = 0x0000001e, .disable = 0x0000001e},
98 {.addr = 0x00419c84, .prod = 0x0003fff8, .disable = 0x0003fffe},
99 {.addr = 0x00419c8c, .prod = 0xffffff84, .disable = 0xfffffffe},
100 {.addr = 0x00419c94, .prod = 0x00080040, .disable = 0x000ffffe},
101 {.addr = 0x00419ca4, .prod = 0x00003ffe, .disable = 0x00003ffe},
102 {.addr = 0x00419cac, .prod = 0x0001fffe, .disable = 0x0001fffe},
103 {.addr = 0x00419a44, .prod = 0x00000008, .disable = 0x0000000e},
104 {.addr = 0x00419a4c, .prod = 0x000001f8, .disable = 0x000001fe},
105 {.addr = 0x00419a54, .prod = 0x0000003c, .disable = 0x0000003e},
106 {.addr = 0x00419a5c, .prod = 0x0000000c, .disable = 0x0000000e},
107 {.addr = 0x00419a64, .prod = 0x000001ba, .disable = 0x000001fe},
108 {.addr = 0x00419a7c, .prod = 0x0000003c, .disable = 0x0000003e},
109 {.addr = 0x00419a84, .prod = 0x0000000c, .disable = 0x0000000e},
110 {.addr = 0x0041be2c, .prod = 0x04115fc0, .disable = 0xfffffffe},
111 {.addr = 0x0041bfec, .prod = 0xfffffff0, .disable = 0xfffffffe},
112 {.addr = 0x0041bed4, .prod = 0xfffffff8, .disable = 0xfffffffe},
113 {.addr = 0x00408814, .prod = 0x00000000, .disable = 0x0001fffe},
114 {.addr = 0x00408a84, .prod = 0x00000000, .disable = 0x0001fffe},
115 {.addr = 0x004089ac, .prod = 0x00000000, .disable = 0x0001fffe},
116 {.addr = 0x00408a24, .prod = 0x00000000, .disable = 0x000000ff},
117};
118
119/* slcg ltc */
120static const struct gating_desc gv11b_slcg_ltc[] = {
121 {.addr = 0x0017e050, .prod = 0x00000000, .disable = 0xfffffffe},
122 {.addr = 0x0017e35c, .prod = 0x00000000, .disable = 0xfffffffe},
123};
124
125/* slcg perf */
126static const struct gating_desc gv11b_slcg_perf[] = {
127 {.addr = 0x00248018, .prod = 0xffffffff, .disable = 0x00000000},
128 {.addr = 0x00248018, .prod = 0xffffffff, .disable = 0x00000000},
129 {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000},
130 {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000},
131 {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000},
132 {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000},
133 {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000},
134 {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000},
135 {.addr = 0x0024a124, .prod = 0x00000001, .disable = 0x00000000},
136};
137
138/* slcg PriRing */
139static const struct gating_desc gv11b_slcg_priring[] = {
140 {.addr = 0x001200a8, .prod = 0x00000000, .disable = 0x00000001},
141};
142
143/* slcg pwr_csb */
144static const struct gating_desc gv11b_slcg_pwr_csb[] = {
145 {.addr = 0x00000134, .prod = 0x00020008, .disable = 0x0003fffe},
146 {.addr = 0x00000e74, .prod = 0x00000000, .disable = 0x0000000f},
147 {.addr = 0x00000a74, .prod = 0x00004040, .disable = 0x00007ffe},
148 {.addr = 0x000206b8, .prod = 0x00000008, .disable = 0x0000000f},
149};
150
151/* slcg pmu */
152static const struct gating_desc gv11b_slcg_pmu[] = {
153 {.addr = 0x0010a134, .prod = 0x00020008, .disable = 0x0003fffe},
154 {.addr = 0x0010aa74, .prod = 0x00004040, .disable = 0x00007ffe},
155 {.addr = 0x0010ae74, .prod = 0x00000000, .disable = 0x0000000f},
156};
157
158/* therm gr */
159static const struct gating_desc gv11b_slcg_therm[] = {
160 {.addr = 0x000206b8, .prod = 0x00000008, .disable = 0x0000000f},
161};
162
163/* slcg Xbar */
164static const struct gating_desc gv11b_slcg_xbar[] = {
165 {.addr = 0x0013c824, .prod = 0x00000000, .disable = 0x7ffffffe},
166 {.addr = 0x0013dc08, .prod = 0x00000000, .disable = 0xfffffffe},
167 {.addr = 0x0013c924, .prod = 0x00000000, .disable = 0x7ffffffe},
168 {.addr = 0x0013cbe4, .prod = 0x00000000, .disable = 0x1ffffffe},
169 {.addr = 0x0013cc04, .prod = 0x00000000, .disable = 0x1ffffffe},
170};
171
172/* blcg bus */
173static const struct gating_desc gv11b_blcg_bus[] = {
174 {.addr = 0x00001c00, .prod = 0x00000042, .disable = 0x00000000},
175};
176
177/* blcg ce */
178static const struct gating_desc gv11b_blcg_ce[] = {
179 {.addr = 0x00104200, .prod = 0x0000c242, .disable = 0x00000000},
180};
181
182/* blcg ctxsw prog */
183static const struct gating_desc gv11b_blcg_ctxsw_prog[] = {
184};
185
186/* blcg fb */
187static const struct gating_desc gv11b_blcg_fb[] = {
188 {.addr = 0x00100d10, .prod = 0x0000c242, .disable = 0x00000000},
189 {.addr = 0x00100d30, .prod = 0x0000c242, .disable = 0x00000000},
190 {.addr = 0x00100d3c, .prod = 0x00000242, .disable = 0x00000000},
191 {.addr = 0x00100d48, .prod = 0x0000c242, .disable = 0x00000000},
192 {.addr = 0x00100d1c, .prod = 0x00000042, .disable = 0x00000000},
193 {.addr = 0x00100c98, .prod = 0x00004242, .disable = 0x00000000},
194};
195
196/* blcg fifo */
197static const struct gating_desc gv11b_blcg_fifo[] = {
198 {.addr = 0x000026e0, .prod = 0x0000c244, .disable = 0x00000000},
199};
200
201/* blcg gr */
202static const struct gating_desc gv11b_blcg_gr[] = {
203 {.addr = 0x004041f0, .prod = 0x0000c646, .disable = 0x00000000},
204 {.addr = 0x00409890, .prod = 0x0000007f, .disable = 0x00000000},
205 {.addr = 0x004098b0, .prod = 0x0000007f, .disable = 0x00000000},
206 {.addr = 0x004078c0, .prod = 0x00004242, .disable = 0x00000000},
207 {.addr = 0x00406000, .prod = 0x0000c444, .disable = 0x00000000},
208 {.addr = 0x00405860, .prod = 0x0000c242, .disable = 0x00000000},
209 {.addr = 0x0040590c, .prod = 0x0000c444, .disable = 0x00000000},
210 {.addr = 0x00408040, .prod = 0x0000c444, .disable = 0x00000000},
211 {.addr = 0x00407000, .prod = 0x4000c242, .disable = 0x00000000},
212 {.addr = 0x00405bf0, .prod = 0x0000c444, .disable = 0x00000000},
213 {.addr = 0x0041a890, .prod = 0x0000427f, .disable = 0x00000000},
214 {.addr = 0x0041a8b0, .prod = 0x0000007f, .disable = 0x00000000},
215 {.addr = 0x00418500, .prod = 0x0000c244, .disable = 0x00000000},
216 {.addr = 0x00418608, .prod = 0x0000c242, .disable = 0x00000000},
217 {.addr = 0x00418688, .prod = 0x0000c242, .disable = 0x00000000},
218 {.addr = 0x00418718, .prod = 0x00000042, .disable = 0x00000000},
219 {.addr = 0x00418828, .prod = 0x00008444, .disable = 0x00000000},
220 {.addr = 0x00418bbc, .prod = 0x0000c242, .disable = 0x00000000},
221 {.addr = 0x00418970, .prod = 0x0000c242, .disable = 0x00000000},
222 {.addr = 0x00418c70, .prod = 0x0000c444, .disable = 0x00000000},
223 {.addr = 0x00418cf0, .prod = 0x0000c444, .disable = 0x00000000},
224 {.addr = 0x00418d70, .prod = 0x0000c444, .disable = 0x00000000},
225 {.addr = 0x00418f0c, .prod = 0x0000c444, .disable = 0x00000000},
226 {.addr = 0x00418e0c, .prod = 0x0000c444, .disable = 0x00000000},
227 {.addr = 0x00419020, .prod = 0x0000c242, .disable = 0x00000000},
228 {.addr = 0x00419038, .prod = 0x00000042, .disable = 0x00000000},
229 {.addr = 0x00418898, .prod = 0x00004242, .disable = 0x00000000},
230 {.addr = 0x00419868, .prod = 0x00008243, .disable = 0x00000000},
231 {.addr = 0x00419c70, .prod = 0x0000c444, .disable = 0x00000000},
232 {.addr = 0x00419c80, .prod = 0x00004045, .disable = 0x00000000},
233 {.addr = 0x00419c88, .prod = 0x00004043, .disable = 0x00000000},
234 {.addr = 0x00419c90, .prod = 0x0000004a, .disable = 0x00000000},
235 {.addr = 0x00419c98, .prod = 0x00000042, .disable = 0x00000000},
236 {.addr = 0x00419ca0, .prod = 0x00000043, .disable = 0x00000000},
237 {.addr = 0x00419ca8, .prod = 0x00000003, .disable = 0x00000000},
238 {.addr = 0x00419cb0, .prod = 0x00000002, .disable = 0x00000000},
239 {.addr = 0x00419a40, .prod = 0x00000242, .disable = 0x00000000},
240 {.addr = 0x00419a48, .prod = 0x00000242, .disable = 0x00000000},
241 {.addr = 0x00419a50, .prod = 0x00000242, .disable = 0x00000000},
242 {.addr = 0x00419a58, .prod = 0x00000242, .disable = 0x00000000},
243 {.addr = 0x00419a60, .prod = 0x00000202, .disable = 0x00000000},
244 {.addr = 0x00419a68, .prod = 0x00000202, .disable = 0x00000000},
245 {.addr = 0x00419a78, .prod = 0x00000242, .disable = 0x00000000},
246 {.addr = 0x00419a80, .prod = 0x00000242, .disable = 0x00000000},
247 {.addr = 0x0041be28, .prod = 0x00008242, .disable = 0x00000000},
248 {.addr = 0x0041bfe8, .prod = 0x0000c444, .disable = 0x00000000},
249 {.addr = 0x0041bed0, .prod = 0x0000c444, .disable = 0x00000000},
250 {.addr = 0x00408810, .prod = 0x0000c242, .disable = 0x00000000},
251 {.addr = 0x00408a80, .prod = 0x0000c242, .disable = 0x00000000},
252 {.addr = 0x004089a8, .prod = 0x0000c242, .disable = 0x00000000},
253};
254
255/* blcg ltc */
256static const struct gating_desc gv11b_blcg_ltc[] = {
257 {.addr = 0x0017e030, .prod = 0x00000044, .disable = 0x00000000},
258 {.addr = 0x0017e040, .prod = 0x00000044, .disable = 0x00000000},
259 {.addr = 0x0017e3e0, .prod = 0x00000044, .disable = 0x00000000},
260 {.addr = 0x0017e3c8, .prod = 0x00000044, .disable = 0x00000000},
261};
262
263/* blcg pwr_csb */
264static const struct gating_desc gv11b_blcg_pwr_csb[] = {
265 {.addr = 0x00000a70, .prod = 0x00000045, .disable = 0x00000000},
266};
267
268/* blcg pmu */
269static const struct gating_desc gv11b_blcg_pmu[] = {
270 {.addr = 0x0010aa70, .prod = 0x00000045, .disable = 0x00000000},
271};
272
273/* blcg Xbar */
274static const struct gating_desc gv11b_blcg_xbar[] = {
275 {.addr = 0x0013c820, .prod = 0x0001004a, .disable = 0x00000000},
276 {.addr = 0x0013dc04, .prod = 0x0001004a, .disable = 0x00000000},
277 {.addr = 0x0013c920, .prod = 0x0000004a, .disable = 0x00000000},
278 {.addr = 0x0013cbe0, .prod = 0x00000042, .disable = 0x00000000},
279 {.addr = 0x0013cc00, .prod = 0x00000042, .disable = 0x00000000},
280};
281
282/* pg gr */
283static const struct gating_desc gv11b_pg_gr[] = {
284};
285
286/* inline functions */
287void gv11b_slcg_bus_load_gating_prod(struct gk20a *g,
288 bool prod)
289{
290 u32 i;
291 u32 size = sizeof(gv11b_slcg_bus) / sizeof(struct gating_desc);
292
293 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
294 return;
295
296 for (i = 0; i < size; i++) {
297 if (prod)
298 gk20a_writel(g, gv11b_slcg_bus[i].addr,
299 gv11b_slcg_bus[i].prod);
300 else
301 gk20a_writel(g, gv11b_slcg_bus[i].addr,
302 gv11b_slcg_bus[i].disable);
303 }
304}
305
306void gv11b_slcg_ce2_load_gating_prod(struct gk20a *g,
307 bool prod)
308{
309 u32 i;
310 u32 size = sizeof(gv11b_slcg_ce2) / sizeof(struct gating_desc);
311
312 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
313 return;
314
315 for (i = 0; i < size; i++) {
316 if (prod)
317 gk20a_writel(g, gv11b_slcg_ce2[i].addr,
318 gv11b_slcg_ce2[i].prod);
319 else
320 gk20a_writel(g, gv11b_slcg_ce2[i].addr,
321 gv11b_slcg_ce2[i].disable);
322 }
323}
324
325void gv11b_slcg_chiplet_load_gating_prod(struct gk20a *g,
326 bool prod)
327{
328 u32 i;
329 u32 size = sizeof(gv11b_slcg_chiplet) / sizeof(struct gating_desc);
330
331 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
332 return;
333
334 for (i = 0; i < size; i++) {
335 if (prod)
336 gk20a_writel(g, gv11b_slcg_chiplet[i].addr,
337 gv11b_slcg_chiplet[i].prod);
338 else
339 gk20a_writel(g, gv11b_slcg_chiplet[i].addr,
340 gv11b_slcg_chiplet[i].disable);
341 }
342}
343
344void gv11b_slcg_ctxsw_firmware_load_gating_prod(struct gk20a *g,
345 bool prod)
346{
347}
348
349void gv11b_slcg_fb_load_gating_prod(struct gk20a *g,
350 bool prod)
351{
352 u32 i;
353 u32 size = sizeof(gv11b_slcg_fb) / sizeof(struct gating_desc);
354
355 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
356 return;
357
358 for (i = 0; i < size; i++) {
359 if (prod)
360 gk20a_writel(g, gv11b_slcg_fb[i].addr,
361 gv11b_slcg_fb[i].prod);
362 else
363 gk20a_writel(g, gv11b_slcg_fb[i].addr,
364 gv11b_slcg_fb[i].disable);
365 }
366}
367
368void gv11b_slcg_fifo_load_gating_prod(struct gk20a *g,
369 bool prod)
370{
371 u32 i;
372 u32 size = sizeof(gv11b_slcg_fifo) / sizeof(struct gating_desc);
373
374 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
375 return;
376
377 for (i = 0; i < size; i++) {
378 if (prod)
379 gk20a_writel(g, gv11b_slcg_fifo[i].addr,
380 gv11b_slcg_fifo[i].prod);
381 else
382 gk20a_writel(g, gv11b_slcg_fifo[i].addr,
383 gv11b_slcg_fifo[i].disable);
384 }
385}
386
387void gr_gv11b_slcg_gr_load_gating_prod(struct gk20a *g,
388 bool prod)
389{
390 u32 i;
391 u32 size = sizeof(gv11b_slcg_gr) / sizeof(struct gating_desc);
392
393 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
394 return;
395
396 for (i = 0; i < size; i++) {
397 if (prod)
398 gk20a_writel(g, gv11b_slcg_gr[i].addr,
399 gv11b_slcg_gr[i].prod);
400 else
401 gk20a_writel(g, gv11b_slcg_gr[i].addr,
402 gv11b_slcg_gr[i].disable);
403 }
404}
405
406void ltc_gv11b_slcg_ltc_load_gating_prod(struct gk20a *g,
407 bool prod)
408{
409 u32 i;
410 u32 size = sizeof(gv11b_slcg_ltc) / sizeof(struct gating_desc);
411
412 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
413 return;
414
415 for (i = 0; i < size; i++) {
416 if (prod)
417 gk20a_writel(g, gv11b_slcg_ltc[i].addr,
418 gv11b_slcg_ltc[i].prod);
419 else
420 gk20a_writel(g, gv11b_slcg_ltc[i].addr,
421 gv11b_slcg_ltc[i].disable);
422 }
423}
424
425void gv11b_slcg_perf_load_gating_prod(struct gk20a *g,
426 bool prod)
427{
428 u32 i;
429 u32 size = sizeof(gv11b_slcg_perf) / sizeof(struct gating_desc);
430
431 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
432 return;
433
434 for (i = 0; i < size; i++) {
435 if (prod)
436 gk20a_writel(g, gv11b_slcg_perf[i].addr,
437 gv11b_slcg_perf[i].prod);
438 else
439 gk20a_writel(g, gv11b_slcg_perf[i].addr,
440 gv11b_slcg_perf[i].disable);
441 }
442}
443
444void gv11b_slcg_priring_load_gating_prod(struct gk20a *g,
445 bool prod)
446{
447 u32 i;
448 u32 size = sizeof(gv11b_slcg_priring) / sizeof(struct gating_desc);
449
450 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
451 return;
452
453 for (i = 0; i < size; i++) {
454 if (prod)
455 gk20a_writel(g, gv11b_slcg_priring[i].addr,
456 gv11b_slcg_priring[i].prod);
457 else
458 gk20a_writel(g, gv11b_slcg_priring[i].addr,
459 gv11b_slcg_priring[i].disable);
460 }
461}
462
463void gv11b_slcg_pwr_csb_load_gating_prod(struct gk20a *g,
464 bool prod)
465{
466 u32 i;
467 u32 size = sizeof(gv11b_slcg_pwr_csb) / sizeof(struct gating_desc);
468
469 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
470 return;
471
472 for (i = 0; i < size; i++) {
473 if (prod)
474 gk20a_writel(g, gv11b_slcg_pwr_csb[i].addr,
475 gv11b_slcg_pwr_csb[i].prod);
476 else
477 gk20a_writel(g, gv11b_slcg_pwr_csb[i].addr,
478 gv11b_slcg_pwr_csb[i].disable);
479 }
480}
481
482void gv11b_slcg_pmu_load_gating_prod(struct gk20a *g,
483 bool prod)
484{
485 u32 i;
486 u32 size = sizeof(gv11b_slcg_pmu) / sizeof(struct gating_desc);
487
488 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
489 return;
490
491 for (i = 0; i < size; i++) {
492 if (prod)
493 gk20a_writel(g, gv11b_slcg_pmu[i].addr,
494 gv11b_slcg_pmu[i].prod);
495 else
496 gk20a_writel(g, gv11b_slcg_pmu[i].addr,
497 gv11b_slcg_pmu[i].disable);
498 }
499}
500
501void gv11b_slcg_therm_load_gating_prod(struct gk20a *g,
502 bool prod)
503{
504 u32 i;
505 u32 size = sizeof(gv11b_slcg_therm) / sizeof(struct gating_desc);
506
507 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
508 return;
509
510 for (i = 0; i < size; i++) {
511 if (prod)
512 gk20a_writel(g, gv11b_slcg_therm[i].addr,
513 gv11b_slcg_therm[i].prod);
514 else
515 gk20a_writel(g, gv11b_slcg_therm[i].addr,
516 gv11b_slcg_therm[i].disable);
517 }
518}
519
520void gv11b_slcg_xbar_load_gating_prod(struct gk20a *g,
521 bool prod)
522{
523 u32 i;
524 u32 size = sizeof(gv11b_slcg_xbar) / sizeof(struct gating_desc);
525
526 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG))
527 return;
528
529 for (i = 0; i < size; i++) {
530 if (prod)
531 gk20a_writel(g, gv11b_slcg_xbar[i].addr,
532 gv11b_slcg_xbar[i].prod);
533 else
534 gk20a_writel(g, gv11b_slcg_xbar[i].addr,
535 gv11b_slcg_xbar[i].disable);
536 }
537}
538
539void gv11b_blcg_bus_load_gating_prod(struct gk20a *g,
540 bool prod)
541{
542 u32 i;
543 u32 size = sizeof(gv11b_blcg_bus) / sizeof(struct gating_desc);
544
545 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
546 return;
547
548 for (i = 0; i < size; i++) {
549 if (prod)
550 gk20a_writel(g, gv11b_blcg_bus[i].addr,
551 gv11b_blcg_bus[i].prod);
552 else
553 gk20a_writel(g, gv11b_blcg_bus[i].addr,
554 gv11b_blcg_bus[i].disable);
555 }
556}
557
558void gv11b_blcg_ce_load_gating_prod(struct gk20a *g,
559 bool prod)
560{
561 u32 i;
562 u32 size = sizeof(gv11b_blcg_ce) / sizeof(struct gating_desc);
563
564 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
565 return;
566
567 for (i = 0; i < size; i++) {
568 if (prod)
569 gk20a_writel(g, gv11b_blcg_ce[i].addr,
570 gv11b_blcg_ce[i].prod);
571 else
572 gk20a_writel(g, gv11b_blcg_ce[i].addr,
573 gv11b_blcg_ce[i].disable);
574 }
575}
576
577void gv11b_blcg_ctxsw_firmware_load_gating_prod(struct gk20a *g,
578 bool prod)
579{
580 u32 i;
581 u32 size = sizeof(gv11b_blcg_ctxsw_prog) / sizeof(struct gating_desc);
582
583 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
584 return;
585
586 for (i = 0; i < size; i++) {
587 if (prod)
588 gk20a_writel(g, gv11b_blcg_ctxsw_prog[i].addr,
589 gv11b_blcg_ctxsw_prog[i].prod);
590 else
591 gk20a_writel(g, gv11b_blcg_ctxsw_prog[i].addr,
592 gv11b_blcg_ctxsw_prog[i].disable);
593 }
594}
595
596void gv11b_blcg_fb_load_gating_prod(struct gk20a *g,
597 bool prod)
598{
599 u32 i;
600 u32 size = sizeof(gv11b_blcg_fb) / sizeof(struct gating_desc);
601
602 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
603 return;
604
605 for (i = 0; i < size; i++) {
606 if (prod)
607 gk20a_writel(g, gv11b_blcg_fb[i].addr,
608 gv11b_blcg_fb[i].prod);
609 else
610 gk20a_writel(g, gv11b_blcg_fb[i].addr,
611 gv11b_blcg_fb[i].disable);
612 }
613}
614
615void gv11b_blcg_fifo_load_gating_prod(struct gk20a *g,
616 bool prod)
617{
618 u32 i;
619 u32 size = sizeof(gv11b_blcg_fifo) / sizeof(struct gating_desc);
620
621 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
622 return;
623
624 for (i = 0; i < size; i++) {
625 if (prod)
626 gk20a_writel(g, gv11b_blcg_fifo[i].addr,
627 gv11b_blcg_fifo[i].prod);
628 else
629 gk20a_writel(g, gv11b_blcg_fifo[i].addr,
630 gv11b_blcg_fifo[i].disable);
631 }
632}
633
634void gv11b_blcg_gr_load_gating_prod(struct gk20a *g,
635 bool prod)
636{
637 u32 i;
638 u32 size = sizeof(gv11b_blcg_gr) / sizeof(struct gating_desc);
639
640 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
641 return;
642
643 for (i = 0; i < size; i++) {
644 if (prod)
645 gk20a_writel(g, gv11b_blcg_gr[i].addr,
646 gv11b_blcg_gr[i].prod);
647 else
648 gk20a_writel(g, gv11b_blcg_gr[i].addr,
649 gv11b_blcg_gr[i].disable);
650 }
651}
652
653void gv11b_blcg_ltc_load_gating_prod(struct gk20a *g,
654 bool prod)
655{
656 u32 i;
657 u32 size = sizeof(gv11b_blcg_ltc) / sizeof(struct gating_desc);
658
659 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
660 return;
661
662 for (i = 0; i < size; i++) {
663 if (prod)
664 gk20a_writel(g, gv11b_blcg_ltc[i].addr,
665 gv11b_blcg_ltc[i].prod);
666 else
667 gk20a_writel(g, gv11b_blcg_ltc[i].addr,
668 gv11b_blcg_ltc[i].disable);
669 }
670}
671
672void gv11b_blcg_pwr_csb_load_gating_prod(struct gk20a *g,
673 bool prod)
674{
675 u32 i;
676 u32 size = sizeof(gv11b_blcg_pwr_csb) / sizeof(struct gating_desc);
677
678 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
679 return;
680
681 for (i = 0; i < size; i++) {
682 if (prod)
683 gk20a_writel(g, gv11b_blcg_pwr_csb[i].addr,
684 gv11b_blcg_pwr_csb[i].prod);
685 else
686 gk20a_writel(g, gv11b_blcg_pwr_csb[i].addr,
687 gv11b_blcg_pwr_csb[i].disable);
688 }
689}
690
691void gv11b_blcg_pmu_load_gating_prod(struct gk20a *g,
692 bool prod)
693{
694 u32 i;
695 u32 size = sizeof(gv11b_blcg_pmu) / sizeof(struct gating_desc);
696
697 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
698 return;
699
700 for (i = 0; i < size; i++) {
701 if (prod)
702 gk20a_writel(g, gv11b_blcg_pmu[i].addr,
703 gv11b_blcg_pmu[i].prod);
704 else
705 gk20a_writel(g, gv11b_blcg_pmu[i].addr,
706 gv11b_blcg_pmu[i].disable);
707 }
708}
709
710void gv11b_blcg_xbar_load_gating_prod(struct gk20a *g,
711 bool prod)
712{
713 u32 i;
714 u32 size = sizeof(gv11b_blcg_xbar) / sizeof(struct gating_desc);
715
716 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
717 return;
718
719 for (i = 0; i < size; i++) {
720 if (prod)
721 gk20a_writel(g, gv11b_blcg_xbar[i].addr,
722 gv11b_blcg_xbar[i].prod);
723 else
724 gk20a_writel(g, gv11b_blcg_xbar[i].addr,
725 gv11b_blcg_xbar[i].disable);
726 }
727}
728
729void gr_gv11b_pg_gr_load_gating_prod(struct gk20a *g,
730 bool prod)
731{
732 u32 i;
733 u32 size = sizeof(gv11b_pg_gr) / sizeof(struct gating_desc);
734
735 if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG))
736 return;
737
738 for (i = 0; i < size; i++) {
739 if (prod)
740 gk20a_writel(g, gv11b_pg_gr[i].addr,
741 gv11b_pg_gr[i].prod);
742 else
743 gk20a_writel(g, gv11b_pg_gr[i].addr,
744 gv11b_pg_gr[i].disable);
745 }
746}
747
748#endif /* __gv11b_gating_reglist_h__ */
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h
new file mode 100644
index 00000000..233189e0
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h
@@ -0,0 +1,99 @@
1/*
2 * Copyright (c) 2016, NVIDIA Corporation. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include "gk20a/gk20a.h"
24
25void gv11b_slcg_bus_load_gating_prod(struct gk20a *g,
26 bool prod);
27
28void gv11b_slcg_ce2_load_gating_prod(struct gk20a *g,
29 bool prod);
30
31void gv11b_slcg_chiplet_load_gating_prod(struct gk20a *g,
32 bool prod);
33
34void gv11b_slcg_ctxsw_firmware_load_gating_prod(struct gk20a *g,
35 bool prod);
36
37void gv11b_slcg_fb_load_gating_prod(struct gk20a *g,
38 bool prod);
39
40void gv11b_slcg_fifo_load_gating_prod(struct gk20a *g,
41 bool prod);
42
43void gr_gv11b_slcg_gr_load_gating_prod(struct gk20a *g,
44 bool prod);
45
46void ltc_gv11b_slcg_ltc_load_gating_prod(struct gk20a *g,
47 bool prod);
48
49void gv11b_slcg_perf_load_gating_prod(struct gk20a *g,
50 bool prod);
51
52void gv11b_slcg_priring_load_gating_prod(struct gk20a *g,
53 bool prod);
54
55void gv11b_slcg_pwr_csb_load_gating_prod(struct gk20a *g,
56 bool prod);
57
58void gv11b_slcg_pmu_load_gating_prod(struct gk20a *g,
59 bool prod);
60
61void gv11b_slcg_therm_load_gating_prod(struct gk20a *g,
62 bool prod);
63
64void gv11b_slcg_xbar_load_gating_prod(struct gk20a *g,
65 bool prod);
66
67void gv11b_blcg_bus_load_gating_prod(struct gk20a *g,
68 bool prod);
69
70void gv11b_blcg_ce_load_gating_prod(struct gk20a *g,
71 bool prod);
72
73void gv11b_blcg_ctxsw_firmware_load_gating_prod(struct gk20a *g,
74 bool prod);
75
76void gv11b_blcg_fb_load_gating_prod(struct gk20a *g,
77 bool prod);
78
79void gv11b_blcg_fifo_load_gating_prod(struct gk20a *g,
80 bool prod);
81
82void gv11b_blcg_gr_load_gating_prod(struct gk20a *g,
83 bool prod);
84
85void gv11b_blcg_ltc_load_gating_prod(struct gk20a *g,
86 bool prod);
87
88void gv11b_blcg_pwr_csb_load_gating_prod(struct gk20a *g,
89 bool prod);
90
91void gv11b_blcg_pmu_load_gating_prod(struct gk20a *g,
92 bool prod);
93
94void gv11b_blcg_xbar_load_gating_prod(struct gk20a *g,
95 bool prod);
96
97void gr_gv11b_pg_gr_load_gating_prod(struct gk20a *g,
98 bool prod);
99
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
new file mode 100644
index 00000000..fc059caa
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
@@ -0,0 +1,778 @@
1/*
2 * GV11B Tegra HAL interface
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26#include <linux/printk.h>
27
28#include <linux/types.h>
29#include <linux/tegra_gpu_t19x.h>
30
31#include "gk20a/gk20a.h"
32#include "gk20a/fifo_gk20a.h"
33#include "gk20a/fecs_trace_gk20a.h"
34#include "gk20a/css_gr_gk20a.h"
35#include "gk20a/mc_gk20a.h"
36#include "gk20a/mm_gk20a.h"
37#include "gk20a/dbg_gpu_gk20a.h"
38#include "gk20a/bus_gk20a.h"
39#include "gk20a/flcn_gk20a.h"
40#include "gk20a/regops_gk20a.h"
41#include "gk20a/fb_gk20a.h"
42#include "gk20a/pmu_gk20a.h"
43#include "gk20a/gr_gk20a.h"
44
45#include "gm20b/ltc_gm20b.h"
46#include "gm20b/gr_gm20b.h"
47#include "gm20b/fb_gm20b.h"
48#include "gm20b/fifo_gm20b.h"
49#include "gm20b/mm_gm20b.h"
50#include "gm20b/acr_gm20b.h"
51#include "gm20b/pmu_gm20b.h"
52
53#include "gp10b/ltc_gp10b.h"
54#include "gp10b/therm_gp10b.h"
55#include "gp10b/mc_gp10b.h"
56#include "gp10b/ce_gp10b.h"
57#include "gp10b/priv_ring_gp10b.h"
58#include "gp10b/fifo_gp10b.h"
59#include "gp10b/fecs_trace_gp10b.h"
60#include "gp10b/fb_gp10b.h"
61#include "gp10b/mm_gp10b.h"
62#include "gp10b/pmu_gp10b.h"
63#include "gp10b/gr_gp10b.h"
64
65#include "gp106/pmu_gp106.h"
66#include "gp106/acr_gp106.h"
67
68#include "gv100/gr_gv100.h"
69
70#include "dbg_gpu_gv11b.h"
71#include "hal_gv11b.h"
72#include "css_gr_gv11b.h"
73#include "gr_gv11b.h"
74#include "mc_gv11b.h"
75#include "ltc_gv11b.h"
76#include "gv11b.h"
77#include "ce_gv11b.h"
78#include "gr_ctx_gv11b.h"
79#include "mm_gv11b.h"
80#include "pmu_gv11b.h"
81#include "acr_gv11b.h"
82#include "fb_gv11b.h"
83#include "fifo_gv11b.h"
84#include "gv11b_gating_reglist.h"
85#include "regops_gv11b.h"
86#include "subctx_gv11b.h"
87#include "therm_gv11b.h"
88
89#include <nvgpu/bus.h>
90#include <nvgpu/debug.h>
91#include <nvgpu/enabled.h>
92#include <nvgpu/ctxsw_trace.h>
93
94#include <nvgpu/hw/gv11b/hw_proj_gv11b.h>
95#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
96#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
97#include <nvgpu/hw/gv11b/hw_top_gv11b.h>
98#include <nvgpu/hw/gv11b/hw_pwr_gv11b.h>
99#include <nvgpu/hw/gv11b/hw_fuse_gv11b.h>
100
101int gv11b_get_litter_value(struct gk20a *g, int value)
102{
103 int ret = EINVAL;
104 switch (value) {
105 case GPU_LIT_NUM_GPCS:
106 ret = proj_scal_litter_num_gpcs_v();
107 break;
108 case GPU_LIT_NUM_PES_PER_GPC:
109 ret = proj_scal_litter_num_pes_per_gpc_v();
110 break;
111 case GPU_LIT_NUM_ZCULL_BANKS:
112 ret = proj_scal_litter_num_zcull_banks_v();
113 break;
114 case GPU_LIT_NUM_TPC_PER_GPC:
115 ret = proj_scal_litter_num_tpc_per_gpc_v();
116 break;
117 case GPU_LIT_NUM_SM_PER_TPC:
118 ret = proj_scal_litter_num_sm_per_tpc_v();
119 break;
120 case GPU_LIT_NUM_FBPS:
121 ret = proj_scal_litter_num_fbps_v();
122 break;
123 case GPU_LIT_GPC_BASE:
124 ret = proj_gpc_base_v();
125 break;
126 case GPU_LIT_GPC_STRIDE:
127 ret = proj_gpc_stride_v();
128 break;
129 case GPU_LIT_GPC_SHARED_BASE:
130 ret = proj_gpc_shared_base_v();
131 break;
132 case GPU_LIT_TPC_IN_GPC_BASE:
133 ret = proj_tpc_in_gpc_base_v();
134 break;
135 case GPU_LIT_TPC_IN_GPC_STRIDE:
136 ret = proj_tpc_in_gpc_stride_v();
137 break;
138 case GPU_LIT_TPC_IN_GPC_SHARED_BASE:
139 ret = proj_tpc_in_gpc_shared_base_v();
140 break;
141 case GPU_LIT_PPC_IN_GPC_BASE:
142 ret = proj_ppc_in_gpc_base_v();
143 break;
144 case GPU_LIT_PPC_IN_GPC_SHARED_BASE:
145 ret = proj_ppc_in_gpc_shared_base_v();
146 break;
147 case GPU_LIT_PPC_IN_GPC_STRIDE:
148 ret = proj_ppc_in_gpc_stride_v();
149 break;
150 case GPU_LIT_ROP_BASE:
151 ret = proj_rop_base_v();
152 break;
153 case GPU_LIT_ROP_STRIDE:
154 ret = proj_rop_stride_v();
155 break;
156 case GPU_LIT_ROP_SHARED_BASE:
157 ret = proj_rop_shared_base_v();
158 break;
159 case GPU_LIT_HOST_NUM_ENGINES:
160 ret = proj_host_num_engines_v();
161 break;
162 case GPU_LIT_HOST_NUM_PBDMA:
163 ret = proj_host_num_pbdma_v();
164 break;
165 case GPU_LIT_LTC_STRIDE:
166 ret = proj_ltc_stride_v();
167 break;
168 case GPU_LIT_LTS_STRIDE:
169 ret = proj_lts_stride_v();
170 break;
171 case GPU_LIT_SM_PRI_STRIDE:
172 ret = proj_sm_stride_v();
173 break;
174 case GPU_LIT_SMPC_PRI_BASE:
175 ret = proj_smpc_base_v();
176 break;
177 case GPU_LIT_SMPC_PRI_SHARED_BASE:
178 ret = proj_smpc_shared_base_v();
179 break;
180 case GPU_LIT_SMPC_PRI_UNIQUE_BASE:
181 ret = proj_smpc_unique_base_v();
182 break;
183 case GPU_LIT_SMPC_PRI_STRIDE:
184 ret = proj_smpc_stride_v();
185 break;
186 /* Even though GV11B doesn't have an FBPA unit, the HW reports one,
187 * and the microcode as a result leaves space in the context buffer
188 * for one, so make sure SW accounts for this also.
189 */
190 case GPU_LIT_NUM_FBPAS:
191 ret = proj_scal_litter_num_fbpas_v();
192 break;
193 /* Hardcode FBPA values other than NUM_FBPAS to 0. */
194 case GPU_LIT_FBPA_STRIDE:
195 case GPU_LIT_FBPA_BASE:
196 case GPU_LIT_FBPA_SHARED_BASE:
197 ret = 0;
198 break;
199 case GPU_LIT_TWOD_CLASS:
200 ret = FERMI_TWOD_A;
201 break;
202 case GPU_LIT_THREED_CLASS:
203 ret = VOLTA_A;
204 break;
205 case GPU_LIT_COMPUTE_CLASS:
206 ret = VOLTA_COMPUTE_A;
207 break;
208 case GPU_LIT_GPFIFO_CLASS:
209 ret = VOLTA_CHANNEL_GPFIFO_A;
210 break;
211 case GPU_LIT_I2M_CLASS:
212 ret = KEPLER_INLINE_TO_MEMORY_B;
213 break;
214 case GPU_LIT_DMA_COPY_CLASS:
215 ret = VOLTA_DMA_COPY_A;
216 break;
217
218 default:
219 nvgpu_err(g, "Missing definition %d", value);
220 BUG();
221 break;
222 }
223
224 return ret;
225}
226
227static const struct gpu_ops gv11b_ops = {
228 .ltc = {
229 .determine_L2_size_bytes = gp10b_determine_L2_size_bytes,
230 .set_zbc_s_entry = gv11b_ltc_set_zbc_stencil_entry,
231 .set_zbc_color_entry = gm20b_ltc_set_zbc_color_entry,
232 .set_zbc_depth_entry = gm20b_ltc_set_zbc_depth_entry,
233 .init_cbc = NULL,
234 .init_fs_state = gv11b_ltc_init_fs_state,
235 .init_comptags = gp10b_ltc_init_comptags,
236 .cbc_ctrl = gm20b_ltc_cbc_ctrl,
237 .isr = gv11b_ltc_isr,
238 .cbc_fix_config = gv11b_ltc_cbc_fix_config,
239 .flush = gm20b_flush_ltc,
240 .set_enabled = gp10b_ltc_set_enabled,
241 },
242 .ce2 = {
243 .isr_stall = gv11b_ce_isr,
244 .isr_nonstall = gp10b_ce_nonstall_isr,
245 .get_num_pce = gv11b_ce_get_num_pce,
246 },
247 .gr = {
248 .get_patch_slots = gr_gv100_get_patch_slots,
249 .init_gpc_mmu = gr_gv11b_init_gpc_mmu,
250 .bundle_cb_defaults = gr_gv11b_bundle_cb_defaults,
251 .cb_size_default = gr_gv11b_cb_size_default,
252 .calc_global_ctx_buffer_size =
253 gr_gv11b_calc_global_ctx_buffer_size,
254 .commit_global_attrib_cb = gr_gv11b_commit_global_attrib_cb,
255 .commit_global_bundle_cb = gr_gp10b_commit_global_bundle_cb,
256 .commit_global_cb_manager = gr_gp10b_commit_global_cb_manager,
257 .commit_global_pagepool = gr_gp10b_commit_global_pagepool,
258 .handle_sw_method = gr_gv11b_handle_sw_method,
259 .set_alpha_circular_buffer_size =
260 gr_gv11b_set_alpha_circular_buffer_size,
261 .set_circular_buffer_size = gr_gv11b_set_circular_buffer_size,
262 .enable_hww_exceptions = gr_gv11b_enable_hww_exceptions,
263 .is_valid_class = gr_gv11b_is_valid_class,
264 .is_valid_gfx_class = gr_gv11b_is_valid_gfx_class,
265 .is_valid_compute_class = gr_gv11b_is_valid_compute_class,
266 .get_sm_dsm_perf_regs = gv11b_gr_get_sm_dsm_perf_regs,
267 .get_sm_dsm_perf_ctrl_regs = gv11b_gr_get_sm_dsm_perf_ctrl_regs,
268 .init_fs_state = gr_gv11b_init_fs_state,
269 .set_hww_esr_report_mask = gv11b_gr_set_hww_esr_report_mask,
270 .falcon_load_ucode = gr_gm20b_load_ctxsw_ucode_segments,
271 .load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode,
272 .set_gpc_tpc_mask = gr_gv11b_set_gpc_tpc_mask,
273 .get_gpc_tpc_mask = gr_gm20b_get_gpc_tpc_mask,
274 .free_channel_ctx = gk20a_free_channel_ctx,
275 .alloc_obj_ctx = gk20a_alloc_obj_ctx,
276 .bind_ctxsw_zcull = gr_gk20a_bind_ctxsw_zcull,
277 .get_zcull_info = gr_gk20a_get_zcull_info,
278 .is_tpc_addr = gr_gm20b_is_tpc_addr,
279 .get_tpc_num = gr_gm20b_get_tpc_num,
280 .detect_sm_arch = gr_gv11b_detect_sm_arch,
281 .add_zbc_color = gr_gp10b_add_zbc_color,
282 .add_zbc_depth = gr_gp10b_add_zbc_depth,
283 .zbc_set_table = gk20a_gr_zbc_set_table,
284 .zbc_query_table = gr_gk20a_query_zbc,
285 .pmu_save_zbc = gk20a_pmu_save_zbc,
286 .add_zbc = gr_gk20a_add_zbc,
287 .pagepool_default_size = gr_gv11b_pagepool_default_size,
288 .init_ctx_state = gr_gp10b_init_ctx_state,
289 .alloc_gr_ctx = gr_gp10b_alloc_gr_ctx,
290 .free_gr_ctx = gr_gp10b_free_gr_ctx,
291 .update_ctxsw_preemption_mode =
292 gr_gp10b_update_ctxsw_preemption_mode,
293 .dump_gr_regs = gr_gv11b_dump_gr_status_regs,
294 .update_pc_sampling = gr_gm20b_update_pc_sampling,
295 .get_fbp_en_mask = gr_gm20b_get_fbp_en_mask,
296 .get_max_ltc_per_fbp = gr_gm20b_get_max_ltc_per_fbp,
297 .get_max_lts_per_ltc = gr_gm20b_get_max_lts_per_ltc,
298 .get_rop_l2_en_mask = gr_gm20b_rop_l2_en_mask,
299 .get_max_fbps_count = gr_gm20b_get_max_fbps_count,
300 .init_sm_dsm_reg_info = gv11b_gr_init_sm_dsm_reg_info,
301 .wait_empty = gr_gv11b_wait_empty,
302 .init_cyclestats = gr_gm20b_init_cyclestats,
303 .set_sm_debug_mode = gv11b_gr_set_sm_debug_mode,
304 .enable_cde_in_fecs = gr_gm20b_enable_cde_in_fecs,
305 .bpt_reg_info = gv11b_gr_bpt_reg_info,
306 .get_access_map = gr_gv11b_get_access_map,
307 .handle_fecs_error = gr_gv11b_handle_fecs_error,
308 .handle_sm_exception = gr_gk20a_handle_sm_exception,
309 .handle_tex_exception = gr_gv11b_handle_tex_exception,
310 .enable_gpc_exceptions = gr_gv11b_enable_gpc_exceptions,
311 .enable_exceptions = gr_gv11b_enable_exceptions,
312 .get_lrf_tex_ltc_dram_override = get_ecc_override_val,
313 .update_smpc_ctxsw_mode = gr_gk20a_update_smpc_ctxsw_mode,
314 .update_hwpm_ctxsw_mode = gr_gk20a_update_hwpm_ctxsw_mode,
315 .record_sm_error_state = gv11b_gr_record_sm_error_state,
316 .update_sm_error_state = gv11b_gr_update_sm_error_state,
317 .clear_sm_error_state = gm20b_gr_clear_sm_error_state,
318 .suspend_contexts = gr_gp10b_suspend_contexts,
319 .resume_contexts = gr_gk20a_resume_contexts,
320 .get_preemption_mode_flags = gr_gp10b_get_preemption_mode_flags,
321 .init_sm_id_table = gr_gv100_init_sm_id_table,
322 .load_smid_config = gr_gv11b_load_smid_config,
323 .program_sm_id_numbering = gr_gv11b_program_sm_id_numbering,
324 .is_ltcs_ltss_addr = gr_gm20b_is_ltcs_ltss_addr,
325 .is_ltcn_ltss_addr = gr_gm20b_is_ltcn_ltss_addr,
326 .split_lts_broadcast_addr = gr_gm20b_split_lts_broadcast_addr,
327 .split_ltc_broadcast_addr = gr_gm20b_split_ltc_broadcast_addr,
328 .setup_rop_mapping = gr_gv11b_setup_rop_mapping,
329 .program_zcull_mapping = gr_gv11b_program_zcull_mapping,
330 .commit_global_timeslice = gr_gv11b_commit_global_timeslice,
331 .commit_inst = gr_gv11b_commit_inst,
332 .write_zcull_ptr = gr_gv11b_write_zcull_ptr,
333 .write_pm_ptr = gr_gv11b_write_pm_ptr,
334 .init_elcg_mode = gr_gv11b_init_elcg_mode,
335 .load_tpc_mask = gr_gv11b_load_tpc_mask,
336 .inval_icache = gr_gk20a_inval_icache,
337 .trigger_suspend = gv11b_gr_sm_trigger_suspend,
338 .wait_for_pause = gr_gk20a_wait_for_pause,
339 .resume_from_pause = gv11b_gr_resume_from_pause,
340 .clear_sm_errors = gr_gk20a_clear_sm_errors,
341 .tpc_enabled_exceptions = gr_gk20a_tpc_enabled_exceptions,
342 .get_esr_sm_sel = gv11b_gr_get_esr_sm_sel,
343 .sm_debugger_attached = gv11b_gr_sm_debugger_attached,
344 .suspend_single_sm = gv11b_gr_suspend_single_sm,
345 .suspend_all_sms = gv11b_gr_suspend_all_sms,
346 .resume_single_sm = gv11b_gr_resume_single_sm,
347 .resume_all_sms = gv11b_gr_resume_all_sms,
348 .get_sm_hww_warp_esr = gv11b_gr_get_sm_hww_warp_esr,
349 .get_sm_hww_global_esr = gv11b_gr_get_sm_hww_global_esr,
350 .get_sm_no_lock_down_hww_global_esr_mask =
351 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask,
352 .lock_down_sm = gv11b_gr_lock_down_sm,
353 .wait_for_sm_lock_down = gv11b_gr_wait_for_sm_lock_down,
354 .clear_sm_hww = gv11b_gr_clear_sm_hww,
355 .init_ovr_sm_dsm_perf = gv11b_gr_init_ovr_sm_dsm_perf,
356 .get_ovr_perf_regs = gv11b_gr_get_ovr_perf_regs,
357 .disable_rd_coalesce = gm20a_gr_disable_rd_coalesce,
358 .set_boosted_ctx = gr_gp10b_set_boosted_ctx,
359 .set_preemption_mode = gr_gp10b_set_preemption_mode,
360 .set_czf_bypass = NULL,
361 .pre_process_sm_exception = gr_gv11b_pre_process_sm_exception,
362 .set_preemption_buffer_va = gr_gv11b_set_preemption_buffer_va,
363 .init_preemption_state = NULL,
364 .update_boosted_ctx = gr_gp10b_update_boosted_ctx,
365 .set_bes_crop_debug3 = gr_gp10b_set_bes_crop_debug3,
366 .create_gr_sysfs = gr_gv11b_create_sysfs,
367 .set_ctxsw_preemption_mode = gr_gp10b_set_ctxsw_preemption_mode,
368 .is_etpc_addr = gv11b_gr_pri_is_etpc_addr,
369 .egpc_etpc_priv_addr_table = gv11b_gr_egpc_etpc_priv_addr_table,
370 .handle_tpc_mpc_exception = gr_gv11b_handle_tpc_mpc_exception,
371 .zbc_s_query_table = gr_gv11b_zbc_s_query_table,
372 .load_zbc_s_default_tbl = gr_gv11b_load_stencil_default_tbl,
373 .handle_gpc_gpcmmu_exception =
374 gr_gv11b_handle_gpc_gpcmmu_exception,
375 .add_zbc_type_s = gr_gv11b_add_zbc_type_s,
376 .get_egpc_base = gv11b_gr_get_egpc_base,
377 .get_egpc_etpc_num = gv11b_gr_get_egpc_etpc_num,
378 .handle_gpc_gpccs_exception =
379 gr_gv11b_handle_gpc_gpccs_exception,
380 .load_zbc_s_tbl = gr_gv11b_load_stencil_tbl,
381 .access_smpc_reg = gv11b_gr_access_smpc_reg,
382 .is_egpc_addr = gv11b_gr_pri_is_egpc_addr,
383 .add_zbc_s = gr_gv11b_add_zbc_stencil,
384 .handle_gcc_exception = gr_gv11b_handle_gcc_exception,
385 .init_sw_veid_bundle = gr_gv11b_init_sw_veid_bundle,
386 .handle_tpc_sm_ecc_exception =
387 gr_gv11b_handle_tpc_sm_ecc_exception,
388 .decode_egpc_addr = gv11b_gr_decode_egpc_addr,
389 .init_ctxsw_hdr_data = gr_gp10b_init_ctxsw_hdr_data,
390 },
391 .fb = {
392 .reset = gv11b_fb_reset,
393 .init_hw = gk20a_fb_init_hw,
394 .init_fs_state = gv11b_fb_init_fs_state,
395 .init_cbc = gv11b_fb_init_cbc,
396 .set_mmu_page_size = gm20b_fb_set_mmu_page_size,
397 .set_use_full_comp_tag_line =
398 gm20b_fb_set_use_full_comp_tag_line,
399 .compression_page_size = gp10b_fb_compression_page_size,
400 .compressible_page_size = gp10b_fb_compressible_page_size,
401 .vpr_info_fetch = gm20b_fb_vpr_info_fetch,
402 .dump_vpr_wpr_info = gm20b_fb_dump_vpr_wpr_info,
403 .read_wpr_info = gm20b_fb_read_wpr_info,
404 .is_debug_mode_enabled = gm20b_fb_debug_mode_enabled,
405 .set_debug_mode = gm20b_fb_set_debug_mode,
406 .tlb_invalidate = gk20a_fb_tlb_invalidate,
407 .hub_isr = gv11b_fb_hub_isr,
408 .mem_unlock = NULL,
409 },
410 .clock_gating = {
411 .slcg_bus_load_gating_prod =
412 gv11b_slcg_bus_load_gating_prod,
413 .slcg_ce2_load_gating_prod =
414 gv11b_slcg_ce2_load_gating_prod,
415 .slcg_chiplet_load_gating_prod =
416 gv11b_slcg_chiplet_load_gating_prod,
417 .slcg_ctxsw_firmware_load_gating_prod =
418 gv11b_slcg_ctxsw_firmware_load_gating_prod,
419 .slcg_fb_load_gating_prod =
420 gv11b_slcg_fb_load_gating_prod,
421 .slcg_fifo_load_gating_prod =
422 gv11b_slcg_fifo_load_gating_prod,
423 .slcg_gr_load_gating_prod =
424 gr_gv11b_slcg_gr_load_gating_prod,
425 .slcg_ltc_load_gating_prod =
426 ltc_gv11b_slcg_ltc_load_gating_prod,
427 .slcg_perf_load_gating_prod =
428 gv11b_slcg_perf_load_gating_prod,
429 .slcg_priring_load_gating_prod =
430 gv11b_slcg_priring_load_gating_prod,
431 .slcg_pmu_load_gating_prod =
432 gv11b_slcg_pmu_load_gating_prod,
433 .slcg_therm_load_gating_prod =
434 gv11b_slcg_therm_load_gating_prod,
435 .slcg_xbar_load_gating_prod =
436 gv11b_slcg_xbar_load_gating_prod,
437 .blcg_bus_load_gating_prod =
438 gv11b_blcg_bus_load_gating_prod,
439 .blcg_ce_load_gating_prod =
440 gv11b_blcg_ce_load_gating_prod,
441 .blcg_ctxsw_firmware_load_gating_prod =
442 gv11b_blcg_ctxsw_firmware_load_gating_prod,
443 .blcg_fb_load_gating_prod =
444 gv11b_blcg_fb_load_gating_prod,
445 .blcg_fifo_load_gating_prod =
446 gv11b_blcg_fifo_load_gating_prod,
447 .blcg_gr_load_gating_prod =
448 gv11b_blcg_gr_load_gating_prod,
449 .blcg_ltc_load_gating_prod =
450 gv11b_blcg_ltc_load_gating_prod,
451 .blcg_pwr_csb_load_gating_prod =
452 gv11b_blcg_pwr_csb_load_gating_prod,
453 .blcg_pmu_load_gating_prod =
454 gv11b_blcg_pmu_load_gating_prod,
455 .blcg_xbar_load_gating_prod =
456 gv11b_blcg_xbar_load_gating_prod,
457 .pg_gr_load_gating_prod =
458 gr_gv11b_pg_gr_load_gating_prod,
459 },
460 .fifo = {
461 .get_preempt_timeout = gv11b_fifo_get_preempt_timeout,
462 .init_fifo_setup_hw = gv11b_init_fifo_setup_hw,
463 .bind_channel = channel_gm20b_bind,
464 .unbind_channel = channel_gv11b_unbind,
465 .disable_channel = gk20a_fifo_disable_channel,
466 .enable_channel = gk20a_fifo_enable_channel,
467 .alloc_inst = gk20a_fifo_alloc_inst,
468 .free_inst = gk20a_fifo_free_inst,
469 .setup_ramfc = channel_gv11b_setup_ramfc,
470 .channel_set_timeslice = gk20a_fifo_set_timeslice,
471 .default_timeslice_us = gk20a_fifo_default_timeslice_us,
472 .setup_userd = gk20a_fifo_setup_userd,
473 .userd_gp_get = gv11b_userd_gp_get,
474 .userd_gp_put = gv11b_userd_gp_put,
475 .userd_pb_get = gv11b_userd_pb_get,
476 .pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
477 .preempt_channel = gv11b_fifo_preempt_channel,
478 .preempt_tsg = gv11b_fifo_preempt_tsg,
479 .enable_tsg = gv11b_fifo_enable_tsg,
480 .disable_tsg = gk20a_disable_tsg,
481 .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status,
482 .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload,
483 .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted,
484 .update_runlist = gk20a_fifo_update_runlist,
485 .trigger_mmu_fault = NULL,
486 .get_mmu_fault_info = NULL,
487 .wait_engine_idle = gk20a_fifo_wait_engine_idle,
488 .get_num_fifos = gv11b_fifo_get_num_fifos,
489 .get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
490 .set_runlist_interleave = gk20a_fifo_set_runlist_interleave,
491 .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
492 .force_reset_ch = gk20a_fifo_force_reset_ch,
493 .engine_enum_from_type = gp10b_fifo_engine_enum_from_type,
494 .device_info_data_parse = gp10b_device_info_data_parse,
495 .eng_runlist_base_size = fifo_eng_runlist_base__size_1_v,
496 .init_engine_info = gk20a_fifo_init_engine_info,
497 .runlist_entry_size = ram_rl_entry_size_v,
498 .get_tsg_runlist_entry = gv11b_get_tsg_runlist_entry,
499 .get_ch_runlist_entry = gv11b_get_ch_runlist_entry,
500 .is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
501 .dump_pbdma_status = gk20a_dump_pbdma_status,
502 .dump_eng_status = gv11b_dump_eng_status,
503 .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc,
504 .intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
505 .is_preempt_pending = gv11b_fifo_is_preempt_pending,
506 .init_pbdma_intr_descs = gv11b_fifo_init_pbdma_intr_descs,
507 .reset_enable_hw = gv11b_init_fifo_reset_enable_hw,
508 .teardown_ch_tsg = gv11b_fifo_teardown_ch_tsg,
509 .handle_sched_error = gv11b_fifo_handle_sched_error,
510 .handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0,
511 .handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
512 .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
513 .deinit_eng_method_buffers =
514 gv11b_fifo_deinit_eng_method_buffers,
515 .tsg_bind_channel = gk20a_tsg_bind_channel,
516 .tsg_unbind_channel = gk20a_tsg_unbind_channel,
517#ifdef CONFIG_TEGRA_GK20A_NVHOST
518 .alloc_syncpt_buf = gv11b_fifo_alloc_syncpt_buf,
519 .free_syncpt_buf = gv11b_fifo_free_syncpt_buf,
520 .add_syncpt_wait_cmd = gv11b_fifo_add_syncpt_wait_cmd,
521 .get_syncpt_wait_cmd_size = gv11b_fifo_get_syncpt_wait_cmd_size,
522 .add_syncpt_incr_cmd = gv11b_fifo_add_syncpt_incr_cmd,
523 .get_syncpt_incr_cmd_size = gv11b_fifo_get_syncpt_incr_cmd_size,
524#endif
525 .resetup_ramfc = NULL,
526 .device_info_fault_id = top_device_info_data_fault_id_enum_v,
527 .free_channel_ctx_header = gv11b_free_subctx_header,
528 .preempt_ch_tsg = gv11b_fifo_preempt_ch_tsg,
529 .handle_ctxsw_timeout = gv11b_fifo_handle_ctxsw_timeout,
530 },
531 .gr_ctx = {
532 .get_netlist_name = gr_gv11b_get_netlist_name,
533 .is_fw_defined = gr_gv11b_is_firmware_defined,
534 },
535#ifdef CONFIG_GK20A_CTXSW_TRACE
536 .fecs_trace = {
537 .alloc_user_buffer = NULL,
538 .free_user_buffer = NULL,
539 .mmap_user_buffer = NULL,
540 .init = NULL,
541 .deinit = NULL,
542 .enable = NULL,
543 .disable = NULL,
544 .is_enabled = NULL,
545 .reset = NULL,
546 .flush = NULL,
547 .poll = NULL,
548 .bind_channel = NULL,
549 .unbind_channel = NULL,
550 .max_entries = NULL,
551 },
552#endif /* CONFIG_GK20A_CTXSW_TRACE */
553 .mm = {
554 .support_sparse = gm20b_mm_support_sparse,
555 .gmmu_map = gk20a_locked_gmmu_map,
556 .gmmu_unmap = gk20a_locked_gmmu_unmap,
557 .vm_bind_channel = gk20a_vm_bind_channel,
558 .fb_flush = gk20a_mm_fb_flush,
559 .l2_invalidate = gk20a_mm_l2_invalidate,
560 .l2_flush = gv11b_mm_l2_flush,
561 .cbc_clean = gk20a_mm_cbc_clean,
562 .set_big_page_size = gm20b_mm_set_big_page_size,
563 .get_big_page_sizes = gm20b_mm_get_big_page_sizes,
564 .get_default_big_page_size = gp10b_mm_get_default_big_page_size,
565 .gpu_phys_addr = gv11b_gpu_phys_addr,
566 .get_iommu_bit = gp10b_mm_get_iommu_bit,
567 .get_mmu_levels = gp10b_mm_get_mmu_levels,
568 .init_pdb = gp10b_mm_init_pdb,
569 .init_mm_setup_hw = gv11b_init_mm_setup_hw,
570 .is_bar1_supported = gv11b_mm_is_bar1_supported,
571 .alloc_inst_block = gk20a_alloc_inst_block,
572 .init_inst_block = gv11b_init_inst_block,
573 .mmu_fault_pending = gv11b_mm_mmu_fault_pending,
574 .get_kind_invalid = gm20b_get_kind_invalid,
575 .get_kind_pitch = gm20b_get_kind_pitch,
576 .init_bar2_vm = gb10b_init_bar2_vm,
577 .init_bar2_mm_hw_setup = gv11b_init_bar2_mm_hw_setup,
578 .remove_bar2_vm = gv11b_mm_remove_bar2_vm,
579 .fault_info_mem_destroy = gv11b_mm_fault_info_mem_destroy,
580 },
581 .therm = {
582 .init_therm_setup_hw = gp10b_init_therm_setup_hw,
583 .elcg_init_idle_filters = gv11b_elcg_init_idle_filters,
584 },
585 .pmu = {
586 .pmu_setup_elpg = gp10b_pmu_setup_elpg,
587 .pmu_get_queue_head = pwr_pmu_queue_head_r,
588 .pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v,
589 .pmu_get_queue_tail = pwr_pmu_queue_tail_r,
590 .pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v,
591 .pmu_queue_head = gk20a_pmu_queue_head,
592 .pmu_queue_tail = gk20a_pmu_queue_tail,
593 .pmu_msgq_tail = gk20a_pmu_msgq_tail,
594 .pmu_mutex_size = pwr_pmu_mutex__size_1_v,
595 .pmu_mutex_acquire = gk20a_pmu_mutex_acquire,
596 .pmu_mutex_release = gk20a_pmu_mutex_release,
597 .write_dmatrfbase = gp10b_write_dmatrfbase,
598 .pmu_elpg_statistics = gp106_pmu_elpg_statistics,
599 .pmu_pg_init_param = gv11b_pg_gr_init,
600 .pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list,
601 .pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list,
602 .dump_secure_fuses = pmu_dump_security_fuses_gp10b,
603 .reset_engine = gp106_pmu_engine_reset,
604 .is_engine_in_reset = gp106_pmu_is_engine_in_reset,
605 .pmu_nsbootstrap = gv11b_pmu_bootstrap,
606 .pmu_pg_set_sub_feature_mask = gv11b_pg_set_subfeature_mask,
607 .is_pmu_supported = gv11b_is_pmu_supported,
608 },
609 .regops = {
610 .get_global_whitelist_ranges =
611 gv11b_get_global_whitelist_ranges,
612 .get_global_whitelist_ranges_count =
613 gv11b_get_global_whitelist_ranges_count,
614 .get_context_whitelist_ranges =
615 gv11b_get_context_whitelist_ranges,
616 .get_context_whitelist_ranges_count =
617 gv11b_get_context_whitelist_ranges_count,
618 .get_runcontrol_whitelist = gv11b_get_runcontrol_whitelist,
619 .get_runcontrol_whitelist_count =
620 gv11b_get_runcontrol_whitelist_count,
621 .get_runcontrol_whitelist_ranges =
622 gv11b_get_runcontrol_whitelist_ranges,
623 .get_runcontrol_whitelist_ranges_count =
624 gv11b_get_runcontrol_whitelist_ranges_count,
625 .get_qctl_whitelist = gv11b_get_qctl_whitelist,
626 .get_qctl_whitelist_count = gv11b_get_qctl_whitelist_count,
627 .get_qctl_whitelist_ranges = gv11b_get_qctl_whitelist_ranges,
628 .get_qctl_whitelist_ranges_count =
629 gv11b_get_qctl_whitelist_ranges_count,
630 .apply_smpc_war = gv11b_apply_smpc_war,
631 },
632 .mc = {
633 .intr_enable = mc_gv11b_intr_enable,
634 .intr_unit_config = mc_gp10b_intr_unit_config,
635 .isr_stall = mc_gp10b_isr_stall,
636 .intr_stall = mc_gp10b_intr_stall,
637 .intr_stall_pause = mc_gp10b_intr_stall_pause,
638 .intr_stall_resume = mc_gp10b_intr_stall_resume,
639 .intr_nonstall = mc_gp10b_intr_nonstall,
640 .intr_nonstall_pause = mc_gp10b_intr_nonstall_pause,
641 .intr_nonstall_resume = mc_gp10b_intr_nonstall_resume,
642 .enable = gk20a_mc_enable,
643 .disable = gk20a_mc_disable,
644 .reset = gk20a_mc_reset,
645 .boot_0 = gk20a_mc_boot_0,
646 .is_intr1_pending = mc_gp10b_is_intr1_pending,
647 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
648 },
649 .debug = {
650 .show_dump = gk20a_debug_show_dump,
651 },
652 .dbg_session_ops = {
653 .exec_reg_ops = exec_regops_gk20a,
654 .dbg_set_powergate = dbg_set_powergate,
655 .check_and_set_global_reservation =
656 nvgpu_check_and_set_global_reservation,
657 .check_and_set_context_reservation =
658 nvgpu_check_and_set_context_reservation,
659 .release_profiler_reservation =
660 nvgpu_release_profiler_reservation,
661 .perfbuffer_enable = gv11b_perfbuf_enable_locked,
662 .perfbuffer_disable = gv11b_perfbuf_disable_locked,
663 },
664 .bus = {
665 .init_hw = gk20a_bus_init_hw,
666 .isr = gk20a_bus_isr,
667 .read_ptimer = gk20a_read_ptimer,
668 .get_timestamps_zipper = nvgpu_get_timestamps_zipper,
669 .bar1_bind = NULL,
670 },
671#if defined(CONFIG_GK20A_CYCLE_STATS)
672 .css = {
673 .enable_snapshot = gv11b_css_hw_enable_snapshot,
674 .disable_snapshot = gv11b_css_hw_disable_snapshot,
675 .check_data_available = gv11b_css_hw_check_data_available,
676 .set_handled_snapshots = css_hw_set_handled_snapshots,
677 .allocate_perfmon_ids = css_gr_allocate_perfmon_ids,
678 .release_perfmon_ids = css_gr_release_perfmon_ids,
679 },
680#endif
681 .falcon = {
682 .falcon_hal_sw_init = gk20a_falcon_hal_sw_init,
683 },
684 .priv_ring = {
685 .isr = gp10b_priv_ring_isr,
686 },
687 .chip_init_gpu_characteristics = gv11b_init_gpu_characteristics,
688 .get_litter_value = gv11b_get_litter_value,
689};
690
691int gv11b_init_hal(struct gk20a *g)
692{
693 struct gpu_ops *gops = &g->ops;
694 u32 val;
695 bool priv_security;
696
697 gops->ltc = gv11b_ops.ltc;
698 gops->ce2 = gv11b_ops.ce2;
699 gops->gr = gv11b_ops.gr;
700 gops->fb = gv11b_ops.fb;
701 gops->clock_gating = gv11b_ops.clock_gating;
702 gops->fifo = gv11b_ops.fifo;
703 gops->gr_ctx = gv11b_ops.gr_ctx;
704 gops->mm = gv11b_ops.mm;
705#ifdef CONFIG_GK20A_CTXSW_TRACE
706 gops->fecs_trace = gv11b_ops.fecs_trace;
707#endif
708 gops->therm = gv11b_ops.therm;
709 gops->pmu = gv11b_ops.pmu;
710 gops->regops = gv11b_ops.regops;
711 gops->mc = gv11b_ops.mc;
712 gops->debug = gv11b_ops.debug;
713 gops->dbg_session_ops = gv11b_ops.dbg_session_ops;
714 gops->bus = gv11b_ops.bus;
715#if defined(CONFIG_GK20A_CYCLE_STATS)
716 gops->css = gv11b_ops.css;
717#endif
718 gops->falcon = gv11b_ops.falcon;
719 gops->priv_ring = gv11b_ops.priv_ring;
720
721 /* Lone functions */
722 gops->chip_init_gpu_characteristics =
723 gv11b_ops.chip_init_gpu_characteristics;
724 gops->get_litter_value = gv11b_ops.get_litter_value;
725
726 val = gk20a_readl(g, fuse_opt_priv_sec_en_r());
727 if (val) {
728 priv_security = true;
729 pr_err("priv security is enabled\n");
730 } else {
731 priv_security = false;
732 pr_err("priv security is disabled\n");
733 }
734 __nvgpu_set_enabled(g, NVGPU_GR_USE_DMA_FOR_FW_BOOTSTRAP, false);
735 __nvgpu_set_enabled(g, NVGPU_SEC_PRIVSECURITY, priv_security);
736 __nvgpu_set_enabled(g, NVGPU_SEC_SECUREGPCCS, priv_security);
737
738 /* priv security dependent ops */
739 if (nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
740 /* Add in ops from gm20b acr */
741 gops->pmu.prepare_ucode = gp106_prepare_ucode_blob,
742 gops->pmu.pmu_setup_hw_and_bootstrap = gv11b_bootstrap_hs_flcn,
743 gops->pmu.get_wpr = gm20b_wpr_info,
744 gops->pmu.alloc_blob_space = gm20b_alloc_blob_space,
745 gops->pmu.pmu_populate_loader_cfg =
746 gp106_pmu_populate_loader_cfg,
747 gops->pmu.flcn_populate_bl_dmem_desc =
748 gp106_flcn_populate_bl_dmem_desc,
749 gops->pmu.falcon_wait_for_halt = pmu_wait_for_halt,
750 gops->pmu.falcon_clear_halt_interrupt_status =
751 clear_halt_interrupt_status,
752 gops->pmu.init_falcon_setup_hw = gv11b_init_pmu_setup_hw1,
753
754 gops->pmu.init_wpr_region = gm20b_pmu_init_acr;
755 gops->pmu.load_lsfalcon_ucode = gp10b_load_falcon_ucode;
756 gops->pmu.is_lazy_bootstrap = gv11b_is_lazy_bootstrap,
757 gops->pmu.is_priv_load = gv11b_is_priv_load,
758
759 gops->gr.load_ctxsw_ucode = gr_gm20b_load_ctxsw_ucode;
760 } else {
761 /* Inherit from gk20a */
762 gops->pmu.prepare_ucode = nvgpu_pmu_prepare_ns_ucode_blob,
763 gops->pmu.pmu_setup_hw_and_bootstrap = gk20a_init_pmu_setup_hw1,
764
765 gops->pmu.load_lsfalcon_ucode = NULL;
766 gops->pmu.init_wpr_region = NULL;
767 gops->pmu.pmu_setup_hw_and_bootstrap = gp10b_init_pmu_setup_hw1;
768
769 gops->gr.load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode;
770 }
771
772 __nvgpu_set_enabled(g, NVGPU_PMU_FECS_BOOTSTRAP_DONE, false);
773 g->bootstrap_owner = LSF_BOOTSTRAP_OWNER_DEFAULT;
774
775 g->name = "gv11b";
776
777 return 0;
778}
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.h b/drivers/gpu/nvgpu/gv11b/hal_gv11b.h
new file mode 100644
index 00000000..668353dc
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.h
@@ -0,0 +1,31 @@
1/*
2 * GV11B Tegra HAL interface
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef _NVGPU_HAL_GV11B_H
26#define _NVGPU_HAL_GV11B_H
27struct gk20a;
28
29int gv11b_init_hal(struct gk20a *gops);
30int gv11b_get_litter_value(struct gk20a *g, int value);
31#endif
diff --git a/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c
new file mode 100644
index 00000000..a199e024
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c
@@ -0,0 +1,205 @@
1/*
2 * GV11B LTC
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "gk20a/gk20a.h"
26#include "gp10b/ltc_gp10b.h"
27
28#include "ltc_gv11b.h"
29
30#include <nvgpu/hw/gv11b/hw_ltc_gv11b.h>
31#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
32#include <nvgpu/hw/gv11b/hw_top_gv11b.h>
33#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
34#include <nvgpu/hw/gv11b/hw_pri_ringmaster_gv11b.h>
35
36/*
37 * Sets the ZBC stencil for the passed index.
38 */
39void gv11b_ltc_set_zbc_stencil_entry(struct gk20a *g,
40 struct zbc_entry *stencil_val,
41 u32 index)
42{
43 u32 real_index = index + GK20A_STARTOF_ZBC_TABLE;
44
45 gk20a_writel(g, ltc_ltcs_ltss_dstg_zbc_index_r(),
46 ltc_ltcs_ltss_dstg_zbc_index_address_f(real_index));
47
48 gk20a_writel(g, ltc_ltcs_ltss_dstg_zbc_stencil_clear_value_r(),
49 stencil_val->depth);
50
51 gk20a_readl(g, ltc_ltcs_ltss_dstg_zbc_index_r());
52}
53
54void gv11b_ltc_init_fs_state(struct gk20a *g)
55{
56 u32 ltc_intr;
57 u32 reg;
58
59 gk20a_dbg_info("initialize gv11b l2");
60
61 g->ops.mc.reset(g, mc_enable_pfb_enabled_f() |
62 mc_enable_l2_enabled_f());
63
64 reg = gk20a_readl(g, mc_elpg_enable_r());
65 reg |= mc_elpg_enable_l2_enabled_f();
66 gk20a_writel(g, mc_elpg_enable_r(), reg);
67
68 g->max_ltc_count = gk20a_readl(g, top_num_ltcs_r());
69 g->ltc_count = gk20a_readl(g, pri_ringmaster_enum_ltc_r());
70 gk20a_dbg_info("%u ltcs out of %u", g->ltc_count, g->max_ltc_count);
71
72 gk20a_writel(g, ltc_ltcs_ltss_dstg_cfg0_r(),
73 gk20a_readl(g, ltc_ltc0_lts0_dstg_cfg0_r()) |
74 ltc_ltcs_ltss_dstg_cfg0_vdc_4to2_disable_m());
75
76 /* Disable LTC interrupts */
77 reg = gk20a_readl(g, ltc_ltcs_ltss_intr_r());
78 reg &= ~ltc_ltcs_ltss_intr_en_evicted_cb_m();
79 reg &= ~ltc_ltcs_ltss_intr_en_illegal_compstat_access_m();
80 gk20a_writel(g, ltc_ltcs_ltss_intr_r(), reg);
81
82 /* Enable ECC interrupts */
83 ltc_intr = gk20a_readl(g, ltc_ltcs_ltss_intr_r());
84 ltc_intr |= ltc_ltcs_ltss_intr_en_ecc_sec_error_enabled_f() |
85 ltc_ltcs_ltss_intr_en_ecc_ded_error_enabled_f();
86 gk20a_writel(g, ltc_ltcs_ltss_intr_r(),
87 ltc_intr);
88}
89
90void gv11b_ltc_isr(struct gk20a *g)
91{
92 u32 mc_intr, ltc_intr3;
93 unsigned int ltc, slice;
94 u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE);
95 u32 lts_stride = nvgpu_get_litter_value(g, GPU_LIT_LTS_STRIDE);
96 u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt;
97 u32 corrected_delta, uncorrected_delta;
98 u32 corrected_overflow, uncorrected_overflow;
99 u32 ltc_corrected, ltc_uncorrected;
100
101 mc_intr = gk20a_readl(g, mc_intr_ltc_r());
102 for (ltc = 0; ltc < g->ltc_count; ltc++) {
103 if ((mc_intr & 1 << ltc) == 0)
104 continue;
105 ltc_corrected = ltc_uncorrected = 0;
106
107 for (slice = 0; slice < g->gr.slices_per_ltc; slice++) {
108 u32 offset = ltc_stride * ltc + lts_stride * slice;
109 ltc_intr3 = gk20a_readl(g, ltc_ltc0_lts0_intr3_r() +
110 offset);
111
112 /* Detect and handle ECC PARITY errors */
113
114 if (ltc_intr3 &
115 (ltc_ltcs_ltss_intr3_ecc_uncorrected_m() |
116 ltc_ltcs_ltss_intr3_ecc_corrected_m())) {
117
118 ecc_status = gk20a_readl(g,
119 ltc_ltc0_lts0_l2_cache_ecc_status_r() +
120 offset);
121 ecc_addr = gk20a_readl(g,
122 ltc_ltc0_lts0_l2_cache_ecc_address_r() +
123 offset);
124 corrected_cnt = gk20a_readl(g,
125 ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_r() + offset);
126 uncorrected_cnt = gk20a_readl(g,
127 ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_r() + offset);
128
129 corrected_delta =
130 ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_total_v(corrected_cnt);
131 uncorrected_delta =
132 ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_total_v(uncorrected_cnt);
133 corrected_overflow = ecc_status &
134 ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_total_counter_overflow_m();
135
136 uncorrected_overflow = ecc_status &
137 ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_total_counter_overflow_m();
138
139 /* clear the interrupt */
140 if ((corrected_delta > 0) || corrected_overflow) {
141 gk20a_writel(g, ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_r() + offset, 0);
142 }
143 if ((uncorrected_delta > 0) || uncorrected_overflow) {
144 gk20a_writel(g,
145 ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_r() + offset, 0);
146 }
147
148 gk20a_writel(g, ltc_ltc0_lts0_l2_cache_ecc_status_r() + offset,
149 ltc_ltc0_lts0_l2_cache_ecc_status_reset_task_f());
150
151 /* update counters per slice */
152 if (corrected_overflow)
153 corrected_delta += (0x1UL << ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_total_s());
154 if (uncorrected_overflow)
155 uncorrected_delta += (0x1UL << ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_total_s());
156
157 ltc_corrected += corrected_delta;
158 ltc_uncorrected += uncorrected_delta;
159 nvgpu_log(g, gpu_dbg_intr,
160 "ltc:%d lts: %d cache ecc interrupt intr: 0x%x", ltc, slice, ltc_intr3);
161
162 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_rstg_m())
163 nvgpu_log(g, gpu_dbg_intr, "rstg ecc error corrected");
164 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_rstg_m())
165 nvgpu_log(g, gpu_dbg_intr, "rstg ecc error uncorrected");
166 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_tstg_m())
167 nvgpu_log(g, gpu_dbg_intr, "tstg ecc error corrected");
168 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_tstg_m())
169 nvgpu_log(g, gpu_dbg_intr, "tstg ecc error uncorrected");
170 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_dstg_m())
171 nvgpu_log(g, gpu_dbg_intr, "dstg ecc error corrected");
172 if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_dstg_m())
173 nvgpu_log(g, gpu_dbg_intr, "dstg ecc error uncorrected");
174
175 if (corrected_overflow || uncorrected_overflow)
176 nvgpu_info(g, "ecc counter overflow!");
177
178 nvgpu_log(g, gpu_dbg_intr,
179 "ecc error address: 0x%x", ecc_addr);
180
181 }
182
183 }
184 g->ecc.ltc.t19x.l2_cache_corrected_err_count.counters[ltc] +=
185 ltc_corrected;
186 g->ecc.ltc.t19x.l2_cache_uncorrected_err_count.counters[ltc] +=
187 ltc_uncorrected;
188
189 }
190
191 /* fallback to other interrupts */
192 gp10b_ltc_isr(g);
193}
194
195u32 gv11b_ltc_cbc_fix_config(struct gk20a *g, int base)
196{
197 u32 val = gk20a_readl(g, ltc_ltcs_ltss_cbc_num_active_ltcs_r());
198
199 if (ltc_ltcs_ltss_cbc_num_active_ltcs__v(val) == 2)
200 return base * 2;
201 else if (ltc_ltcs_ltss_cbc_num_active_ltcs__v(val) != 1) {
202 nvgpu_err(g, "Invalid number of active ltcs: %08x", val);
203 }
204 return base;
205}
diff --git a/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h
new file mode 100644
index 00000000..9b46e74c
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h
@@ -0,0 +1,34 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef LTC_GV11B_H
24#define LTC_GV11B_H
25struct gk20a;
26
27void gv11b_ltc_set_zbc_stencil_entry(struct gk20a *g,
28 struct zbc_entry *stencil_val,
29 u32 index);
30void gv11b_ltc_init_fs_state(struct gk20a *g);
31void gv11b_ltc_isr(struct gk20a *g);
32u32 gv11b_ltc_cbc_fix_config(struct gk20a *g, int base);
33
34#endif
diff --git a/drivers/gpu/nvgpu/gv11b/mc_gv11b.c b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c
new file mode 100644
index 00000000..74c5c4d6
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c
@@ -0,0 +1,92 @@
1/*
2 * GV11B master
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26
27#include "gk20a/gk20a.h"
28
29#include "gp10b/mc_gp10b.h"
30
31#include "mc_gv11b.h"
32#include "fb_gv11b.h"
33
34#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
35
36void mc_gv11b_intr_enable(struct gk20a *g)
37{
38 u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g);
39
40 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING),
41 0xffffffff);
42 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING),
43 0xffffffff);
44 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL);
45
46 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] =
47 mc_intr_pfifo_pending_f() |
48 mc_intr_hub_pending_f() |
49 mc_intr_priv_ring_pending_f() |
50 mc_intr_pbus_pending_f() |
51 mc_intr_ltc_pending_f() |
52 eng_intr_mask;
53
54 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] =
55 mc_intr_pfifo_pending_f()
56 | eng_intr_mask;
57
58 /* TODO: Enable PRI faults for HUB ECC err intr */
59 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types);
60
61 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING),
62 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]);
63
64 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING),
65 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]);
66
67}
68
69bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0)
70{
71 return ((mc_intr_0 & mc_intr_hub_pending_f()) ? true : false);
72}
73
74bool gv11b_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id)
75{
76 u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0));
77 u32 stall_intr, eng_intr_mask;
78
79 eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id);
80 if (mc_intr_0 & eng_intr_mask)
81 return true;
82
83 stall_intr = mc_intr_pfifo_pending_f() |
84 mc_intr_hub_pending_f() |
85 mc_intr_priv_ring_pending_f() |
86 mc_intr_pbus_pending_f() |
87 mc_intr_ltc_pending_f();
88 if (mc_intr_0 & stall_intr)
89 return true;
90
91 return false;
92}
diff --git a/drivers/gpu/nvgpu/gv11b/mc_gv11b.h b/drivers/gpu/nvgpu/gv11b/mc_gv11b.h
new file mode 100644
index 00000000..eb9d0e4e
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/mc_gv11b.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef MC_GV11B_H
24#define MC_GV11B_H
25struct gk20a;
26
27void mc_gv11b_intr_enable(struct gk20a *g);
28bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0);
29bool gv11b_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id);
30#endif
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
new file mode 100644
index 00000000..fdc506ac
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c
@@ -0,0 +1,330 @@
1/*
2 * GV11B MMU
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/pm_runtime.h>
26
27#include <nvgpu/kmem.h>
28#include <nvgpu/dma.h>
29#include <nvgpu/log.h>
30#include <nvgpu/mm.h>
31
32#include "gk20a/gk20a.h"
33#include "gk20a/mm_gk20a.h"
34
35#include "gp10b/mm_gp10b.h"
36#include "gp10b/mc_gp10b.h"
37
38#include "mm_gv11b.h"
39#include "fb_gv11b.h"
40
41#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
42#include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h>
43#include <nvgpu/hw/gv11b/hw_bus_gv11b.h>
44
45#define NVGPU_L3_ALLOC_BIT BIT(36)
46
47bool gv11b_mm_is_bar1_supported(struct gk20a *g)
48{
49 return false;
50}
51
52void gv11b_init_inst_block(struct nvgpu_mem *inst_block,
53 struct vm_gk20a *vm, u32 big_page_size)
54{
55 struct gk20a *g = gk20a_from_vm(vm);
56
57 gk20a_dbg_info("inst block phys = 0x%llx, kv = 0x%p",
58 nvgpu_inst_block_addr(g, inst_block), inst_block->cpu_va);
59
60 g->ops.mm.init_pdb(g, inst_block, vm);
61
62 if (big_page_size && g->ops.mm.set_big_page_size)
63 g->ops.mm.set_big_page_size(g, inst_block, big_page_size);
64}
65
66bool gv11b_mm_mmu_fault_pending(struct gk20a *g)
67{
68 return gv11b_fb_mmu_fault_pending(g);
69}
70
71void gv11b_mm_fault_info_mem_destroy(struct gk20a *g)
72{
73 nvgpu_log_fn(g, " ");
74
75 nvgpu_mutex_acquire(&g->mm.hub_isr_mutex);
76
77 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_OTHER |
78 HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY);
79
80 nvgpu_kfree(g, g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY]);
81
82 g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY] = NULL;
83 g->mm.fault_info[FAULT_TYPE_REPLAY] = NULL;
84
85 nvgpu_mutex_release(&g->mm.hub_isr_mutex);
86 nvgpu_mutex_destroy(&g->mm.hub_isr_mutex);
87}
88
89static int gv11b_mm_mmu_fault_info_buf_init(struct gk20a *g,
90 u32 *hub_intr_types)
91{
92 struct mmu_fault_info *fault_info_mem;
93
94 fault_info_mem = nvgpu_kzalloc(g, sizeof(struct mmu_fault_info) *
95 FAULT_TYPE_NUM);
96 if (!fault_info_mem) {
97 nvgpu_log_info(g, "failed to alloc shadow fault info");
98 return -ENOMEM;
99 }
100 /* shadow buffer for copying mmu fault info */
101 g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY] =
102 &fault_info_mem[FAULT_TYPE_OTHER_AND_NONREPLAY];
103
104 g->mm.fault_info[FAULT_TYPE_REPLAY] =
105 &fault_info_mem[FAULT_TYPE_REPLAY];
106
107 *hub_intr_types |= HUB_INTR_TYPE_OTHER;
108 return 0;
109}
110
111static void gv11b_mm_mmu_hw_fault_buf_init(struct gk20a *g,
112 u32 *hub_intr_types)
113{
114 struct vm_gk20a *vm = g->mm.bar2.vm;
115 int err = 0;
116 size_t fb_size;
117
118 /* Max entries take care of 1 entry used for full detection */
119 fb_size = (g->ops.fifo.get_num_fifos(g) + 1) *
120 gmmu_fault_buf_size_v();
121
122 err = nvgpu_dma_alloc_map_sys(vm, fb_size,
123 &g->mm.hw_fault_buf[FAULT_TYPE_OTHER_AND_NONREPLAY]);
124 if (err) {
125 nvgpu_err(g,
126 "Error in hw mmu fault buf [0] alloc in bar2 vm ");
127 /* Fault will be snapped in pri reg but not in buffer */
128 return;
129 }
130
131 g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] =
132 HW_FAULT_BUF_STATUS_ALLOC_TRUE;
133 *hub_intr_types |= HUB_INTR_TYPE_NONREPLAY;
134
135 err = nvgpu_dma_alloc_map_sys(vm, fb_size,
136 &g->mm.hw_fault_buf[FAULT_TYPE_REPLAY]);
137 if (err) {
138 nvgpu_err(g,
139 "Error in hw mmu fault buf [1] alloc in bar2 vm ");
140 /* Fault will be snapped in pri reg but not in buffer */
141 return;
142 }
143 g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] =
144 HW_FAULT_BUF_STATUS_ALLOC_TRUE;
145 *hub_intr_types |= HUB_INTR_TYPE_REPLAY;
146}
147
148static void gv11b_mm_mmu_hw_fault_buf_deinit(struct gk20a *g)
149{
150 struct vm_gk20a *vm = g->mm.bar2.vm;
151
152 nvgpu_log_fn(g, " ");
153
154 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_NONREPLAY |
155 HUB_INTR_TYPE_REPLAY);
156
157 g->mm.hub_intr_types &= (~(HUB_INTR_TYPE_NONREPLAY |
158 HUB_INTR_TYPE_REPLAY));
159
160 if ((gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX))) {
161 gv11b_fb_fault_buf_set_state_hw(g, NONREPLAY_REG_INDEX,
162 FAULT_BUF_DISABLED);
163 }
164
165 if ((gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX))) {
166 gv11b_fb_fault_buf_set_state_hw(g, REPLAY_REG_INDEX,
167 FAULT_BUF_DISABLED);
168 }
169
170 if (g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] ==
171 HW_FAULT_BUF_STATUS_ALLOC_TRUE) {
172 nvgpu_dma_unmap_free(vm,
173 &g->mm.hw_fault_buf[FAULT_TYPE_OTHER_AND_NONREPLAY]);
174 g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] =
175 HW_FAULT_BUF_STATUS_ALLOC_FALSE;
176 }
177
178 if (g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] ==
179 HW_FAULT_BUF_STATUS_ALLOC_TRUE) {
180 nvgpu_dma_unmap_free(vm,
181 &g->mm.hw_fault_buf[FAULT_TYPE_REPLAY]);
182 g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] =
183 HW_FAULT_BUF_STATUS_ALLOC_FALSE;
184 }
185}
186
187void gv11b_mm_remove_bar2_vm(struct gk20a *g)
188{
189 struct mm_gk20a *mm = &g->mm;
190
191 nvgpu_log_fn(g, " ");
192
193 gv11b_mm_mmu_hw_fault_buf_deinit(g);
194
195 nvgpu_free_inst_block(g, &mm->bar2.inst_block);
196 nvgpu_vm_put(mm->bar2.vm);
197}
198
199static void gv11b_mm_mmu_fault_setup_hw(struct gk20a *g)
200{
201 if (g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] ==
202 HW_FAULT_BUF_STATUS_ALLOC_TRUE) {
203 gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX);
204 }
205 if (g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] ==
206 HW_FAULT_BUF_STATUS_ALLOC_TRUE) {
207 gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX);
208 }
209}
210
211static int gv11b_mm_mmu_fault_setup_sw(struct gk20a *g)
212{
213 int err;
214
215 nvgpu_log_fn(g, " ");
216
217 nvgpu_mutex_init(&g->mm.hub_isr_mutex);
218
219 g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] =
220 HW_FAULT_BUF_STATUS_ALLOC_FALSE;
221 g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] =
222 HW_FAULT_BUF_STATUS_ALLOC_FALSE;
223
224 g->mm.hub_intr_types = HUB_INTR_TYPE_ECC_UNCORRECTED;
225
226 err = gv11b_mm_mmu_fault_info_buf_init(g, &g->mm.hub_intr_types);
227
228 if (!err)
229 gv11b_mm_mmu_hw_fault_buf_init(g, &g->mm.hub_intr_types);
230
231 return err;
232}
233
234int gv11b_init_mm_setup_hw(struct gk20a *g)
235{
236 int err = 0;
237
238 nvgpu_log_fn(g, " ");
239
240 g->ops.fb.set_mmu_page_size(g);
241 g->ops.fb.init_hw(g);
242
243 err = g->ops.mm.init_bar2_mm_hw_setup(g);
244 if (err)
245 return err;
246
247 if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g))
248 return -EBUSY;
249
250 err = gv11b_mm_mmu_fault_setup_sw(g);
251 if (!err)
252 gv11b_mm_mmu_fault_setup_hw(g);
253
254 nvgpu_log_fn(g, "end");
255
256 return err;
257}
258
259void gv11b_mm_l2_flush(struct gk20a *g, bool invalidate)
260{
261 nvgpu_log(g, gpu_dbg_fn, "gv11b_mm_l2_flush");
262
263 g->ops.mm.fb_flush(g);
264 gk20a_mm_l2_flush(g, invalidate);
265 g->ops.mm.fb_flush(g);
266}
267
268/*
269 * On Volta the GPU determines whether to do L3 allocation for a mapping by
270 * checking bit 36 of the phsyical address. So if a mapping should allocte lines
271 * in the L3 this bit must be set.
272 */
273u64 gv11b_gpu_phys_addr(struct gk20a *g,
274 struct nvgpu_gmmu_attrs *attrs, u64 phys)
275{
276 if (attrs && attrs->t19x_attrs.l3_alloc)
277 return phys | NVGPU_L3_ALLOC_BIT;
278
279 return phys;
280}
281
282int gv11b_init_bar2_mm_hw_setup(struct gk20a *g)
283{
284 struct mm_gk20a *mm = &g->mm;
285 struct nvgpu_mem *inst_block = &mm->bar2.inst_block;
286 u64 inst_pa = nvgpu_inst_block_addr(g, inst_block);
287 u32 reg_val;
288 struct nvgpu_timeout timeout;
289 u32 delay = GR_IDLE_CHECK_DEFAULT;
290
291 nvgpu_log_fn(g, " ");
292
293 g->ops.fb.set_mmu_page_size(g);
294
295 inst_pa = (u32)(inst_pa >> bus_bar2_block_ptr_shift_v());
296 nvgpu_log_info(g, "bar2 inst block ptr: 0x%08x", (u32)inst_pa);
297
298 gk20a_writel(g, bus_bar2_block_r(),
299 nvgpu_aperture_mask(g, inst_block,
300 bus_bar2_block_target_sys_mem_ncoh_f(),
301 bus_bar2_block_target_vid_mem_f()) |
302 bus_bar2_block_mode_virtual_f() |
303 bus_bar2_block_ptr_f(inst_pa));
304
305 /* This is needed as BAR1 support is removed and there is no way
306 * to know if gpu successfully accessed memory.
307 * To avoid deadlocks and non-deterministic virtual address translation
308 * behavior, after writing BAR2_BLOCK to bind BAR2 to a virtual address
309 * space, SW must ensure that the bind has completed prior to issuing
310 * any further BAR2 requests by polling for both
311 * BUS_BIND_STATUS_BAR2_PENDING to return to EMPTY and
312 * BUS_BIND_STATUS_BAR2_OUTSTANDING to return to FALSE
313 */
314 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
315 NVGPU_TIMER_CPU_TIMER);
316 nvgpu_log_info(g, "check bar2 bind status");
317 do {
318 reg_val = gk20a_readl(g, bus_bind_status_r());
319
320 if (!((reg_val & bus_bind_status_bar2_pending_busy_f()) ||
321 (reg_val & bus_bind_status_bar2_outstanding_true_f())))
322 return 0;
323
324 nvgpu_usleep_range(delay, delay * 2);
325 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
326 } while (!nvgpu_timeout_expired_msg(&timeout, "bar2 bind timedout"));
327
328 nvgpu_err(g, "bar2 bind failed. gpu unable to access memory");
329 return -EBUSY;
330}
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.h b/drivers/gpu/nvgpu/gv11b/mm_gv11b.h
new file mode 100644
index 00000000..d830b7cc
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.h
@@ -0,0 +1,46 @@
1/*
2 * GV11B MM
3 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24#ifndef MM_GV11B_H
25#define MM_GV11B_H
26
27#define HW_FAULT_BUF_STATUS_ALLOC_TRUE 1
28#define HW_FAULT_BUF_STATUS_ALLOC_FALSE 0
29
30struct gk20a;
31struct nvgpu_mem;
32struct vm_gk20a;
33
34bool gv11b_mm_is_bar1_supported(struct gk20a *g);
35void gv11b_init_inst_block(struct nvgpu_mem *inst_block,
36 struct vm_gk20a *vm, u32 big_page_size);
37bool gv11b_mm_mmu_fault_pending(struct gk20a *g);
38void gv11b_mm_remove_bar2_vm(struct gk20a *g);
39int gv11b_init_mm_setup_hw(struct gk20a *g);
40int gv11b_init_bar2_mm_hw_setup(struct gk20a *g);
41void gv11b_mm_l2_flush(struct gk20a *g, bool invalidate);
42u64 gv11b_gpu_phys_addr(struct gk20a *g,
43 struct nvgpu_gmmu_attrs *attrs, u64 phys);
44void gv11b_mm_fault_info_mem_destroy(struct gk20a *g);
45
46#endif
diff --git a/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c b/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c
new file mode 100644
index 00000000..95d82254
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c
@@ -0,0 +1,549 @@
1/*
2 * GV11B Tegra Platform Interface
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/of_platform.h>
26#include <linux/debugfs.h>
27#include <linux/dma-buf.h>
28#include <linux/nvmap.h>
29#include <linux/reset.h>
30#include <linux/hashtable.h>
31#include <linux/clk.h>
32#include <nvgpu/nvhost.h>
33#include <nvgpu/nvhost_t19x.h>
34
35#include <uapi/linux/nvgpu.h>
36
37#include <soc/tegra/tegra_bpmp.h>
38#include <soc/tegra/tegra_powergate.h>
39
40#include "gk20a/gk20a.h"
41#include "common/linux/platform_gk20a.h"
42#include "common/linux/clk.h"
43
44#include "gp10b/platform_gp10b.h"
45#include "common/linux/platform_gp10b_tegra.h"
46
47#include "common/linux/os_linux.h"
48#include "common/linux/platform_gk20a_tegra.h"
49#include "gr_gv11b.h"
50#include "nvgpu_gpuid_t19x.h"
51
52static void gr_gv11b_remove_sysfs(struct device *dev);
53
54static int gv11b_tegra_probe(struct device *dev)
55{
56 struct gk20a_platform *platform = dev_get_drvdata(dev);
57#ifdef CONFIG_TEGRA_GK20A_NVHOST
58 struct gk20a *g = platform->g;
59 int err = 0;
60
61 err = nvgpu_get_nvhost_dev(g);
62 if (err) {
63 dev_err(dev, "host1x device not available");
64 return err;
65 }
66
67 err = nvgpu_nvhost_syncpt_unit_interface_get_aperture(
68 g->nvhost_dev,
69 &g->syncpt_unit_base,
70 &g->syncpt_unit_size);
71 if (err) {
72 dev_err(dev, "Failed to get syncpt interface");
73 return -ENOSYS;
74 }
75 g->syncpt_size = nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(1);
76 gk20a_dbg_info("syncpt_unit_base %llx syncpt_unit_size %zx size %x\n",
77 g->syncpt_unit_base, g->syncpt_unit_size,
78 g->syncpt_size);
79#endif
80
81 platform->bypass_smmu = !device_is_iommuable(dev);
82 platform->disable_bigpage = platform->bypass_smmu;
83
84 platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close
85 = false;
86 platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close
87 = false;
88
89 platform->g->gr.t18x.ctx_vars.force_preemption_gfxp = false;
90 platform->g->gr.t18x.ctx_vars.force_preemption_cilp = false;
91
92 gp10b_tegra_get_clocks(dev);
93 nvgpu_linux_init_clk_support(platform->g);
94
95 return 0;
96}
97
98static int gv11b_tegra_remove(struct device *dev)
99{
100 gp10b_tegra_remove(dev);
101
102 gr_gv11b_remove_sysfs(dev);
103
104 return 0;
105}
106
107static bool gv11b_tegra_is_railgated(struct device *dev)
108{
109 bool ret = false;
110#ifdef TEGRA194_POWER_DOMAIN_GPU
111 struct gk20a *g = get_gk20a(dev);
112
113 if (tegra_bpmp_running()) {
114 nvgpu_log(g, gpu_dbg_info, "bpmp running");
115 ret = !tegra_powergate_is_powered(TEGRA194_POWER_DOMAIN_GPU);
116
117 nvgpu_log(g, gpu_dbg_info, "railgated? %s", ret ? "yes" : "no");
118 } else {
119 nvgpu_log(g, gpu_dbg_info, "bpmp not running");
120 }
121#endif
122 return ret;
123}
124
125static int gv11b_tegra_railgate(struct device *dev)
126{
127#ifdef TEGRA194_POWER_DOMAIN_GPU
128 struct gk20a_platform *platform = gk20a_get_platform(dev);
129 struct gk20a *g = get_gk20a(dev);
130 int i;
131
132 if (tegra_bpmp_running()) {
133 nvgpu_log(g, gpu_dbg_info, "bpmp running");
134 if (!tegra_powergate_is_powered(TEGRA194_POWER_DOMAIN_GPU)) {
135 nvgpu_log(g, gpu_dbg_info, "powergate is not powered");
136 return 0;
137 }
138 nvgpu_log(g, gpu_dbg_info, "clk_disable_unprepare");
139 for (i = 0; i < platform->num_clks; i++) {
140 if (platform->clk[i])
141 clk_disable_unprepare(platform->clk[i]);
142 }
143 nvgpu_log(g, gpu_dbg_info, "powergate_partition");
144 tegra_powergate_partition(TEGRA194_POWER_DOMAIN_GPU);
145 } else {
146 nvgpu_log(g, gpu_dbg_info, "bpmp not running");
147 }
148#endif
149 return 0;
150}
151
152static int gv11b_tegra_unrailgate(struct device *dev)
153{
154 int ret = 0;
155#ifdef TEGRA194_POWER_DOMAIN_GPU
156 struct gk20a_platform *platform = gk20a_get_platform(dev);
157 struct gk20a *g = get_gk20a(dev);
158 int i;
159
160 if (tegra_bpmp_running()) {
161 nvgpu_log(g, gpu_dbg_info, "bpmp running");
162 ret = tegra_unpowergate_partition(TEGRA194_POWER_DOMAIN_GPU);
163 if (ret) {
164 nvgpu_log(g, gpu_dbg_info,
165 "unpowergate partition failed");
166 return ret;
167 }
168 nvgpu_log(g, gpu_dbg_info, "clk_prepare_enable");
169 for (i = 0; i < platform->num_clks; i++) {
170 if (platform->clk[i])
171 clk_prepare_enable(platform->clk[i]);
172 }
173 } else {
174 nvgpu_log(g, gpu_dbg_info, "bpmp not running");
175 }
176#endif
177 return ret;
178}
179
180static int gv11b_tegra_suspend(struct device *dev)
181{
182 return 0;
183}
184
185struct gk20a_platform t19x_gpu_tegra_platform = {
186 .has_syncpoints = true,
187
188 /* power management configuration */
189
190 /* ptimer src frequency in hz*/
191 .ptimer_src_freq = 31250000,
192
193 .probe = gv11b_tegra_probe,
194 .remove = gv11b_tegra_remove,
195
196 .enable_slcg = false,
197 .enable_blcg = false,
198 .enable_elcg = false,
199 .can_slcg = false,
200 .can_blcg = false,
201 .can_elcg = false,
202
203 /* power management callbacks */
204 .suspend = gv11b_tegra_suspend,
205 .railgate = gv11b_tegra_railgate,
206 .unrailgate = gv11b_tegra_unrailgate,
207 .is_railgated = gv11b_tegra_is_railgated,
208
209 .busy = gk20a_tegra_busy,
210 .idle = gk20a_tegra_idle,
211
212 .dump_platform_dependencies = gk20a_tegra_debug_dump,
213
214 .soc_name = "tegra19x",
215
216 .honors_aperture = true,
217 .unified_memory = true,
218
219 .reset_assert = gp10b_tegra_reset_assert,
220 .reset_deassert = gp10b_tegra_reset_deassert,
221};
222
223static struct device_attribute *dev_attr_sm_l1_tag_ecc_corrected_err_count_array;
224static struct device_attribute *dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array;
225static struct device_attribute *dev_attr_sm_cbu_ecc_corrected_err_count_array;
226static struct device_attribute *dev_attr_sm_cbu_ecc_uncorrected_err_count_array;
227static struct device_attribute *dev_attr_sm_l1_data_ecc_corrected_err_count_array;
228static struct device_attribute *dev_attr_sm_l1_data_ecc_uncorrected_err_count_array;
229static struct device_attribute *dev_attr_sm_icache_ecc_corrected_err_count_array;
230static struct device_attribute *dev_attr_sm_icache_ecc_uncorrected_err_count_array;
231static struct device_attribute *dev_attr_gcc_l15_ecc_corrected_err_count_array;
232static struct device_attribute *dev_attr_gcc_l15_ecc_uncorrected_err_count_array;
233static struct device_attribute *dev_attr_mmu_l1tlb_ecc_corrected_err_count_array;
234static struct device_attribute *dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array;
235
236static struct device_attribute *dev_attr_fecs_ecc_corrected_err_count_array;
237static struct device_attribute *dev_attr_fecs_ecc_uncorrected_err_count_array;
238static struct device_attribute *dev_attr_gpccs_ecc_corrected_err_count_array;
239static struct device_attribute *dev_attr_gpccs_ecc_uncorrected_err_count_array;
240
241static struct device_attribute *dev_attr_l2_cache_ecc_corrected_err_count_array;
242static struct device_attribute *dev_attr_l2_cache_ecc_uncorrected_err_count_array;
243
244static struct device_attribute *dev_attr_mmu_l2tlb_ecc_corrected_err_count_array;
245static struct device_attribute *dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array;
246static struct device_attribute *dev_attr_mmu_hubtlb_ecc_corrected_err_count_array;
247static struct device_attribute *dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array;
248static struct device_attribute *dev_attr_mmu_fillunit_ecc_corrected_err_count_array;
249static struct device_attribute *dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array;
250
251void gr_gv11b_create_sysfs(struct gk20a *g)
252{
253 struct device *dev = dev_from_gk20a(g);
254 int error = 0;
255 /* This stat creation function is called on GR init. GR can get
256 initialized multiple times but we only need to create the ECC
257 stats once. Therefore, add the following check to avoid
258 creating duplicate stat sysfs nodes. */
259 if (g->ecc.gr.t19x.sm_l1_tag_corrected_err_count.counters != NULL)
260 return;
261
262 gr_gp10b_create_sysfs(g);
263
264 error |= gr_gp10b_ecc_stat_create(dev,
265 0,
266 "sm_l1_tag_ecc_corrected_err_count",
267 &g->ecc.gr.t19x.sm_l1_tag_corrected_err_count,
268 &dev_attr_sm_l1_tag_ecc_corrected_err_count_array);
269
270 error |= gr_gp10b_ecc_stat_create(dev,
271 0,
272 "sm_l1_tag_ecc_uncorrected_err_count",
273 &g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count,
274 &dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array);
275
276 error |= gr_gp10b_ecc_stat_create(dev,
277 0,
278 "sm_cbu_ecc_corrected_err_count",
279 &g->ecc.gr.t19x.sm_cbu_corrected_err_count,
280 &dev_attr_sm_cbu_ecc_corrected_err_count_array);
281
282 error |= gr_gp10b_ecc_stat_create(dev,
283 0,
284 "sm_cbu_ecc_uncorrected_err_count",
285 &g->ecc.gr.t19x.sm_cbu_uncorrected_err_count,
286 &dev_attr_sm_cbu_ecc_uncorrected_err_count_array);
287
288 error |= gr_gp10b_ecc_stat_create(dev,
289 0,
290 "sm_l1_data_ecc_corrected_err_count",
291 &g->ecc.gr.t19x.sm_l1_data_corrected_err_count,
292 &dev_attr_sm_l1_data_ecc_corrected_err_count_array);
293
294 error |= gr_gp10b_ecc_stat_create(dev,
295 0,
296 "sm_l1_data_ecc_uncorrected_err_count",
297 &g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count,
298 &dev_attr_sm_l1_data_ecc_uncorrected_err_count_array);
299
300 error |= gr_gp10b_ecc_stat_create(dev,
301 0,
302 "sm_icache_ecc_corrected_err_count",
303 &g->ecc.gr.t19x.sm_icache_corrected_err_count,
304 &dev_attr_sm_icache_ecc_corrected_err_count_array);
305
306 error |= gr_gp10b_ecc_stat_create(dev,
307 0,
308 "sm_icache_ecc_uncorrected_err_count",
309 &g->ecc.gr.t19x.sm_icache_uncorrected_err_count,
310 &dev_attr_sm_icache_ecc_uncorrected_err_count_array);
311
312 error |= gr_gp10b_ecc_stat_create(dev,
313 0,
314 "gcc_l15_ecc_corrected_err_count",
315 &g->ecc.gr.t19x.gcc_l15_corrected_err_count,
316 &dev_attr_gcc_l15_ecc_corrected_err_count_array);
317
318 error |= gr_gp10b_ecc_stat_create(dev,
319 0,
320 "gcc_l15_ecc_uncorrected_err_count",
321 &g->ecc.gr.t19x.gcc_l15_uncorrected_err_count,
322 &dev_attr_gcc_l15_ecc_uncorrected_err_count_array);
323
324 error |= gp10b_ecc_stat_create(dev,
325 g->ltc_count,
326 "ltc",
327 "l2_cache_uncorrected_err_count",
328 &g->ecc.ltc.t19x.l2_cache_uncorrected_err_count,
329 &dev_attr_l2_cache_ecc_uncorrected_err_count_array);
330
331 error |= gp10b_ecc_stat_create(dev,
332 g->ltc_count,
333 "ltc",
334 "l2_cache_corrected_err_count",
335 &g->ecc.ltc.t19x.l2_cache_corrected_err_count,
336 &dev_attr_l2_cache_ecc_corrected_err_count_array);
337
338 error |= gp10b_ecc_stat_create(dev,
339 1,
340 "gpc",
341 "fecs_ecc_uncorrected_err_count",
342 &g->ecc.gr.t19x.fecs_uncorrected_err_count,
343 &dev_attr_fecs_ecc_uncorrected_err_count_array);
344
345 error |= gp10b_ecc_stat_create(dev,
346 1,
347 "gpc",
348 "fecs_ecc_corrected_err_count",
349 &g->ecc.gr.t19x.fecs_corrected_err_count,
350 &dev_attr_fecs_ecc_corrected_err_count_array);
351
352 error |= gp10b_ecc_stat_create(dev,
353 g->gr.gpc_count,
354 "gpc",
355 "gpccs_ecc_uncorrected_err_count",
356 &g->ecc.gr.t19x.gpccs_uncorrected_err_count,
357 &dev_attr_gpccs_ecc_uncorrected_err_count_array);
358
359 error |= gp10b_ecc_stat_create(dev,
360 g->gr.gpc_count,
361 "gpc",
362 "gpccs_ecc_corrected_err_count",
363 &g->ecc.gr.t19x.gpccs_corrected_err_count,
364 &dev_attr_gpccs_ecc_corrected_err_count_array);
365
366 error |= gp10b_ecc_stat_create(dev,
367 g->gr.gpc_count,
368 "gpc",
369 "mmu_l1tlb_ecc_uncorrected_err_count",
370 &g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count,
371 &dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array);
372
373 error |= gp10b_ecc_stat_create(dev,
374 g->gr.gpc_count,
375 "gpc",
376 "mmu_l1tlb_ecc_corrected_err_count",
377 &g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count,
378 &dev_attr_mmu_l1tlb_ecc_corrected_err_count_array);
379
380 error |= gp10b_ecc_stat_create(dev,
381 1,
382 "eng",
383 "mmu_l2tlb_ecc_uncorrected_err_count",
384 &g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count,
385 &dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array);
386
387 error |= gp10b_ecc_stat_create(dev,
388 1,
389 "eng",
390 "mmu_l2tlb_ecc_corrected_err_count",
391 &g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count,
392 &dev_attr_mmu_l2tlb_ecc_corrected_err_count_array);
393
394 error |= gp10b_ecc_stat_create(dev,
395 1,
396 "eng",
397 "mmu_hubtlb_ecc_uncorrected_err_count",
398 &g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count,
399 &dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array);
400
401 error |= gp10b_ecc_stat_create(dev,
402 1,
403 "eng",
404 "mmu_hubtlb_ecc_corrected_err_count",
405 &g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count,
406 &dev_attr_mmu_hubtlb_ecc_corrected_err_count_array);
407
408 error |= gp10b_ecc_stat_create(dev,
409 1,
410 "eng",
411 "mmu_fillunit_ecc_uncorrected_err_count",
412 &g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count,
413 &dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array);
414
415 error |= gp10b_ecc_stat_create(dev,
416 1,
417 "eng",
418 "mmu_fillunit_ecc_corrected_err_count",
419 &g->ecc.eng.t19x.mmu_fillunit_corrected_err_count,
420 &dev_attr_mmu_fillunit_ecc_corrected_err_count_array);
421
422 if (error)
423 dev_err(dev, "Failed to create gv11b sysfs attributes!\n");
424}
425
426static void gr_gv11b_remove_sysfs(struct device *dev)
427{
428 struct gk20a *g = get_gk20a(dev);
429
430 gr_gp10b_ecc_stat_remove(dev,
431 0,
432 &g->ecc.gr.t19x.sm_l1_tag_corrected_err_count,
433 dev_attr_sm_l1_tag_ecc_corrected_err_count_array);
434
435 gr_gp10b_ecc_stat_remove(dev,
436 0,
437 &g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count,
438 dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array);
439
440 gr_gp10b_ecc_stat_remove(dev,
441 0,
442 &g->ecc.gr.t19x.sm_cbu_corrected_err_count,
443 dev_attr_sm_cbu_ecc_corrected_err_count_array);
444
445 gr_gp10b_ecc_stat_remove(dev,
446 0,
447 &g->ecc.gr.t19x.sm_cbu_uncorrected_err_count,
448 dev_attr_sm_cbu_ecc_uncorrected_err_count_array);
449
450 gr_gp10b_ecc_stat_remove(dev,
451 0,
452 &g->ecc.gr.t19x.sm_l1_data_corrected_err_count,
453 dev_attr_sm_l1_data_ecc_corrected_err_count_array);
454
455 gr_gp10b_ecc_stat_remove(dev,
456 0,
457 &g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count,
458 dev_attr_sm_l1_data_ecc_uncorrected_err_count_array);
459
460 gr_gp10b_ecc_stat_remove(dev,
461 0,
462 &g->ecc.gr.t19x.sm_icache_corrected_err_count,
463 dev_attr_sm_icache_ecc_corrected_err_count_array);
464
465 gr_gp10b_ecc_stat_remove(dev,
466 0,
467 &g->ecc.gr.t19x.sm_icache_uncorrected_err_count,
468 dev_attr_sm_icache_ecc_uncorrected_err_count_array);
469
470 gr_gp10b_ecc_stat_remove(dev,
471 0,
472 &g->ecc.gr.t19x.gcc_l15_corrected_err_count,
473 dev_attr_gcc_l15_ecc_corrected_err_count_array);
474
475 gr_gp10b_ecc_stat_remove(dev,
476 0,
477 &g->ecc.gr.t19x.gcc_l15_uncorrected_err_count,
478 dev_attr_gcc_l15_ecc_uncorrected_err_count_array);
479
480 gp10b_ecc_stat_remove(dev,
481 g->ltc_count,
482 &g->ecc.ltc.t19x.l2_cache_uncorrected_err_count,
483 dev_attr_l2_cache_ecc_uncorrected_err_count_array);
484
485 gp10b_ecc_stat_remove(dev,
486 g->ltc_count,
487 &g->ecc.ltc.t19x.l2_cache_corrected_err_count,
488 dev_attr_l2_cache_ecc_corrected_err_count_array);
489
490 gp10b_ecc_stat_remove(dev,
491 1,
492 &g->ecc.gr.t19x.fecs_uncorrected_err_count,
493 dev_attr_fecs_ecc_uncorrected_err_count_array);
494
495 gp10b_ecc_stat_remove(dev,
496 1,
497 &g->ecc.gr.t19x.fecs_corrected_err_count,
498 dev_attr_fecs_ecc_corrected_err_count_array);
499
500 gp10b_ecc_stat_remove(dev,
501 g->gr.gpc_count,
502 &g->ecc.gr.t19x.gpccs_uncorrected_err_count,
503 dev_attr_gpccs_ecc_uncorrected_err_count_array);
504
505 gp10b_ecc_stat_remove(dev,
506 g->gr.gpc_count,
507 &g->ecc.gr.t19x.gpccs_corrected_err_count,
508 dev_attr_gpccs_ecc_corrected_err_count_array);
509
510 gp10b_ecc_stat_remove(dev,
511 g->gr.gpc_count,
512 &g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count,
513 dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array);
514
515 gp10b_ecc_stat_remove(dev,
516 g->gr.gpc_count,
517 &g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count,
518 dev_attr_mmu_l1tlb_ecc_corrected_err_count_array);
519
520 gp10b_ecc_stat_remove(dev,
521 1,
522 &g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count,
523 dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array);
524
525 gp10b_ecc_stat_remove(dev,
526 1,
527 &g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count,
528 dev_attr_mmu_l2tlb_ecc_corrected_err_count_array);
529
530 gp10b_ecc_stat_remove(dev,
531 1,
532 &g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count,
533 dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array);
534
535 gp10b_ecc_stat_remove(dev,
536 1,
537 &g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count,
538 dev_attr_mmu_hubtlb_ecc_corrected_err_count_array);
539
540 gp10b_ecc_stat_remove(dev,
541 1,
542 &g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count,
543 dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array);
544
545 gp10b_ecc_stat_remove(dev,
546 1,
547 &g->ecc.eng.t19x.mmu_fillunit_corrected_err_count,
548 dev_attr_mmu_fillunit_ecc_corrected_err_count_array);
549}
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
new file mode 100644
index 00000000..2c7b6457
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c
@@ -0,0 +1,283 @@
1/*
2 * GV11B PMU
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/delay.h> /* for udelay */
26#include <linux/clk.h>
27
28#include <soc/tegra/fuse.h>
29
30#include <nvgpu/pmu.h>
31#include <nvgpu/falcon.h>
32#include <nvgpu/enabled.h>
33#include <nvgpu/mm.h>
34
35#include "gk20a/gk20a.h"
36
37#include "gp10b/pmu_gp10b.h"
38#include "gp106/pmu_gp106.h"
39
40#include "pmu_gv11b.h"
41#include "acr_gv11b.h"
42
43#include <nvgpu/hw/gv11b/hw_pwr_gv11b.h>
44
45#define gv11b_dbg_pmu(fmt, arg...) \
46 gk20a_dbg(gpu_dbg_pmu, fmt, ##arg)
47
48#define ALIGN_4KB 12
49
50bool gv11b_is_pmu_supported(struct gk20a *g)
51{
52 return true;
53}
54
55bool gv11b_is_lazy_bootstrap(u32 falcon_id)
56{
57 bool enable_status = false;
58
59 switch (falcon_id) {
60 case LSF_FALCON_ID_FECS:
61 enable_status = true;
62 break;
63 case LSF_FALCON_ID_GPCCS:
64 enable_status = true;
65 break;
66 default:
67 break;
68 }
69
70 return enable_status;
71}
72
73bool gv11b_is_priv_load(u32 falcon_id)
74{
75 bool enable_status = false;
76
77 switch (falcon_id) {
78 case LSF_FALCON_ID_FECS:
79 enable_status = true;
80 break;
81 case LSF_FALCON_ID_GPCCS:
82 enable_status = true;
83 break;
84 default:
85 break;
86 }
87
88 return enable_status;
89}
90
91int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu)
92{
93 struct gk20a *g = gk20a_from_pmu(pmu);
94 struct mm_gk20a *mm = &g->mm;
95 struct pmu_ucode_desc *desc = pmu->desc;
96 u64 addr_code_lo, addr_data_lo, addr_load_lo;
97 u64 addr_code_hi, addr_data_hi, addr_load_hi;
98 u32 i, blocks, addr_args;
99
100 gk20a_dbg_fn("");
101
102 gk20a_writel(g, pwr_falcon_itfen_r(),
103 gk20a_readl(g, pwr_falcon_itfen_r()) |
104 pwr_falcon_itfen_ctxen_enable_f());
105
106 gk20a_writel(g, pwr_pmu_new_instblk_r(),
107 pwr_pmu_new_instblk_ptr_f(
108 nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB)
109 | pwr_pmu_new_instblk_valid_f(1)
110 | pwr_pmu_new_instblk_target_sys_ncoh_f());
111
112 /* TBD: load all other surfaces */
113 g->ops.pmu_ver.set_pmu_cmdline_args_trace_size(
114 pmu, GK20A_PMU_TRACE_BUFSIZE);
115 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base(pmu);
116 g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx(
117 pmu, GK20A_PMU_DMAIDX_VIRT);
118
119 g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq(pmu,
120 g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_PWRCLK));
121
122 addr_args = (pwr_falcon_hwcfg_dmem_size_v(
123 gk20a_readl(g, pwr_falcon_hwcfg_r()))
124 << GK20A_PMU_DMEM_BLKSIZE2) -
125 g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu);
126
127 nvgpu_flcn_copy_to_dmem(pmu->flcn, addr_args,
128 (u8 *)(g->ops.pmu_ver.get_pmu_cmdline_args_ptr(pmu)),
129 g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu), 0);
130
131 gk20a_writel(g, pwr_falcon_dmemc_r(0),
132 pwr_falcon_dmemc_offs_f(0) |
133 pwr_falcon_dmemc_blk_f(0) |
134 pwr_falcon_dmemc_aincw_f(1));
135
136 addr_code_lo = u64_lo32((pmu->ucode.gpu_va +
137 desc->app_start_offset +
138 desc->app_resident_code_offset) >> 8);
139
140 addr_code_hi = u64_hi32((pmu->ucode.gpu_va +
141 desc->app_start_offset +
142 desc->app_resident_code_offset) >> 8);
143 addr_data_lo = u64_lo32((pmu->ucode.gpu_va +
144 desc->app_start_offset +
145 desc->app_resident_data_offset) >> 8);
146 addr_data_hi = u64_hi32((pmu->ucode.gpu_va +
147 desc->app_start_offset +
148 desc->app_resident_data_offset) >> 8);
149 addr_load_lo = u64_lo32((pmu->ucode.gpu_va +
150 desc->bootloader_start_offset) >> 8);
151 addr_load_hi = u64_hi32((pmu->ucode.gpu_va +
152 desc->bootloader_start_offset) >> 8);
153
154 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
155 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
156 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
157 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
158 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
159 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
160 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
161 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
162 gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE);
163 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code_lo << 8);
164 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code_hi);
165 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_offset);
166 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size);
167 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
168 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0);
169 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry);
170 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data_lo << 8);
171 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data_hi);
172 gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size);
173 gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1);
174 gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args);
175
176 g->ops.pmu.write_dmatrfbase(g,
177 addr_load_lo - (desc->bootloader_imem_offset >> 8));
178
179 blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8;
180
181 for (i = 0; i < blocks; i++) {
182 gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(),
183 desc->bootloader_imem_offset + (i << 8));
184 gk20a_writel(g, pwr_falcon_dmatrffboffs_r(),
185 desc->bootloader_imem_offset + (i << 8));
186 gk20a_writel(g, pwr_falcon_dmatrfcmd_r(),
187 pwr_falcon_dmatrfcmd_imem_f(1) |
188 pwr_falcon_dmatrfcmd_write_f(0) |
189 pwr_falcon_dmatrfcmd_size_f(6) |
190 pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE));
191 }
192
193 nvgpu_flcn_bootstrap(pmu->flcn, desc->bootloader_entry_point);
194
195 gk20a_writel(g, pwr_falcon_os_r(), desc->app_version);
196
197 return 0;
198}
199
200static void pmu_handle_pg_sub_feature_msg(struct gk20a *g, struct pmu_msg *msg,
201 void *param, u32 handle, u32 status)
202{
203 gk20a_dbg_fn("");
204
205 if (status != 0) {
206 nvgpu_err(g, "Sub-feature mask update cmd aborted\n");
207 return;
208 }
209
210 gv11b_dbg_pmu("sub-feature mask update is acknowledged from PMU %x\n",
211 msg->msg.pg.msg_type);
212}
213
214static void pmu_handle_pg_param_msg(struct gk20a *g, struct pmu_msg *msg,
215 void *param, u32 handle, u32 status)
216{
217 gk20a_dbg_fn("");
218
219 if (status != 0) {
220 nvgpu_err(g, "GR PARAM cmd aborted\n");
221 return;
222 }
223
224 gv11b_dbg_pmu("GR PARAM is acknowledged from PMU %x\n",
225 msg->msg.pg.msg_type);
226}
227
228int gv11b_pg_gr_init(struct gk20a *g, u32 pg_engine_id)
229{
230 struct nvgpu_pmu *pmu = &g->pmu;
231 struct pmu_cmd cmd;
232 u32 seq;
233
234 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) {
235 memset(&cmd, 0, sizeof(struct pmu_cmd));
236 cmd.hdr.unit_id = PMU_UNIT_PG;
237 cmd.hdr.size = PMU_CMD_HDR_SIZE +
238 sizeof(struct pmu_pg_cmd_gr_init_param_v1);
239 cmd.cmd.pg.gr_init_param_v1.cmd_type =
240 PMU_PG_CMD_ID_PG_PARAM;
241 cmd.cmd.pg.gr_init_param_v1.sub_cmd_id =
242 PMU_PG_PARAM_CMD_GR_INIT_PARAM;
243 cmd.cmd.pg.gr_init_param_v1.featuremask =
244 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED;
245
246 gv11b_dbg_pmu("cmd post PMU_PG_CMD_ID_PG_PARAM_INIT\n");
247 nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
248 pmu_handle_pg_param_msg, pmu, &seq, ~0);
249
250 } else
251 return -EINVAL;
252
253 return 0;
254}
255
256int gv11b_pg_set_subfeature_mask(struct gk20a *g, u32 pg_engine_id)
257{
258 struct nvgpu_pmu *pmu = &g->pmu;
259 struct pmu_cmd cmd;
260 u32 seq;
261
262 if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) {
263 memset(&cmd, 0, sizeof(struct pmu_cmd));
264 cmd.hdr.unit_id = PMU_UNIT_PG;
265 cmd.hdr.size = PMU_CMD_HDR_SIZE +
266 sizeof(struct pmu_pg_cmd_sub_feature_mask_update);
267 cmd.cmd.pg.sf_mask_update.cmd_type =
268 PMU_PG_CMD_ID_PG_PARAM;
269 cmd.cmd.pg.sf_mask_update.sub_cmd_id =
270 PMU_PG_PARAM_CMD_SUB_FEATURE_MASK_UPDATE;
271 cmd.cmd.pg.sf_mask_update.ctrl_id =
272 PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
273 cmd.cmd.pg.sf_mask_update.enabled_mask =
274 PMU_PG_FEATURE_GR_POWER_GATING_ENABLED;
275
276 gv11b_dbg_pmu("cmd post PMU_PG_CMD_SUB_FEATURE_MASK_UPDATE\n");
277 nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ,
278 pmu_handle_pg_sub_feature_msg, pmu, &seq, ~0);
279 } else
280 return -EINVAL;
281
282 return 0;
283}
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h
new file mode 100644
index 00000000..809970ff
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h
@@ -0,0 +1,37 @@
1/*
2 * GV11B PMU
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef __PMU_GV11B_H_
26#define __PMU_GV11B_H_
27
28struct gk20a;
29
30bool gv11b_is_pmu_supported(struct gk20a *g);
31int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu);
32int gv11b_pg_gr_init(struct gk20a *g, u32 pg_engine_id);
33int gv11b_pg_set_subfeature_mask(struct gk20a *g, u32 pg_engine_id);
34bool gv11b_is_lazy_bootstrap(u32 falcon_id);
35bool gv11b_is_priv_load(u32 falcon_id);
36
37#endif /*__PMU_GV11B_H_*/
diff --git a/drivers/gpu/nvgpu/gv11b/regops_gv11b.c b/drivers/gpu/nvgpu/gv11b/regops_gv11b.c
new file mode 100644
index 00000000..c356785e
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/regops_gv11b.c
@@ -0,0 +1,1548 @@
1/*
2 * Tegra GV11b GPU Driver Register Ops
3 *
4 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/slab.h>
26#include <linux/err.h>
27#include <linux/bsearch.h>
28#include <uapi/linux/nvgpu.h>
29
30#include "gk20a/gk20a.h"
31#include "gk20a/dbg_gpu_gk20a.h"
32#include "gk20a/regops_gk20a.h"
33#include "regops_gv11b.h"
34
35static const struct regop_offset_range gv11b_global_whitelist_ranges[] = {
36 { 0x000004f0, 1},
37 { 0x00001a00, 1},
38 { 0x00009400, 1},
39 { 0x00009410, 1},
40 { 0x00009480, 1},
41 { 0x00020200, 32},
42 { 0x00021c04, 2},
43 { 0x00021c14, 3},
44 { 0x00021c24, 71},
45 { 0x00021d44, 1},
46 { 0x00021d4c, 1},
47 { 0x00021d54, 1},
48 { 0x00021d5c, 1},
49 { 0x00021d68, 19},
50 { 0x00021dbc, 16},
51 { 0x00022430, 7},
52 { 0x00022450, 1},
53 { 0x0002245c, 2},
54 { 0x00070000, 5},
55 { 0x000840a8, 1},
56 { 0x00084b5c, 1},
57 { 0x000870a8, 1},
58 { 0x000884e0, 1},
59 { 0x00100c18, 3},
60 { 0x00100c84, 1},
61 { 0x0010a0a8, 1},
62 { 0x0010a4f0, 1},
63 { 0x0013c808, 2},
64 { 0x0013cc14, 1},
65 { 0x0013ec18, 1},
66 { 0x00140028, 1},
67 { 0x00140280, 1},
68 { 0x001402a0, 1},
69 { 0x00140350, 1},
70 { 0x00140480, 1},
71 { 0x001404a0, 1},
72 { 0x00140550, 1},
73 { 0x00140680, 1},
74 { 0x001406a0, 1},
75 { 0x00140750, 1},
76 { 0x00142028, 1},
77 { 0x00142280, 1},
78 { 0x001422a0, 1},
79 { 0x00142350, 1},
80 { 0x00142480, 1},
81 { 0x001424a0, 1},
82 { 0x00142550, 1},
83 { 0x00142680, 1},
84 { 0x001426a0, 1},
85 { 0x00142750, 1},
86 { 0x0017e028, 1},
87 { 0x0017e280, 1},
88 { 0x0017e294, 1},
89 { 0x0017e29c, 2},
90 { 0x0017e2ac, 1},
91 { 0x0017e350, 1},
92 { 0x0017e39c, 1},
93 { 0x0017e480, 1},
94 { 0x0017e4a0, 1},
95 { 0x0017e550, 1},
96 { 0x0017e680, 1},
97 { 0x0017e6a0, 1},
98 { 0x0017e750, 1},
99 { 0x00180040, 41},
100 { 0x001800ec, 1},
101 { 0x001800f8, 7},
102 { 0x00180120, 2},
103 { 0x00180240, 41},
104 { 0x001802ec, 1},
105 { 0x001802f8, 7},
106 { 0x00180320, 2},
107 { 0x00180440, 41},
108 { 0x001804ec, 1},
109 { 0x001804f8, 7},
110 { 0x00180520, 2},
111 { 0x00180640, 41},
112 { 0x001806ec, 1},
113 { 0x001806f8, 7},
114 { 0x00180720, 2},
115 { 0x00180840, 41},
116 { 0x001808ec, 1},
117 { 0x001808f8, 7},
118 { 0x00180920, 2},
119 { 0x00180a40, 41},
120 { 0x00180aec, 1},
121 { 0x00180af8, 7},
122 { 0x00180b20, 2},
123 { 0x00180c40, 41},
124 { 0x00180cec, 1},
125 { 0x00180cf8, 2},
126 { 0x00180d04, 4},
127 { 0x00180d20, 2},
128 { 0x00180e40, 41},
129 { 0x00180eec, 1},
130 { 0x00180ef8, 2},
131 { 0x00180f04, 4},
132 { 0x00180f20, 2},
133 { 0x00181040, 41},
134 { 0x001810ec, 1},
135 { 0x001810f8, 2},
136 { 0x00181104, 4},
137 { 0x00181120, 2},
138 { 0x00181240, 41},
139 { 0x001812ec, 1},
140 { 0x001812f8, 2},
141 { 0x00181304, 4},
142 { 0x00181320, 2},
143 { 0x00181440, 41},
144 { 0x001814ec, 1},
145 { 0x001814f8, 2},
146 { 0x00181504, 4},
147 { 0x00181520, 2},
148 { 0x00181640, 41},
149 { 0x001816ec, 1},
150 { 0x001816f8, 2},
151 { 0x00181704, 4},
152 { 0x00181720, 2},
153 { 0x00181840, 41},
154 { 0x001818ec, 1},
155 { 0x001818f8, 2},
156 { 0x00181904, 4},
157 { 0x00181920, 2},
158 { 0x00181a40, 41},
159 { 0x00181aec, 1},
160 { 0x00181af8, 2},
161 { 0x00181b04, 4},
162 { 0x00181b20, 2},
163 { 0x00181c40, 41},
164 { 0x00181cec, 1},
165 { 0x00181cf8, 2},
166 { 0x00181d04, 4},
167 { 0x00181d20, 2},
168 { 0x00181e40, 41},
169 { 0x00181eec, 1},
170 { 0x00181ef8, 2},
171 { 0x00181f04, 4},
172 { 0x00181f20, 2},
173 { 0x00182040, 41},
174 { 0x001820ec, 1},
175 { 0x001820f8, 2},
176 { 0x00182104, 4},
177 { 0x00182120, 2},
178 { 0x00182240, 41},
179 { 0x001822ec, 1},
180 { 0x001822f8, 2},
181 { 0x00182304, 4},
182 { 0x00182320, 2},
183 { 0x00182440, 41},
184 { 0x001824ec, 1},
185 { 0x001824f8, 2},
186 { 0x00182504, 4},
187 { 0x00182520, 2},
188 { 0x00182640, 41},
189 { 0x001826ec, 1},
190 { 0x001826f8, 2},
191 { 0x00182704, 4},
192 { 0x00182720, 2},
193 { 0x00182840, 41},
194 { 0x001828ec, 1},
195 { 0x001828f8, 2},
196 { 0x00182904, 4},
197 { 0x00182920, 2},
198 { 0x00182a40, 41},
199 { 0x00182aec, 1},
200 { 0x00182af8, 2},
201 { 0x00182b04, 4},
202 { 0x00182b20, 2},
203 { 0x00182c40, 41},
204 { 0x00182cec, 1},
205 { 0x00182cf8, 2},
206 { 0x00182d04, 4},
207 { 0x00182d20, 2},
208 { 0x00182e40, 41},
209 { 0x00182eec, 1},
210 { 0x00182ef8, 2},
211 { 0x00182f04, 4},
212 { 0x00182f20, 2},
213 { 0x00183040, 41},
214 { 0x001830ec, 1},
215 { 0x001830f8, 2},
216 { 0x00183104, 4},
217 { 0x00183120, 2},
218 { 0x00183240, 41},
219 { 0x001832ec, 1},
220 { 0x001832f8, 2},
221 { 0x00183304, 4},
222 { 0x00183320, 2},
223 { 0x00183440, 41},
224 { 0x001834ec, 1},
225 { 0x001834f8, 2},
226 { 0x00183504, 4},
227 { 0x00183520, 2},
228 { 0x00183640, 41},
229 { 0x001836ec, 1},
230 { 0x001836f8, 2},
231 { 0x00183704, 4},
232 { 0x00183720, 2},
233 { 0x00183840, 41},
234 { 0x001838ec, 1},
235 { 0x001838f8, 2},
236 { 0x00183904, 4},
237 { 0x00183920, 2},
238 { 0x00183a40, 41},
239 { 0x00183aec, 1},
240 { 0x00183af8, 2},
241 { 0x00183b04, 4},
242 { 0x00183b20, 2},
243 { 0x00183c40, 41},
244 { 0x00183cec, 1},
245 { 0x00183cf8, 2},
246 { 0x00183d04, 4},
247 { 0x00183d20, 2},
248 { 0x00183e40, 41},
249 { 0x00183eec, 1},
250 { 0x00183ef8, 2},
251 { 0x00183f04, 4},
252 { 0x00183f20, 2},
253 { 0x001c80a8, 1},
254 { 0x001c9100, 1},
255 { 0x001cc0a8, 1},
256 { 0x001cd100, 1},
257 { 0x001d00a8, 1},
258 { 0x001d1100, 1},
259 { 0x00200040, 41},
260 { 0x002000ec, 1},
261 { 0x002000f8, 7},
262 { 0x00200120, 2},
263 { 0x00200240, 41},
264 { 0x002002ec, 1},
265 { 0x002002f8, 7},
266 { 0x00200320, 2},
267 { 0x00200440, 41},
268 { 0x002004ec, 1},
269 { 0x002004f8, 7},
270 { 0x00200520, 2},
271 { 0x00200640, 41},
272 { 0x002006ec, 1},
273 { 0x002006f8, 7},
274 { 0x00200720, 2},
275 { 0x00200840, 41},
276 { 0x002008ec, 1},
277 { 0x002008f8, 2},
278 { 0x00200904, 4},
279 { 0x00200920, 2},
280 { 0x00200a40, 41},
281 { 0x00200aec, 1},
282 { 0x00200af8, 2},
283 { 0x00200b04, 4},
284 { 0x00200b20, 2},
285 { 0x00200c40, 41},
286 { 0x00200cec, 1},
287 { 0x00200cf8, 2},
288 { 0x00200d04, 4},
289 { 0x00200d20, 2},
290 { 0x00200e40, 41},
291 { 0x00200eec, 1},
292 { 0x00200ef8, 2},
293 { 0x00200f04, 4},
294 { 0x00200f20, 2},
295 { 0x00201040, 41},
296 { 0x002010ec, 1},
297 { 0x002010f8, 2},
298 { 0x00201104, 4},
299 { 0x00201120, 2},
300 { 0x00201240, 41},
301 { 0x002012ec, 1},
302 { 0x002012f8, 2},
303 { 0x00201304, 4},
304 { 0x00201320, 2},
305 { 0x00201440, 41},
306 { 0x002014ec, 1},
307 { 0x002014f8, 2},
308 { 0x00201504, 4},
309 { 0x00201520, 2},
310 { 0x00201640, 41},
311 { 0x002016ec, 1},
312 { 0x002016f8, 2},
313 { 0x00201704, 4},
314 { 0x00201720, 2},
315 { 0x00201840, 41},
316 { 0x002018ec, 1},
317 { 0x002018f8, 2},
318 { 0x00201904, 4},
319 { 0x00201920, 2},
320 { 0x00201a40, 41},
321 { 0x00201aec, 1},
322 { 0x00201af8, 2},
323 { 0x00201b04, 4},
324 { 0x00201b20, 2},
325 { 0x00201c40, 41},
326 { 0x00201cec, 1},
327 { 0x00201cf8, 2},
328 { 0x00201d04, 4},
329 { 0x00201d20, 2},
330 { 0x00201e40, 41},
331 { 0x00201eec, 1},
332 { 0x00201ef8, 2},
333 { 0x00201f04, 4},
334 { 0x00201f20, 2},
335 { 0x00202040, 41},
336 { 0x002020ec, 1},
337 { 0x002020f8, 2},
338 { 0x00202104, 4},
339 { 0x00202120, 2},
340 { 0x00202240, 41},
341 { 0x002022ec, 1},
342 { 0x002022f8, 2},
343 { 0x00202304, 4},
344 { 0x00202320, 2},
345 { 0x00202440, 41},
346 { 0x002024ec, 1},
347 { 0x002024f8, 2},
348 { 0x00202504, 4},
349 { 0x00202520, 2},
350 { 0x00202640, 41},
351 { 0x002026ec, 1},
352 { 0x002026f8, 2},
353 { 0x00202704, 4},
354 { 0x00202720, 2},
355 { 0x00202840, 41},
356 { 0x002028ec, 1},
357 { 0x002028f8, 2},
358 { 0x00202904, 4},
359 { 0x00202920, 2},
360 { 0x00202a40, 41},
361 { 0x00202aec, 1},
362 { 0x00202af8, 2},
363 { 0x00202b04, 4},
364 { 0x00202b20, 2},
365 { 0x00202c40, 41},
366 { 0x00202cec, 1},
367 { 0x00202cf8, 2},
368 { 0x00202d04, 4},
369 { 0x00202d20, 2},
370 { 0x00202e40, 41},
371 { 0x00202eec, 1},
372 { 0x00202ef8, 2},
373 { 0x00202f04, 4},
374 { 0x00202f20, 2},
375 { 0x00203040, 41},
376 { 0x002030ec, 1},
377 { 0x002030f8, 2},
378 { 0x00203104, 4},
379 { 0x00203120, 2},
380 { 0x00203240, 41},
381 { 0x002032ec, 1},
382 { 0x002032f8, 2},
383 { 0x00203304, 4},
384 { 0x00203320, 2},
385 { 0x00203440, 41},
386 { 0x002034ec, 1},
387 { 0x002034f8, 2},
388 { 0x00203504, 4},
389 { 0x00203520, 2},
390 { 0x00203640, 41},
391 { 0x002036ec, 1},
392 { 0x002036f8, 2},
393 { 0x00203704, 4},
394 { 0x00203720, 2},
395 { 0x00203840, 41},
396 { 0x002038ec, 1},
397 { 0x002038f8, 2},
398 { 0x00203904, 4},
399 { 0x00203920, 2},
400 { 0x00203a40, 41},
401 { 0x00203aec, 1},
402 { 0x00203af8, 2},
403 { 0x00203b04, 4},
404 { 0x00203b20, 2},
405 { 0x00203c40, 41},
406 { 0x00203cec, 1},
407 { 0x00203cf8, 2},
408 { 0x00203d04, 4},
409 { 0x00203d20, 2},
410 { 0x00203e40, 41},
411 { 0x00203eec, 1},
412 { 0x00203ef8, 2},
413 { 0x00203f04, 4},
414 { 0x00203f20, 2},
415 { 0x00240040, 41},
416 { 0x002400ec, 1},
417 { 0x002400f8, 7},
418 { 0x00240120, 2},
419 { 0x00240240, 41},
420 { 0x002402ec, 1},
421 { 0x002402f8, 7},
422 { 0x00240320, 2},
423 { 0x00240440, 41},
424 { 0x002404ec, 1},
425 { 0x002404f8, 7},
426 { 0x00240520, 2},
427 { 0x00240640, 41},
428 { 0x002406ec, 1},
429 { 0x002406f8, 7},
430 { 0x00240720, 2},
431 { 0x00240840, 41},
432 { 0x002408ec, 1},
433 { 0x002408f8, 7},
434 { 0x00240920, 2},
435 { 0x00240a40, 41},
436 { 0x00240aec, 1},
437 { 0x00240af8, 7},
438 { 0x00240b20, 2},
439 { 0x00240c40, 41},
440 { 0x00240cec, 1},
441 { 0x00240cf8, 2},
442 { 0x00240d04, 4},
443 { 0x00240d20, 2},
444 { 0x00240e40, 41},
445 { 0x00240eec, 1},
446 { 0x00240ef8, 2},
447 { 0x00240f04, 4},
448 { 0x00240f20, 2},
449 { 0x00241040, 41},
450 { 0x002410ec, 1},
451 { 0x002410f8, 2},
452 { 0x00241104, 4},
453 { 0x00241120, 2},
454 { 0x00241240, 41},
455 { 0x002412ec, 1},
456 { 0x002412f8, 2},
457 { 0x00241304, 4},
458 { 0x00241320, 2},
459 { 0x00241440, 41},
460 { 0x002414ec, 1},
461 { 0x002414f8, 2},
462 { 0x00241504, 4},
463 { 0x00241520, 2},
464 { 0x00241640, 41},
465 { 0x002416ec, 1},
466 { 0x002416f8, 2},
467 { 0x00241704, 4},
468 { 0x00241720, 2},
469 { 0x00241840, 41},
470 { 0x002418ec, 1},
471 { 0x002418f8, 2},
472 { 0x00241904, 4},
473 { 0x00241920, 2},
474 { 0x00241a40, 41},
475 { 0x00241aec, 1},
476 { 0x00241af8, 2},
477 { 0x00241b04, 4},
478 { 0x00241b20, 2},
479 { 0x00241c40, 41},
480 { 0x00241cec, 1},
481 { 0x00241cf8, 2},
482 { 0x00241d04, 4},
483 { 0x00241d20, 2},
484 { 0x00241e40, 41},
485 { 0x00241eec, 1},
486 { 0x00241ef8, 2},
487 { 0x00241f04, 4},
488 { 0x00241f20, 2},
489 { 0x00242040, 41},
490 { 0x002420ec, 1},
491 { 0x002420f8, 2},
492 { 0x00242104, 4},
493 { 0x00242120, 2},
494 { 0x00242240, 41},
495 { 0x002422ec, 1},
496 { 0x002422f8, 2},
497 { 0x00242304, 4},
498 { 0x00242320, 2},
499 { 0x00242440, 41},
500 { 0x002424ec, 1},
501 { 0x002424f8, 2},
502 { 0x00242504, 4},
503 { 0x00242520, 2},
504 { 0x00242640, 41},
505 { 0x002426ec, 1},
506 { 0x002426f8, 2},
507 { 0x00242704, 4},
508 { 0x00242720, 2},
509 { 0x00242840, 41},
510 { 0x002428ec, 1},
511 { 0x002428f8, 2},
512 { 0x00242904, 4},
513 { 0x00242920, 2},
514 { 0x00242a40, 41},
515 { 0x00242aec, 1},
516 { 0x00242af8, 2},
517 { 0x00242b04, 4},
518 { 0x00242b20, 2},
519 { 0x00242c40, 41},
520 { 0x00242cec, 1},
521 { 0x00242cf8, 2},
522 { 0x00242d04, 4},
523 { 0x00242d20, 2},
524 { 0x00242e40, 41},
525 { 0x00242eec, 1},
526 { 0x00242ef8, 2},
527 { 0x00242f04, 4},
528 { 0x00242f20, 2},
529 { 0x00243040, 41},
530 { 0x002430ec, 1},
531 { 0x002430f8, 2},
532 { 0x00243104, 4},
533 { 0x00243120, 2},
534 { 0x00243240, 41},
535 { 0x002432ec, 1},
536 { 0x002432f8, 2},
537 { 0x00243304, 4},
538 { 0x00243320, 2},
539 { 0x00243440, 41},
540 { 0x002434ec, 1},
541 { 0x002434f8, 2},
542 { 0x00243504, 4},
543 { 0x00243520, 2},
544 { 0x00243640, 41},
545 { 0x002436ec, 1},
546 { 0x002436f8, 2},
547 { 0x00243704, 4},
548 { 0x00243720, 2},
549 { 0x00243840, 41},
550 { 0x002438ec, 1},
551 { 0x002438f8, 2},
552 { 0x00243904, 4},
553 { 0x00243920, 2},
554 { 0x00243a40, 41},
555 { 0x00243aec, 1},
556 { 0x00243af8, 2},
557 { 0x00243b04, 4},
558 { 0x00243b20, 2},
559 { 0x00243c40, 41},
560 { 0x00243cec, 1},
561 { 0x00243cf8, 2},
562 { 0x00243d04, 4},
563 { 0x00243d20, 2},
564 { 0x00243e40, 41},
565 { 0x00243eec, 1},
566 { 0x00243ef8, 2},
567 { 0x00243f04, 4},
568 { 0x00243f20, 2},
569 { 0x00244000, 1},
570 { 0x00244008, 1},
571 { 0x00244010, 2},
572 { 0x00246000, 1},
573 { 0x00246008, 1},
574 { 0x00246010, 2},
575 { 0x00248000, 1},
576 { 0x00248008, 1},
577 { 0x00248010, 2},
578 { 0x0024a000, 1},
579 { 0x0024a008, 1},
580 { 0x0024a010, 11},
581 { 0x0024a040, 3},
582 { 0x0024a050, 3},
583 { 0x0024a060, 4},
584 { 0x0024a074, 7},
585 { 0x0024a094, 3},
586 { 0x0024a0a4, 1},
587 { 0x0024a100, 6},
588 { 0x00250040, 25},
589 { 0x002500c8, 7},
590 { 0x002500ec, 1},
591 { 0x002500f8, 2},
592 { 0x00250104, 4},
593 { 0x00250120, 2},
594 { 0x00250240, 25},
595 { 0x002502c8, 7},
596 { 0x002502ec, 1},
597 { 0x002502f8, 2},
598 { 0x00250304, 4},
599 { 0x00250320, 2},
600 { 0x00250840, 25},
601 { 0x002508c8, 7},
602 { 0x002508ec, 1},
603 { 0x002508f8, 2},
604 { 0x00250904, 4},
605 { 0x00250920, 2},
606 { 0x00250a40, 25},
607 { 0x00250ac8, 7},
608 { 0x00250aec, 1},
609 { 0x00250af8, 2},
610 { 0x00250b04, 4},
611 { 0x00250b20, 2},
612 { 0x00251800, 3},
613 { 0x00251810, 2},
614 { 0x00251a00, 3},
615 { 0x00251a10, 2},
616 { 0x00278040, 25},
617 { 0x002780c8, 7},
618 { 0x002780ec, 1},
619 { 0x002780f8, 2},
620 { 0x00278104, 4},
621 { 0x00278120, 2},
622 { 0x00278240, 25},
623 { 0x002782c8, 7},
624 { 0x002782ec, 1},
625 { 0x002782f8, 2},
626 { 0x00278304, 4},
627 { 0x00278320, 2},
628 { 0x00278440, 25},
629 { 0x002784c8, 7},
630 { 0x002784ec, 1},
631 { 0x002784f8, 2},
632 { 0x00278504, 4},
633 { 0x00278520, 2},
634 { 0x00278640, 25},
635 { 0x002786c8, 7},
636 { 0x002786ec, 1},
637 { 0x002786f8, 2},
638 { 0x00278704, 4},
639 { 0x00278720, 2},
640 { 0x00278840, 25},
641 { 0x002788c8, 7},
642 { 0x002788ec, 1},
643 { 0x002788f8, 2},
644 { 0x00278904, 4},
645 { 0x00278920, 2},
646 { 0x00278a40, 25},
647 { 0x00278ac8, 7},
648 { 0x00278aec, 1},
649 { 0x00278af8, 2},
650 { 0x00278b04, 4},
651 { 0x00278b20, 2},
652 { 0x00278c40, 25},
653 { 0x00278cc8, 7},
654 { 0x00278cec, 1},
655 { 0x00278cf8, 2},
656 { 0x00278d04, 4},
657 { 0x00278d20, 2},
658 { 0x00278e40, 25},
659 { 0x00278ec8, 7},
660 { 0x00278eec, 1},
661 { 0x00278ef8, 2},
662 { 0x00278f04, 4},
663 { 0x00278f20, 2},
664 { 0x00279040, 25},
665 { 0x002790c8, 7},
666 { 0x002790ec, 1},
667 { 0x002790f8, 2},
668 { 0x00279104, 4},
669 { 0x00279120, 2},
670 { 0x00279240, 25},
671 { 0x002792c8, 7},
672 { 0x002792ec, 1},
673 { 0x002792f8, 2},
674 { 0x00279304, 4},
675 { 0x00279320, 2},
676 { 0x00279440, 25},
677 { 0x002794c8, 7},
678 { 0x002794ec, 1},
679 { 0x002794f8, 2},
680 { 0x00279504, 4},
681 { 0x00279520, 2},
682 { 0x00279640, 25},
683 { 0x002796c8, 7},
684 { 0x002796ec, 1},
685 { 0x002796f8, 2},
686 { 0x00279704, 4},
687 { 0x00279720, 2},
688 { 0x00279840, 25},
689 { 0x002798c8, 7},
690 { 0x002798ec, 1},
691 { 0x002798f8, 2},
692 { 0x00279904, 4},
693 { 0x00279920, 2},
694 { 0x00279a40, 25},
695 { 0x00279ac8, 7},
696 { 0x00279aec, 1},
697 { 0x00279af8, 2},
698 { 0x00279b04, 4},
699 { 0x00279b20, 2},
700 { 0x00279c40, 25},
701 { 0x00279cc8, 7},
702 { 0x00279cec, 1},
703 { 0x00279cf8, 2},
704 { 0x00279d04, 4},
705 { 0x00279d20, 2},
706 { 0x00279e40, 25},
707 { 0x00279ec8, 7},
708 { 0x00279eec, 1},
709 { 0x00279ef8, 2},
710 { 0x00279f04, 4},
711 { 0x00279f20, 2},
712 { 0x0027a040, 25},
713 { 0x0027a0c8, 7},
714 { 0x0027a0ec, 1},
715 { 0x0027a0f8, 2},
716 { 0x0027a104, 4},
717 { 0x0027a120, 2},
718 { 0x0027a240, 25},
719 { 0x0027a2c8, 7},
720 { 0x0027a2ec, 1},
721 { 0x0027a2f8, 2},
722 { 0x0027a304, 4},
723 { 0x0027a320, 2},
724 { 0x0027a440, 25},
725 { 0x0027a4c8, 7},
726 { 0x0027a4ec, 1},
727 { 0x0027a4f8, 2},
728 { 0x0027a504, 4},
729 { 0x0027a520, 2},
730 { 0x0027a640, 25},
731 { 0x0027a6c8, 7},
732 { 0x0027a6ec, 1},
733 { 0x0027a6f8, 2},
734 { 0x0027a704, 4},
735 { 0x0027a720, 2},
736 { 0x0027a840, 25},
737 { 0x0027a8c8, 7},
738 { 0x0027a8ec, 1},
739 { 0x0027a8f8, 2},
740 { 0x0027a904, 4},
741 { 0x0027a920, 2},
742 { 0x0027aa40, 25},
743 { 0x0027aac8, 7},
744 { 0x0027aaec, 1},
745 { 0x0027aaf8, 2},
746 { 0x0027ab04, 4},
747 { 0x0027ab20, 2},
748 { 0x0027ac40, 25},
749 { 0x0027acc8, 7},
750 { 0x0027acec, 1},
751 { 0x0027acf8, 2},
752 { 0x0027ad04, 4},
753 { 0x0027ad20, 2},
754 { 0x0027ae40, 25},
755 { 0x0027aec8, 7},
756 { 0x0027aeec, 1},
757 { 0x0027aef8, 2},
758 { 0x0027af04, 4},
759 { 0x0027af20, 2},
760 { 0x0027b040, 25},
761 { 0x0027b0c8, 7},
762 { 0x0027b0ec, 1},
763 { 0x0027b0f8, 2},
764 { 0x0027b104, 4},
765 { 0x0027b120, 2},
766 { 0x0027b240, 25},
767 { 0x0027b2c8, 7},
768 { 0x0027b2ec, 1},
769 { 0x0027b2f8, 2},
770 { 0x0027b304, 4},
771 { 0x0027b320, 2},
772 { 0x0027b440, 25},
773 { 0x0027b4c8, 7},
774 { 0x0027b4ec, 1},
775 { 0x0027b4f8, 2},
776 { 0x0027b504, 4},
777 { 0x0027b520, 2},
778 { 0x0027b640, 25},
779 { 0x0027b6c8, 7},
780 { 0x0027b6ec, 1},
781 { 0x0027b6f8, 2},
782 { 0x0027b704, 4},
783 { 0x0027b720, 2},
784 { 0x0027b840, 25},
785 { 0x0027b8c8, 7},
786 { 0x0027b8ec, 1},
787 { 0x0027b8f8, 2},
788 { 0x0027b904, 4},
789 { 0x0027b920, 2},
790 { 0x0027ba40, 25},
791 { 0x0027bac8, 7},
792 { 0x0027baec, 1},
793 { 0x0027baf8, 2},
794 { 0x0027bb04, 4},
795 { 0x0027bb20, 2},
796 { 0x0027bc40, 25},
797 { 0x0027bcc8, 7},
798 { 0x0027bcec, 1},
799 { 0x0027bcf8, 2},
800 { 0x0027bd04, 4},
801 { 0x0027bd20, 2},
802 { 0x0027be40, 25},
803 { 0x0027bec8, 7},
804 { 0x0027beec, 1},
805 { 0x0027bef8, 2},
806 { 0x0027bf04, 4},
807 { 0x0027bf20, 2},
808 { 0x0027c040, 25},
809 { 0x0027c0c8, 7},
810 { 0x0027c0ec, 1},
811 { 0x0027c0f8, 2},
812 { 0x0027c104, 4},
813 { 0x0027c120, 2},
814 { 0x0027c240, 25},
815 { 0x0027c2c8, 7},
816 { 0x0027c2ec, 1},
817 { 0x0027c2f8, 2},
818 { 0x0027c304, 4},
819 { 0x0027c320, 2},
820 { 0x0027c440, 25},
821 { 0x0027c4c8, 7},
822 { 0x0027c4ec, 1},
823 { 0x0027c4f8, 2},
824 { 0x0027c504, 4},
825 { 0x0027c520, 2},
826 { 0x0027c640, 25},
827 { 0x0027c6c8, 7},
828 { 0x0027c6ec, 1},
829 { 0x0027c6f8, 2},
830 { 0x0027c704, 4},
831 { 0x0027c720, 2},
832 { 0x0027c840, 25},
833 { 0x0027c8c8, 7},
834 { 0x0027c8ec, 1},
835 { 0x0027c8f8, 2},
836 { 0x0027c904, 4},
837 { 0x0027c920, 2},
838 { 0x0027ca40, 25},
839 { 0x0027cac8, 7},
840 { 0x0027caec, 1},
841 { 0x0027caf8, 2},
842 { 0x0027cb04, 4},
843 { 0x0027cb20, 2},
844 { 0x0027cc40, 25},
845 { 0x0027ccc8, 7},
846 { 0x0027ccec, 1},
847 { 0x0027ccf8, 2},
848 { 0x0027cd04, 4},
849 { 0x0027cd20, 2},
850 { 0x0027ce40, 25},
851 { 0x0027cec8, 7},
852 { 0x0027ceec, 1},
853 { 0x0027cef8, 2},
854 { 0x0027cf04, 4},
855 { 0x0027cf20, 2},
856 { 0x0027d040, 25},
857 { 0x0027d0c8, 7},
858 { 0x0027d0ec, 1},
859 { 0x0027d0f8, 2},
860 { 0x0027d104, 4},
861 { 0x0027d120, 2},
862 { 0x0027d240, 25},
863 { 0x0027d2c8, 7},
864 { 0x0027d2ec, 1},
865 { 0x0027d2f8, 2},
866 { 0x0027d304, 4},
867 { 0x0027d320, 2},
868 { 0x0027d440, 25},
869 { 0x0027d4c8, 7},
870 { 0x0027d4ec, 1},
871 { 0x0027d4f8, 2},
872 { 0x0027d504, 4},
873 { 0x0027d520, 2},
874 { 0x0027d640, 25},
875 { 0x0027d6c8, 7},
876 { 0x0027d6ec, 1},
877 { 0x0027d6f8, 2},
878 { 0x0027d704, 4},
879 { 0x0027d720, 2},
880 { 0x0027d840, 25},
881 { 0x0027d8c8, 7},
882 { 0x0027d8ec, 1},
883 { 0x0027d8f8, 2},
884 { 0x0027d904, 4},
885 { 0x0027d920, 2},
886 { 0x0027da40, 25},
887 { 0x0027dac8, 7},
888 { 0x0027daec, 1},
889 { 0x0027daf8, 2},
890 { 0x0027db04, 4},
891 { 0x0027db20, 2},
892 { 0x0027dc40, 25},
893 { 0x0027dcc8, 7},
894 { 0x0027dcec, 1},
895 { 0x0027dcf8, 2},
896 { 0x0027dd04, 4},
897 { 0x0027dd20, 2},
898 { 0x0027de40, 25},
899 { 0x0027dec8, 7},
900 { 0x0027deec, 1},
901 { 0x0027def8, 2},
902 { 0x0027df04, 4},
903 { 0x0027df20, 2},
904 { 0x0027e040, 25},
905 { 0x0027e0c8, 7},
906 { 0x0027e0ec, 1},
907 { 0x0027e0f8, 2},
908 { 0x0027e104, 4},
909 { 0x0027e120, 2},
910 { 0x0027e240, 25},
911 { 0x0027e2c8, 7},
912 { 0x0027e2ec, 1},
913 { 0x0027e2f8, 2},
914 { 0x0027e304, 4},
915 { 0x0027e320, 2},
916 { 0x0027e440, 25},
917 { 0x0027e4c8, 7},
918 { 0x0027e4ec, 1},
919 { 0x0027e4f8, 2},
920 { 0x0027e504, 4},
921 { 0x0027e520, 2},
922 { 0x0027e640, 25},
923 { 0x0027e6c8, 7},
924 { 0x0027e6ec, 1},
925 { 0x0027e6f8, 2},
926 { 0x0027e704, 4},
927 { 0x0027e720, 2},
928 { 0x0027e840, 25},
929 { 0x0027e8c8, 7},
930 { 0x0027e8ec, 1},
931 { 0x0027e8f8, 2},
932 { 0x0027e904, 4},
933 { 0x0027e920, 2},
934 { 0x0027ea40, 25},
935 { 0x0027eac8, 7},
936 { 0x0027eaec, 1},
937 { 0x0027eaf8, 2},
938 { 0x0027eb04, 4},
939 { 0x0027eb20, 2},
940 { 0x0027ec40, 25},
941 { 0x0027ecc8, 7},
942 { 0x0027ecec, 1},
943 { 0x0027ecf8, 2},
944 { 0x0027ed04, 4},
945 { 0x0027ed20, 2},
946 { 0x0027ee40, 25},
947 { 0x0027eec8, 7},
948 { 0x0027eeec, 1},
949 { 0x0027eef8, 2},
950 { 0x0027ef04, 4},
951 { 0x0027ef20, 2},
952 { 0x0027f040, 25},
953 { 0x0027f0c8, 7},
954 { 0x0027f0ec, 1},
955 { 0x0027f0f8, 2},
956 { 0x0027f104, 4},
957 { 0x0027f120, 2},
958 { 0x0027f240, 25},
959 { 0x0027f2c8, 7},
960 { 0x0027f2ec, 1},
961 { 0x0027f2f8, 2},
962 { 0x0027f304, 4},
963 { 0x0027f320, 2},
964 { 0x0027f440, 25},
965 { 0x0027f4c8, 7},
966 { 0x0027f4ec, 1},
967 { 0x0027f4f8, 2},
968 { 0x0027f504, 4},
969 { 0x0027f520, 2},
970 { 0x0027f640, 25},
971 { 0x0027f6c8, 7},
972 { 0x0027f6ec, 1},
973 { 0x0027f6f8, 2},
974 { 0x0027f704, 4},
975 { 0x0027f720, 2},
976 { 0x0027f840, 25},
977 { 0x0027f8c8, 7},
978 { 0x0027f8ec, 1},
979 { 0x0027f8f8, 2},
980 { 0x0027f904, 4},
981 { 0x0027f920, 2},
982 { 0x0027fa40, 25},
983 { 0x0027fac8, 7},
984 { 0x0027faec, 1},
985 { 0x0027faf8, 2},
986 { 0x0027fb04, 4},
987 { 0x0027fb20, 2},
988 { 0x0027fc40, 25},
989 { 0x0027fcc8, 7},
990 { 0x0027fcec, 1},
991 { 0x0027fcf8, 2},
992 { 0x0027fd04, 4},
993 { 0x0027fd20, 2},
994 { 0x0027fe40, 25},
995 { 0x0027fec8, 7},
996 { 0x0027feec, 1},
997 { 0x0027fef8, 2},
998 { 0x0027ff04, 4},
999 { 0x0027ff20, 2},
1000 { 0x00400500, 1},
1001 { 0x0040415c, 1},
1002 { 0x00404468, 1},
1003 { 0x00404498, 1},
1004 { 0x00405800, 1},
1005 { 0x00405840, 2},
1006 { 0x00405850, 1},
1007 { 0x00405908, 1},
1008 { 0x00405a00, 1},
1009 { 0x00405b50, 1},
1010 { 0x00406024, 5},
1011 { 0x00407010, 1},
1012 { 0x00407808, 1},
1013 { 0x0040803c, 1},
1014 { 0x00408804, 1},
1015 { 0x0040880c, 1},
1016 { 0x00408900, 2},
1017 { 0x00408910, 1},
1018 { 0x00408944, 1},
1019 { 0x00408984, 1},
1020 { 0x004090a8, 1},
1021 { 0x004098a0, 1},
1022 { 0x00409b00, 1},
1023 { 0x0041000c, 1},
1024 { 0x00410110, 1},
1025 { 0x00410184, 1},
1026 { 0x0041040c, 1},
1027 { 0x00410510, 1},
1028 { 0x00410584, 1},
1029 { 0x00418000, 1},
1030 { 0x00418008, 1},
1031 { 0x00418380, 2},
1032 { 0x00418400, 2},
1033 { 0x004184a0, 1},
1034 { 0x00418604, 1},
1035 { 0x00418680, 1},
1036 { 0x00418704, 1},
1037 { 0x00418714, 1},
1038 { 0x00418800, 1},
1039 { 0x0041881c, 1},
1040 { 0x00418830, 1},
1041 { 0x00418884, 1},
1042 { 0x004188b0, 1},
1043 { 0x004188c8, 3},
1044 { 0x004188fc, 1},
1045 { 0x00418b04, 1},
1046 { 0x00418c04, 1},
1047 { 0x00418c10, 8},
1048 { 0x00418c88, 1},
1049 { 0x00418d00, 1},
1050 { 0x00418e00, 1},
1051 { 0x00418e08, 1},
1052 { 0x00418e34, 1},
1053 { 0x00418e40, 4},
1054 { 0x00418e58, 16},
1055 { 0x00418f08, 1},
1056 { 0x00419000, 1},
1057 { 0x0041900c, 1},
1058 { 0x00419018, 1},
1059 { 0x00419854, 1},
1060 { 0x00419864, 1},
1061 { 0x00419a04, 2},
1062 { 0x00419ab0, 1},
1063 { 0x00419b04, 1},
1064 { 0x00419b3c, 1},
1065 { 0x00419b48, 1},
1066 { 0x00419b50, 1},
1067 { 0x00419ba0, 2},
1068 { 0x00419bb0, 1},
1069 { 0x00419bdc, 1},
1070 { 0x00419c0c, 1},
1071 { 0x00419d00, 1},
1072 { 0x00419d08, 2},
1073 { 0x00419e08, 1},
1074 { 0x00419e80, 8},
1075 { 0x00419ea8, 5},
1076 { 0x00419f00, 8},
1077 { 0x00419f28, 5},
1078 { 0x00419f80, 8},
1079 { 0x00419fa8, 5},
1080 { 0x0041a02c, 2},
1081 { 0x0041a0a8, 1},
1082 { 0x0041a8a0, 3},
1083 { 0x0041b014, 1},
1084 { 0x0041b0cc, 1},
1085 { 0x0041b1dc, 1},
1086 { 0x0041b214, 1},
1087 { 0x0041b2cc, 1},
1088 { 0x0041b3dc, 1},
1089 { 0x0041be0c, 3},
1090 { 0x0041becc, 1},
1091 { 0x0041bfdc, 1},
1092 { 0x0041c054, 1},
1093 { 0x0041c2b0, 1},
1094 { 0x0041c304, 1},
1095 { 0x0041c33c, 1},
1096 { 0x0041c348, 1},
1097 { 0x0041c350, 1},
1098 { 0x0041c3a0, 2},
1099 { 0x0041c3b0, 1},
1100 { 0x0041c3dc, 1},
1101 { 0x0041c40c, 1},
1102 { 0x0041c500, 1},
1103 { 0x0041c508, 2},
1104 { 0x0041c608, 1},
1105 { 0x0041c680, 8},
1106 { 0x0041c6a8, 5},
1107 { 0x0041c700, 8},
1108 { 0x0041c728, 5},
1109 { 0x0041c780, 8},
1110 { 0x0041c7a8, 5},
1111 { 0x0041c854, 1},
1112 { 0x0041cab0, 1},
1113 { 0x0041cb04, 1},
1114 { 0x0041cb3c, 1},
1115 { 0x0041cb48, 1},
1116 { 0x0041cb50, 1},
1117 { 0x0041cba0, 2},
1118 { 0x0041cbb0, 1},
1119 { 0x0041cbdc, 1},
1120 { 0x0041cc0c, 1},
1121 { 0x0041cd00, 1},
1122 { 0x0041cd08, 2},
1123 { 0x0041ce08, 1},
1124 { 0x0041ce80, 8},
1125 { 0x0041cea8, 5},
1126 { 0x0041cf00, 8},
1127 { 0x0041cf28, 5},
1128 { 0x0041cf80, 8},
1129 { 0x0041cfa8, 5},
1130 { 0x0041d054, 1},
1131 { 0x0041d2b0, 1},
1132 { 0x0041d304, 1},
1133 { 0x0041d33c, 1},
1134 { 0x0041d348, 1},
1135 { 0x0041d350, 1},
1136 { 0x0041d3a0, 2},
1137 { 0x0041d3b0, 1},
1138 { 0x0041d3dc, 1},
1139 { 0x0041d40c, 1},
1140 { 0x0041d500, 1},
1141 { 0x0041d508, 2},
1142 { 0x0041d608, 1},
1143 { 0x0041d680, 8},
1144 { 0x0041d6a8, 5},
1145 { 0x0041d700, 8},
1146 { 0x0041d728, 5},
1147 { 0x0041d780, 8},
1148 { 0x0041d7a8, 5},
1149 { 0x0041d854, 1},
1150 { 0x0041dab0, 1},
1151 { 0x0041db04, 1},
1152 { 0x0041db3c, 1},
1153 { 0x0041db48, 1},
1154 { 0x0041db50, 1},
1155 { 0x0041dba0, 2},
1156 { 0x0041dbb0, 1},
1157 { 0x0041dbdc, 1},
1158 { 0x0041dc0c, 1},
1159 { 0x0041dd00, 1},
1160 { 0x0041dd08, 2},
1161 { 0x0041de08, 1},
1162 { 0x0041de80, 8},
1163 { 0x0041dea8, 5},
1164 { 0x0041df00, 8},
1165 { 0x0041df28, 5},
1166 { 0x0041df80, 8},
1167 { 0x0041dfa8, 5},
1168 { 0x00481a00, 19},
1169 { 0x00481b00, 50},
1170 { 0x00481e00, 50},
1171 { 0x00481f00, 50},
1172 { 0x00484200, 19},
1173 { 0x00484300, 50},
1174 { 0x00484600, 50},
1175 { 0x00484700, 50},
1176 { 0x00484a00, 19},
1177 { 0x00484b00, 50},
1178 { 0x00484e00, 50},
1179 { 0x00484f00, 50},
1180 { 0x00485200, 19},
1181 { 0x00485300, 50},
1182 { 0x00485600, 50},
1183 { 0x00485700, 50},
1184 { 0x00485a00, 19},
1185 { 0x00485b00, 50},
1186 { 0x00485e00, 50},
1187 { 0x00485f00, 50},
1188 { 0x00500384, 1},
1189 { 0x005004a0, 1},
1190 { 0x00500604, 1},
1191 { 0x00500680, 1},
1192 { 0x00500714, 1},
1193 { 0x0050081c, 1},
1194 { 0x00500884, 1},
1195 { 0x005008b0, 1},
1196 { 0x005008c8, 3},
1197 { 0x005008fc, 1},
1198 { 0x00500b04, 1},
1199 { 0x00500c04, 1},
1200 { 0x00500c10, 8},
1201 { 0x00500c88, 1},
1202 { 0x00500d00, 1},
1203 { 0x00500e08, 1},
1204 { 0x00500f08, 1},
1205 { 0x00501000, 1},
1206 { 0x0050100c, 1},
1207 { 0x00501018, 1},
1208 { 0x00501854, 1},
1209 { 0x00501ab0, 1},
1210 { 0x00501b04, 1},
1211 { 0x00501b3c, 1},
1212 { 0x00501b48, 1},
1213 { 0x00501b50, 1},
1214 { 0x00501ba0, 2},
1215 { 0x00501bb0, 1},
1216 { 0x00501bdc, 1},
1217 { 0x00501c0c, 1},
1218 { 0x00501d00, 1},
1219 { 0x00501d08, 2},
1220 { 0x00501e08, 1},
1221 { 0x00501e80, 8},
1222 { 0x00501ea8, 5},
1223 { 0x00501f00, 8},
1224 { 0x00501f28, 5},
1225 { 0x00501f80, 8},
1226 { 0x00501fa8, 5},
1227 { 0x0050202c, 2},
1228 { 0x005020a8, 1},
1229 { 0x005028a0, 3},
1230 { 0x00503014, 1},
1231 { 0x005030cc, 1},
1232 { 0x005031dc, 1},
1233 { 0x00503214, 1},
1234 { 0x005032cc, 1},
1235 { 0x005033dc, 1},
1236 { 0x00503e14, 1},
1237 { 0x00503ecc, 1},
1238 { 0x00503fdc, 1},
1239 { 0x00504054, 1},
1240 { 0x005042b0, 1},
1241 { 0x00504304, 1},
1242 { 0x0050433c, 1},
1243 { 0x00504348, 1},
1244 { 0x00504350, 1},
1245 { 0x005043a0, 2},
1246 { 0x005043b0, 1},
1247 { 0x005043dc, 1},
1248 { 0x0050440c, 1},
1249 { 0x00504500, 1},
1250 { 0x00504508, 2},
1251 { 0x00504608, 1},
1252 { 0x00504680, 8},
1253 { 0x005046a8, 5},
1254 { 0x00504700, 8},
1255 { 0x00504728, 5},
1256 { 0x00504780, 8},
1257 { 0x005047a8, 5},
1258 { 0x00504854, 1},
1259 { 0x00504ab0, 1},
1260 { 0x00504b04, 1},
1261 { 0x00504b3c, 1},
1262 { 0x00504b48, 1},
1263 { 0x00504b50, 1},
1264 { 0x00504ba0, 2},
1265 { 0x00504bb0, 1},
1266 { 0x00504bdc, 1},
1267 { 0x00504c0c, 1},
1268 { 0x00504d00, 1},
1269 { 0x00504d08, 2},
1270 { 0x00504e08, 1},
1271 { 0x00504e80, 8},
1272 { 0x00504ea8, 5},
1273 { 0x00504f00, 8},
1274 { 0x00504f28, 5},
1275 { 0x00504f80, 8},
1276 { 0x00504fa8, 5},
1277 { 0x00505054, 1},
1278 { 0x005052b0, 1},
1279 { 0x00505304, 1},
1280 { 0x0050533c, 1},
1281 { 0x00505348, 1},
1282 { 0x00505350, 1},
1283 { 0x005053a0, 2},
1284 { 0x005053b0, 1},
1285 { 0x005053dc, 1},
1286 { 0x0050540c, 1},
1287 { 0x00505500, 1},
1288 { 0x00505508, 2},
1289 { 0x00505608, 1},
1290 { 0x00505680, 8},
1291 { 0x005056a8, 5},
1292 { 0x00505700, 8},
1293 { 0x00505728, 5},
1294 { 0x00505780, 8},
1295 { 0x005057a8, 5},
1296 { 0x00505854, 1},
1297 { 0x00505ab0, 1},
1298 { 0x00505b04, 1},
1299 { 0x00505b3c, 1},
1300 { 0x00505b48, 1},
1301 { 0x00505b50, 1},
1302 { 0x00505ba0, 2},
1303 { 0x00505bb0, 1},
1304 { 0x00505bdc, 1},
1305 { 0x00505c0c, 1},
1306 { 0x00505d00, 1},
1307 { 0x00505d08, 2},
1308 { 0x00505e08, 1},
1309 { 0x00505e80, 8},
1310 { 0x00505ea8, 5},
1311 { 0x00505f00, 8},
1312 { 0x00505f28, 5},
1313 { 0x00505f80, 8},
1314 { 0x00505fa8, 5},
1315 { 0x00581a00, 19},
1316 { 0x00581b00, 50},
1317 { 0x00581e00, 50},
1318 { 0x00581f00, 50},
1319 { 0x00584200, 19},
1320 { 0x00584300, 50},
1321 { 0x00584600, 50},
1322 { 0x00584700, 50},
1323 { 0x00584a00, 19},
1324 { 0x00584b00, 50},
1325 { 0x00584e00, 50},
1326 { 0x00584f00, 50},
1327 { 0x00585200, 19},
1328 { 0x00585300, 50},
1329 { 0x00585600, 50},
1330 { 0x00585700, 50},
1331 { 0x00585a00, 19},
1332 { 0x00585b00, 50},
1333 { 0x00585e00, 50},
1334 { 0x00585f00, 50},
1335 { 0x00900100, 1},
1336 { 0x009a0100, 1},
1337 { 0x00a00160, 2},
1338 { 0x00a007d0, 1},
1339 { 0x00a04200, 1},
1340 { 0x00a04470, 2},
1341 { 0x00a08190, 1},
1342 { 0x00a08198, 4},
1343 { 0x00a0c820, 2},
1344 { 0x00a0cc20, 2},
1345 { 0x00a0e470, 2},
1346 { 0x00a0e490, 9},
1347 { 0x00a0e6a8, 7},
1348 { 0x00a0e6c8, 2},
1349 { 0x00a0e6d4, 7},
1350 { 0x00a0e6f4, 2},
1351 { 0x00a0ec70, 2},
1352 { 0x00a0ec90, 9},
1353 { 0x00a0eea8, 7},
1354 { 0x00a0eec8, 2},
1355 { 0x00a0eed4, 7},
1356 { 0x00a0eef4, 2},
1357 { 0x00a10190, 1},
1358 { 0x00a10198, 4},
1359 { 0x00a14820, 2},
1360 { 0x00a14c20, 2},
1361 { 0x00a16470, 2},
1362 { 0x00a16490, 9},
1363 { 0x00a166a8, 7},
1364 { 0x00a166c8, 2},
1365 { 0x00a166d4, 7},
1366 { 0x00a166f4, 2},
1367 { 0x00a16c70, 2},
1368 { 0x00a16c90, 9},
1369 { 0x00a16ea8, 7},
1370 { 0x00a16ec8, 2},
1371 { 0x00a16ed4, 7},
1372 { 0x00a16ef4, 2},
1373 { 0x00a18190, 1},
1374 { 0x00a18198, 4},
1375 { 0x00a1c820, 2},
1376 { 0x00a1cc20, 2},
1377 { 0x00a1e470, 2},
1378 { 0x00a1e490, 9},
1379 { 0x00a1e6a8, 7},
1380 { 0x00a1e6c8, 2},
1381 { 0x00a1e6d4, 7},
1382 { 0x00a1e6f4, 2},
1383 { 0x00a1ec70, 2},
1384 { 0x00a1ec90, 9},
1385 { 0x00a1eea8, 7},
1386 { 0x00a1eec8, 2},
1387 { 0x00a1eed4, 7},
1388 { 0x00a1eef4, 2},
1389 { 0x00a20190, 1},
1390 { 0x00a20198, 4},
1391 { 0x00a24820, 2},
1392 { 0x00a24c20, 2},
1393 { 0x00a26470, 2},
1394 { 0x00a26490, 9},
1395 { 0x00a266a8, 7},
1396 { 0x00a266c8, 2},
1397 { 0x00a266d4, 7},
1398 { 0x00a266f4, 2},
1399 { 0x00a26c70, 2},
1400 { 0x00a26c90, 9},
1401 { 0x00a26ea8, 7},
1402 { 0x00a26ec8, 2},
1403 { 0x00a26ed4, 7},
1404 { 0x00a26ef4, 2},
1405 { 0x00a28190, 1},
1406 { 0x00a28198, 4},
1407 { 0x00a2c820, 2},
1408 { 0x00a2cc20, 2},
1409 { 0x00a2e470, 2},
1410 { 0x00a2e490, 9},
1411 { 0x00a2e6a8, 7},
1412 { 0x00a2e6c8, 2},
1413 { 0x00a2e6d4, 7},
1414 { 0x00a2e6f4, 2},
1415 { 0x00a2ec70, 2},
1416 { 0x00a2ec90, 9},
1417 { 0x00a2eea8, 7},
1418 { 0x00a2eec8, 2},
1419 { 0x00a2eed4, 7},
1420 { 0x00a2eef4, 2},
1421 { 0x00a30190, 1},
1422 { 0x00a30198, 4},
1423 { 0x00a34820, 2},
1424 { 0x00a34c20, 2},
1425 { 0x00a36470, 2},
1426 { 0x00a36490, 9},
1427 { 0x00a366a8, 7},
1428 { 0x00a366c8, 2},
1429 { 0x00a366d4, 7},
1430 { 0x00a366f4, 2},
1431 { 0x00a36c70, 2},
1432 { 0x00a36c90, 9},
1433 { 0x00a36ea8, 7},
1434 { 0x00a36ec8, 2},
1435 { 0x00a36ed4, 7},
1436 { 0x00a36ef4, 2},
1437 { 0x00a38190, 1},
1438 { 0x00a38198, 4},
1439 { 0x00a3c820, 2},
1440 { 0x00a3cc20, 2},
1441 { 0x00a3e470, 2},
1442 { 0x00a3e490, 9},
1443 { 0x00a3e6a8, 7},
1444 { 0x00a3e6c8, 2},
1445 { 0x00a3e6d4, 7},
1446 { 0x00a3e6f4, 2},
1447 { 0x00a3ec70, 2},
1448 { 0x00a3ec90, 9},
1449 { 0x00a3eea8, 7},
1450 { 0x00a3eec8, 2},
1451 { 0x00a3eed4, 7},
1452 { 0x00a3eef4, 2},
1453};
1454
1455
1456static const u32 gv11b_global_whitelist_ranges_count =
1457 ARRAY_SIZE(gv11b_global_whitelist_ranges);
1458
1459/* context */
1460
1461/* runcontrol */
1462static const u32 gv11b_runcontrol_whitelist[] = {
1463};
1464static const u32 gv11b_runcontrol_whitelist_count =
1465 ARRAY_SIZE(gv11b_runcontrol_whitelist);
1466
1467static const struct regop_offset_range gv11b_runcontrol_whitelist_ranges[] = {
1468};
1469static const u32 gv11b_runcontrol_whitelist_ranges_count =
1470 ARRAY_SIZE(gv11b_runcontrol_whitelist_ranges);
1471
1472
1473/* quad ctl */
1474static const u32 gv11b_qctl_whitelist[] = {
1475};
1476static const u32 gv11b_qctl_whitelist_count =
1477 ARRAY_SIZE(gv11b_qctl_whitelist);
1478
1479static const struct regop_offset_range gv11b_qctl_whitelist_ranges[] = {
1480};
1481static const u32 gv11b_qctl_whitelist_ranges_count =
1482 ARRAY_SIZE(gv11b_qctl_whitelist_ranges);
1483
1484const struct regop_offset_range *gv11b_get_global_whitelist_ranges(void)
1485{
1486 return gv11b_global_whitelist_ranges;
1487}
1488
1489int gv11b_get_global_whitelist_ranges_count(void)
1490{
1491 return gv11b_global_whitelist_ranges_count;
1492}
1493
1494const struct regop_offset_range *gv11b_get_context_whitelist_ranges(void)
1495{
1496 return gv11b_global_whitelist_ranges;
1497}
1498
1499int gv11b_get_context_whitelist_ranges_count(void)
1500{
1501 return gv11b_global_whitelist_ranges_count;
1502}
1503
1504const u32 *gv11b_get_runcontrol_whitelist(void)
1505{
1506 return gv11b_runcontrol_whitelist;
1507}
1508
1509int gv11b_get_runcontrol_whitelist_count(void)
1510{
1511 return gv11b_runcontrol_whitelist_count;
1512}
1513
1514const struct regop_offset_range *gv11b_get_runcontrol_whitelist_ranges(void)
1515{
1516 return gv11b_runcontrol_whitelist_ranges;
1517}
1518
1519int gv11b_get_runcontrol_whitelist_ranges_count(void)
1520{
1521 return gv11b_runcontrol_whitelist_ranges_count;
1522}
1523
1524const u32 *gv11b_get_qctl_whitelist(void)
1525{
1526 return gv11b_qctl_whitelist;
1527}
1528
1529int gv11b_get_qctl_whitelist_count(void)
1530{
1531 return gv11b_qctl_whitelist_count;
1532}
1533
1534const struct regop_offset_range *gv11b_get_qctl_whitelist_ranges(void)
1535{
1536 return gv11b_qctl_whitelist_ranges;
1537}
1538
1539int gv11b_get_qctl_whitelist_ranges_count(void)
1540{
1541 return gv11b_qctl_whitelist_ranges_count;
1542}
1543
1544int gv11b_apply_smpc_war(struct dbg_session_gk20a *dbg_s)
1545{
1546 /* Not needed on gv11b */
1547 return 0;
1548}
diff --git a/drivers/gpu/nvgpu/gv11b/regops_gv11b.h b/drivers/gpu/nvgpu/gv11b/regops_gv11b.h
new file mode 100644
index 00000000..0ee2edfe
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/regops_gv11b.h
@@ -0,0 +1,42 @@
1/*
2 *
3 * Tegra GV11B GPU Driver Register Ops
4 *
5 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 * DEALINGS IN THE SOFTWARE.
24 */
25#ifndef __REGOPS_GV11B_H_
26#define __REGOPS_GV11B_H_
27
28const struct regop_offset_range *gv11b_get_global_whitelist_ranges(void);
29int gv11b_get_global_whitelist_ranges_count(void);
30const struct regop_offset_range *gv11b_get_context_whitelist_ranges(void);
31int gv11b_get_context_whitelist_ranges_count(void);
32const u32 *gv11b_get_runcontrol_whitelist(void);
33int gv11b_get_runcontrol_whitelist_count(void);
34const struct regop_offset_range *gv11b_get_runcontrol_whitelist_ranges(void);
35int gv11b_get_runcontrol_whitelist_ranges_count(void);
36const u32 *gv11b_get_qctl_whitelist(void);
37int gv11b_get_qctl_whitelist_count(void);
38const struct regop_offset_range *gv11b_get_qctl_whitelist_ranges(void);
39int gv11b_get_qctl_whitelist_ranges_count(void);
40int gv11b_apply_smpc_war(struct dbg_session_gk20a *dbg_s);
41
42#endif /* __REGOPS_GV11B_H_ */
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
new file mode 100644
index 00000000..fe1aa8a5
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c
@@ -0,0 +1,185 @@
1/*
2 * Volta GPU series Subcontext
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "gk20a/gk20a.h"
26
27#include "gv11b/subctx_gv11b.h"
28
29#include <nvgpu/dma.h>
30#include <nvgpu/log.h>
31#include <nvgpu/gmmu.h>
32
33#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
34#include <nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h>
35
36static void gv11b_init_subcontext_pdb(struct channel_gk20a *c,
37 struct nvgpu_mem *inst_block);
38
39static void gv11b_subctx_commit_valid_mask(struct channel_gk20a *c,
40 struct nvgpu_mem *inst_block);
41static void gv11b_subctx_commit_pdb(struct channel_gk20a *c,
42 struct nvgpu_mem *inst_block);
43
44void gv11b_free_subctx_header(struct channel_gk20a *c)
45{
46 struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header;
47 struct gk20a *g = c->g;
48
49 nvgpu_log(g, gpu_dbg_fn, "gv11b_free_subctx_header");
50
51 if (ctx->mem.gpu_va) {
52 nvgpu_gmmu_unmap(c->vm, &ctx->mem, ctx->mem.gpu_va);
53
54 nvgpu_dma_free(g, &ctx->mem);
55 }
56}
57
58int gv11b_alloc_subctx_header(struct channel_gk20a *c)
59{
60 struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header;
61 struct gk20a *g = c->g;
62 int ret = 0;
63
64 nvgpu_log(g, gpu_dbg_fn, "gv11b_alloc_subctx_header");
65
66 if (ctx->mem.gpu_va == 0) {
67 ret = nvgpu_dma_alloc_flags_sys(g,
68 0, /* No Special flags */
69 ctxsw_prog_fecs_header_v(),
70 &ctx->mem);
71 if (ret) {
72 nvgpu_err(g, "failed to allocate sub ctx header");
73 return ret;
74 }
75 ctx->mem.gpu_va = nvgpu_gmmu_map(c->vm,
76 &ctx->mem,
77 ctx->mem.size,
78 0, /* not GPU-cacheable */
79 gk20a_mem_flag_none, true,
80 ctx->mem.aperture);
81 if (!ctx->mem.gpu_va) {
82 nvgpu_err(g, "failed to map ctx header");
83 nvgpu_dma_free(g, &ctx->mem);
84 return -ENOMEM;
85 }
86 /* Now clear the buffer */
87 if (nvgpu_mem_begin(g, &ctx->mem))
88 return -ENOMEM;
89
90 nvgpu_memset(g, &ctx->mem, 0, 0, ctx->mem.size);
91 nvgpu_mem_end(g, &ctx->mem);
92
93 gv11b_init_subcontext_pdb(c, &c->inst_block);
94 }
95 return ret;
96}
97
98static void gv11b_init_subcontext_pdb(struct channel_gk20a *c,
99 struct nvgpu_mem *inst_block)
100{
101 struct gk20a *g = c->g;
102
103 gv11b_subctx_commit_pdb(c, inst_block);
104 gv11b_subctx_commit_valid_mask(c, inst_block);
105
106 nvgpu_log(g, gpu_dbg_info, " subctx %d instblk set", c->t19x.subctx_id);
107 nvgpu_mem_wr32(g, inst_block, ram_in_engine_wfi_veid_w(),
108 ram_in_engine_wfi_veid_f(c->t19x.subctx_id));
109
110}
111
112int gv11b_update_subctx_header(struct channel_gk20a *c, u64 gpu_va)
113{
114 struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header;
115 struct nvgpu_mem *gr_mem;
116 struct gk20a *g = c->g;
117 int ret = 0;
118 u32 addr_lo, addr_hi;
119
120 addr_lo = u64_lo32(gpu_va);
121 addr_hi = u64_hi32(gpu_va);
122
123 gr_mem = &ctx->mem;
124 g->ops.mm.l2_flush(g, true);
125 if (nvgpu_mem_begin(g, gr_mem))
126 return -ENOMEM;
127
128 nvgpu_mem_wr(g, gr_mem,
129 ctxsw_prog_main_image_context_buffer_ptr_hi_o(), addr_hi);
130 nvgpu_mem_wr(g, gr_mem,
131 ctxsw_prog_main_image_context_buffer_ptr_o(), addr_lo);
132
133 nvgpu_mem_wr(g, gr_mem,
134 ctxsw_prog_main_image_ctl_o(),
135 ctxsw_prog_main_image_ctl_type_per_veid_header_v());
136 nvgpu_mem_end(g, gr_mem);
137 return ret;
138}
139
140void gv11b_subctx_commit_valid_mask(struct channel_gk20a *c,
141 struct nvgpu_mem *inst_block)
142{
143 struct gk20a *g = c->g;
144
145 /* Make all subctx pdbs valid */
146 nvgpu_mem_wr32(g, inst_block, 166, 0xffffffff);
147 nvgpu_mem_wr32(g, inst_block, 167, 0xffffffff);
148}
149
150void gv11b_subctx_commit_pdb(struct channel_gk20a *c,
151 struct nvgpu_mem *inst_block)
152{
153 struct gk20a *g = c->g;
154 struct fifo_gk20a *f = &g->fifo;
155 struct vm_gk20a *vm = c->vm;
156 u32 lo, hi;
157 u32 subctx_id = 0;
158 u32 format_word;
159 u32 pdb_addr_lo, pdb_addr_hi;
160 u64 pdb_addr;
161 u32 aperture = nvgpu_aperture_mask(g, vm->pdb.mem,
162 ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(),
163 ram_in_sc_page_dir_base_target_vid_mem_v());
164
165 pdb_addr = nvgpu_mem_get_addr(g, vm->pdb.mem);
166 pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v());
167 pdb_addr_hi = u64_hi32(pdb_addr);
168 format_word = ram_in_sc_page_dir_base_target_f(
169 aperture, 0) |
170 ram_in_sc_page_dir_base_vol_f(
171 ram_in_sc_page_dir_base_vol_true_v(), 0) |
172 ram_in_sc_page_dir_base_fault_replay_tex_f(1, 0) |
173 ram_in_sc_page_dir_base_fault_replay_gcc_f(1, 0) |
174 ram_in_sc_use_ver2_pt_format_f(1, 0) |
175 ram_in_sc_big_page_size_f(1, 0) |
176 ram_in_sc_page_dir_base_lo_0_f(pdb_addr_lo);
177 nvgpu_log(g, gpu_dbg_info, " pdb info lo %x hi %x",
178 format_word, pdb_addr_hi);
179 for (subctx_id = 0; subctx_id < f->t19x.max_subctx_count; subctx_id++) {
180 lo = ram_in_sc_page_dir_base_vol_0_w() + (4 * subctx_id);
181 hi = ram_in_sc_page_dir_base_hi_0_w() + (4 * subctx_id);
182 nvgpu_mem_wr32(g, inst_block, lo, format_word);
183 nvgpu_mem_wr32(g, inst_block, hi, pdb_addr_hi);
184 }
185}
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h
new file mode 100644
index 00000000..10dc0ba5
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h
@@ -0,0 +1,34 @@
1/*
2 *
3 * Volta GPU series Subcontext
4 *
5 * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 * DEALINGS IN THE SOFTWARE.
24 */
25#ifndef __SUBCONTEXT_GV11B_H__
26#define __SUBCONTEXT_GV11B_H__
27
28int gv11b_alloc_subctx_header(struct channel_gk20a *c);
29
30void gv11b_free_subctx_header(struct channel_gk20a *c);
31
32int gv11b_update_subctx_header(struct channel_gk20a *c, u64 gpu_va);
33
34#endif /* __SUBCONTEXT_GV11B_H__ */
diff --git a/drivers/gpu/nvgpu/gv11b/therm_gv11b.c b/drivers/gpu/nvgpu/gv11b/therm_gv11b.c
new file mode 100644
index 00000000..18987119
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/therm_gv11b.c
@@ -0,0 +1,75 @@
1/*
2 * GV11B Therm
3 *
4 * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include "gk20a/gk20a.h"
26
27#include <nvgpu/soc.h>
28
29#include <nvgpu/hw/gv11b/hw_therm_gv11b.h>
30
31int gv11b_elcg_init_idle_filters(struct gk20a *g)
32{
33 u32 gate_ctrl, idle_filter;
34 u32 engine_id;
35 u32 active_engine_id = 0;
36 struct fifo_gk20a *f = &g->fifo;
37
38 if (nvgpu_platform_is_simulation(g))
39 return 0;
40
41 gk20a_dbg_info("init clock/power gate reg");
42
43 for (engine_id = 0; engine_id < f->num_engines; engine_id++) {
44 active_engine_id = f->active_engines_list[engine_id];
45
46 gate_ctrl = gk20a_readl(g, therm_gate_ctrl_r(active_engine_id));
47 gate_ctrl = set_field(gate_ctrl,
48 therm_gate_ctrl_eng_idle_filt_exp_m(),
49 therm_gate_ctrl_eng_idle_filt_exp__prod_f());
50 gate_ctrl = set_field(gate_ctrl,
51 therm_gate_ctrl_eng_idle_filt_mant_m(),
52 therm_gate_ctrl_eng_idle_filt_mant__prod_f());
53 gate_ctrl = set_field(gate_ctrl,
54 therm_gate_ctrl_eng_delay_before_m(),
55 therm_gate_ctrl_eng_delay_before__prod_f());
56 gate_ctrl = set_field(gate_ctrl,
57 therm_gate_ctrl_eng_delay_after_m(),
58 therm_gate_ctrl_eng_delay_after__prod_f());
59 gk20a_writel(g, therm_gate_ctrl_r(active_engine_id), gate_ctrl);
60 }
61
62 idle_filter = gk20a_readl(g, therm_fecs_idle_filter_r());
63 idle_filter = set_field(idle_filter,
64 therm_fecs_idle_filter_value_m(),
65 therm_fecs_idle_filter_value__prod_f());
66 gk20a_writel(g, therm_fecs_idle_filter_r(), idle_filter);
67
68 idle_filter = gk20a_readl(g, therm_hubmmu_idle_filter_r());
69 idle_filter = set_field(idle_filter,
70 therm_hubmmu_idle_filter_value_m(),
71 therm_hubmmu_idle_filter_value__prod_f());
72 gk20a_writel(g, therm_hubmmu_idle_filter_r(), idle_filter);
73
74 return 0;
75}
diff --git a/drivers/gpu/nvgpu/gv11b/therm_gv11b.h b/drivers/gpu/nvgpu/gv11b/therm_gv11b.h
new file mode 100644
index 00000000..1d89597b
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/therm_gv11b.h
@@ -0,0 +1,28 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#ifndef THERM_GV11B_H
23#define THERM_GV11B_H
24
25struct gk20a;
26int gv11b_elcg_init_idle_filters(struct gk20a *g);
27
28#endif /* THERM_GV11B_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-08-22 14:19:16 -0400