diff options
author | Deepak Nibade <dnibade@nvidia.com> | 2017-11-16 02:21:19 -0500 |
---|---|---|
committer | Deepak Nibade <dnibade@nvidia.com> | 2017-11-16 02:21:35 -0500 |
commit | ba8dc318595f597308902ad16ffed89bdbe7000f (patch) | |
tree | e882886e0cbc05ac39473b95ead16ee50bd69c15 /drivers/gpu/nvgpu/gv11b | |
parent | 69e032653df5aae335764f6346703a1e55c96a2d (diff) | |
parent | 77a90d0b8d2eb1bbb207ae5f46b357f2d7cd07ab (diff) |
Merge remote-tracking branch 'remotes/origin/dev/linux-nvgpu-t19x' into linux-nvgpu
Bug 200363166
Change-Id: Ic662d7b44b673db28dc0aeba338ae67cf2a43d64
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gv11b')
38 files changed, 13757 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.c b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c new file mode 100644 index 00000000..b245dbc6 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.c | |||
@@ -0,0 +1,294 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifdef CONFIG_DEBUG_FS | ||
24 | #include <linux/debugfs.h> | ||
25 | #endif | ||
26 | |||
27 | #include <nvgpu/types.h> | ||
28 | #include <linux/platform/tegra/mc.h> | ||
29 | |||
30 | #include <nvgpu/dma.h> | ||
31 | #include <nvgpu/gmmu.h> | ||
32 | #include <nvgpu/timers.h> | ||
33 | #include <nvgpu/nvgpu_common.h> | ||
34 | #include <nvgpu/kmem.h> | ||
35 | #include <nvgpu/nvgpu_mem.h> | ||
36 | #include <nvgpu/acr/nvgpu_acr.h> | ||
37 | #include <nvgpu/firmware.h> | ||
38 | #include <nvgpu/mm.h> | ||
39 | |||
40 | #include "gk20a/gk20a.h" | ||
41 | #include "acr_gv11b.h" | ||
42 | #include "pmu_gv11b.h" | ||
43 | #include "gk20a/pmu_gk20a.h" | ||
44 | #include "gm20b/mm_gm20b.h" | ||
45 | #include "gm20b/acr_gm20b.h" | ||
46 | #include "gp106/acr_gp106.h" | ||
47 | |||
48 | #include <nvgpu/hw/gv11b/hw_pwr_gv11b.h> | ||
49 | |||
50 | /*Defines*/ | ||
51 | #define gv11b_dbg_pmu(fmt, arg...) \ | ||
52 | gk20a_dbg(gpu_dbg_pmu, fmt, ##arg) | ||
53 | |||
54 | static void flcn64_set_dma(struct falc_u64 *dma_addr, u64 value) | ||
55 | { | ||
56 | dma_addr->lo |= u64_lo32(value); | ||
57 | dma_addr->hi |= u64_hi32(value); | ||
58 | } | ||
59 | /*Externs*/ | ||
60 | |||
61 | /*Forwards*/ | ||
62 | |||
63 | /*Loads ACR bin to FB mem and bootstraps PMU with bootloader code | ||
64 | * start and end are addresses of ucode blob in non-WPR region*/ | ||
65 | int gv11b_bootstrap_hs_flcn(struct gk20a *g) | ||
66 | { | ||
67 | struct mm_gk20a *mm = &g->mm; | ||
68 | struct vm_gk20a *vm = mm->pmu.vm; | ||
69 | int err = 0; | ||
70 | u64 *acr_dmem; | ||
71 | u32 img_size_in_bytes = 0; | ||
72 | u32 status, size, index; | ||
73 | u64 start; | ||
74 | struct acr_desc *acr = &g->acr; | ||
75 | struct nvgpu_firmware *acr_fw = acr->acr_fw; | ||
76 | struct flcn_bl_dmem_desc_v1 *bl_dmem_desc = &acr->bl_dmem_desc_v1; | ||
77 | u32 *acr_ucode_header_t210_load; | ||
78 | u32 *acr_ucode_data_t210_load; | ||
79 | |||
80 | start = nvgpu_mem_get_addr(g, &acr->ucode_blob); | ||
81 | size = acr->ucode_blob.size; | ||
82 | |||
83 | gv11b_dbg_pmu("acr ucode blob start %llx\n", start); | ||
84 | gv11b_dbg_pmu("acr ucode blob size %x\n", size); | ||
85 | |||
86 | gv11b_dbg_pmu(""); | ||
87 | |||
88 | if (!acr_fw) { | ||
89 | /*First time init case*/ | ||
90 | acr_fw = nvgpu_request_firmware(g, | ||
91 | GM20B_HSBIN_PMU_UCODE_IMAGE, 0); | ||
92 | if (!acr_fw) { | ||
93 | nvgpu_err(g, "pmu ucode get fail"); | ||
94 | return -ENOENT; | ||
95 | } | ||
96 | acr->acr_fw = acr_fw; | ||
97 | acr->hsbin_hdr = (struct bin_hdr *)acr_fw->data; | ||
98 | acr->fw_hdr = (struct acr_fw_header *)(acr_fw->data + | ||
99 | acr->hsbin_hdr->header_offset); | ||
100 | acr_ucode_data_t210_load = (u32 *)(acr_fw->data + | ||
101 | acr->hsbin_hdr->data_offset); | ||
102 | acr_ucode_header_t210_load = (u32 *)(acr_fw->data + | ||
103 | acr->fw_hdr->hdr_offset); | ||
104 | img_size_in_bytes = ALIGN((acr->hsbin_hdr->data_size), 256); | ||
105 | |||
106 | gv11b_dbg_pmu("sig dbg offset %u\n", | ||
107 | acr->fw_hdr->sig_dbg_offset); | ||
108 | gv11b_dbg_pmu("sig dbg size %u\n", acr->fw_hdr->sig_dbg_size); | ||
109 | gv11b_dbg_pmu("sig prod offset %u\n", | ||
110 | acr->fw_hdr->sig_prod_offset); | ||
111 | gv11b_dbg_pmu("sig prod size %u\n", | ||
112 | acr->fw_hdr->sig_prod_size); | ||
113 | gv11b_dbg_pmu("patch loc %u\n", acr->fw_hdr->patch_loc); | ||
114 | gv11b_dbg_pmu("patch sig %u\n", acr->fw_hdr->patch_sig); | ||
115 | gv11b_dbg_pmu("header offset %u\n", acr->fw_hdr->hdr_offset); | ||
116 | gv11b_dbg_pmu("header size %u\n", acr->fw_hdr->hdr_size); | ||
117 | |||
118 | /* Lets patch the signatures first.. */ | ||
119 | if (acr_ucode_patch_sig(g, acr_ucode_data_t210_load, | ||
120 | (u32 *)(acr_fw->data + | ||
121 | acr->fw_hdr->sig_prod_offset), | ||
122 | (u32 *)(acr_fw->data + | ||
123 | acr->fw_hdr->sig_dbg_offset), | ||
124 | (u32 *)(acr_fw->data + | ||
125 | acr->fw_hdr->patch_loc), | ||
126 | (u32 *)(acr_fw->data + | ||
127 | acr->fw_hdr->patch_sig)) < 0) { | ||
128 | nvgpu_err(g, "patch signatures fail"); | ||
129 | err = -1; | ||
130 | goto err_release_acr_fw; | ||
131 | } | ||
132 | err = nvgpu_dma_alloc_map_sys(vm, img_size_in_bytes, | ||
133 | &acr->acr_ucode); | ||
134 | if (err) { | ||
135 | err = -ENOMEM; | ||
136 | goto err_release_acr_fw; | ||
137 | } | ||
138 | |||
139 | for (index = 0; index < 9; index++) | ||
140 | gv11b_dbg_pmu("acr_ucode_header_t210_load %u\n", | ||
141 | acr_ucode_header_t210_load[index]); | ||
142 | |||
143 | acr_dmem = (u64 *) | ||
144 | &(((u8 *)acr_ucode_data_t210_load)[ | ||
145 | acr_ucode_header_t210_load[2]]); | ||
146 | acr->acr_dmem_desc_v1 = (struct flcn_acr_desc_v1 *)((u8 *)( | ||
147 | acr->acr_ucode.cpu_va) + acr_ucode_header_t210_load[2]); | ||
148 | ((struct flcn_acr_desc_v1 *)acr_dmem)->nonwpr_ucode_blob_start = | ||
149 | (start); | ||
150 | ((struct flcn_acr_desc_v1 *)acr_dmem)->nonwpr_ucode_blob_size = | ||
151 | size; | ||
152 | ((struct flcn_acr_desc_v1 *)acr_dmem)->regions.no_regions = 2; | ||
153 | ((struct flcn_acr_desc_v1 *)acr_dmem)->wpr_offset = 0; | ||
154 | |||
155 | nvgpu_mem_wr_n(g, &acr->acr_ucode, 0, | ||
156 | acr_ucode_data_t210_load, img_size_in_bytes); | ||
157 | /* | ||
158 | * In order to execute this binary, we will be using | ||
159 | * a bootloader which will load this image into PMU IMEM/DMEM. | ||
160 | * Fill up the bootloader descriptor for PMU HAL to use.. | ||
161 | * TODO: Use standard descriptor which the generic bootloader is | ||
162 | * checked in. | ||
163 | */ | ||
164 | bl_dmem_desc->signature[0] = 0; | ||
165 | bl_dmem_desc->signature[1] = 0; | ||
166 | bl_dmem_desc->signature[2] = 0; | ||
167 | bl_dmem_desc->signature[3] = 0; | ||
168 | bl_dmem_desc->ctx_dma = GK20A_PMU_DMAIDX_VIRT; | ||
169 | flcn64_set_dma(&bl_dmem_desc->code_dma_base, | ||
170 | acr->acr_ucode.gpu_va); | ||
171 | bl_dmem_desc->non_sec_code_off = acr_ucode_header_t210_load[0]; | ||
172 | bl_dmem_desc->non_sec_code_size = acr_ucode_header_t210_load[1]; | ||
173 | bl_dmem_desc->sec_code_off = acr_ucode_header_t210_load[5]; | ||
174 | bl_dmem_desc->sec_code_size = acr_ucode_header_t210_load[6]; | ||
175 | bl_dmem_desc->code_entry_point = 0; /* Start at 0th offset */ | ||
176 | flcn64_set_dma(&bl_dmem_desc->data_dma_base, | ||
177 | acr->acr_ucode.gpu_va + | ||
178 | acr_ucode_header_t210_load[2]); | ||
179 | bl_dmem_desc->data_size = acr_ucode_header_t210_load[3]; | ||
180 | } else | ||
181 | acr->acr_dmem_desc_v1->nonwpr_ucode_blob_size = 0; | ||
182 | status = pmu_exec_gen_bl(g, bl_dmem_desc, 1); | ||
183 | if (status != 0) { | ||
184 | err = status; | ||
185 | goto err_free_ucode_map; | ||
186 | } | ||
187 | |||
188 | return 0; | ||
189 | err_free_ucode_map: | ||
190 | nvgpu_dma_unmap_free(vm, &acr->acr_ucode); | ||
191 | err_release_acr_fw: | ||
192 | nvgpu_release_firmware(g, acr_fw); | ||
193 | acr->acr_fw = NULL; | ||
194 | |||
195 | return err; | ||
196 | } | ||
197 | |||
198 | static int bl_bootstrap(struct nvgpu_pmu *pmu, | ||
199 | struct flcn_bl_dmem_desc_v1 *pbl_desc, u32 bl_sz) | ||
200 | { | ||
201 | struct gk20a *g = gk20a_from_pmu(pmu); | ||
202 | struct acr_desc *acr = &g->acr; | ||
203 | struct mm_gk20a *mm = &g->mm; | ||
204 | u32 virt_addr = 0; | ||
205 | struct hsflcn_bl_desc *pmu_bl_gm10x_desc = g->acr.pmu_hsbl_desc; | ||
206 | u32 dst; | ||
207 | |||
208 | gk20a_dbg_fn(""); | ||
209 | |||
210 | gk20a_writel(g, pwr_falcon_itfen_r(), | ||
211 | gk20a_readl(g, pwr_falcon_itfen_r()) | | ||
212 | pwr_falcon_itfen_ctxen_enable_f()); | ||
213 | gk20a_writel(g, pwr_pmu_new_instblk_r(), | ||
214 | pwr_pmu_new_instblk_ptr_f( | ||
215 | nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> 12) | | ||
216 | pwr_pmu_new_instblk_valid_f(1) | | ||
217 | pwr_pmu_new_instblk_target_sys_ncoh_f()); | ||
218 | |||
219 | /*copy bootloader interface structure to dmem*/ | ||
220 | nvgpu_flcn_copy_to_dmem(pmu->flcn, 0, (u8 *)pbl_desc, | ||
221 | sizeof(struct flcn_bl_dmem_desc_v1), 0); | ||
222 | |||
223 | /* copy bootloader to TOP of IMEM */ | ||
224 | dst = (pwr_falcon_hwcfg_imem_size_v( | ||
225 | gk20a_readl(g, pwr_falcon_hwcfg_r())) << 8) - bl_sz; | ||
226 | |||
227 | nvgpu_flcn_copy_to_imem(pmu->flcn, dst, | ||
228 | (u8 *)(acr->hsbl_ucode.cpu_va), bl_sz, 0, 0, | ||
229 | pmu_bl_gm10x_desc->bl_start_tag); | ||
230 | |||
231 | gv11b_dbg_pmu("Before starting falcon with BL\n"); | ||
232 | |||
233 | virt_addr = pmu_bl_gm10x_desc->bl_start_tag << 8; | ||
234 | |||
235 | nvgpu_flcn_bootstrap(pmu->flcn, virt_addr); | ||
236 | |||
237 | return 0; | ||
238 | } | ||
239 | |||
240 | int gv11b_init_pmu_setup_hw1(struct gk20a *g, | ||
241 | void *desc, u32 bl_sz) | ||
242 | { | ||
243 | |||
244 | struct nvgpu_pmu *pmu = &g->pmu; | ||
245 | int err; | ||
246 | |||
247 | gk20a_dbg_fn(""); | ||
248 | |||
249 | nvgpu_mutex_acquire(&pmu->isr_mutex); | ||
250 | nvgpu_flcn_reset(pmu->flcn); | ||
251 | pmu->isr_enabled = true; | ||
252 | nvgpu_mutex_release(&pmu->isr_mutex); | ||
253 | |||
254 | /* setup apertures - virtual */ | ||
255 | gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE), | ||
256 | pwr_fbif_transcfg_mem_type_physical_f() | | ||
257 | pwr_fbif_transcfg_target_noncoherent_sysmem_f()); | ||
258 | gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_VIRT), | ||
259 | pwr_fbif_transcfg_mem_type_virtual_f()); | ||
260 | /* setup apertures - physical */ | ||
261 | gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_VID), | ||
262 | pwr_fbif_transcfg_mem_type_physical_f() | | ||
263 | pwr_fbif_transcfg_target_noncoherent_sysmem_f()); | ||
264 | gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_COH), | ||
265 | pwr_fbif_transcfg_mem_type_physical_f() | | ||
266 | pwr_fbif_transcfg_target_coherent_sysmem_f()); | ||
267 | gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_PHYS_SYS_NCOH), | ||
268 | pwr_fbif_transcfg_mem_type_physical_f() | | ||
269 | pwr_fbif_transcfg_target_noncoherent_sysmem_f()); | ||
270 | |||
271 | /*Copying pmu cmdline args*/ | ||
272 | g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq(pmu, | ||
273 | g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_PWRCLK)); | ||
274 | g->ops.pmu_ver.set_pmu_cmdline_args_secure_mode(pmu, 1); | ||
275 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_size( | ||
276 | pmu, GK20A_PMU_TRACE_BUFSIZE); | ||
277 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base(pmu); | ||
278 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx( | ||
279 | pmu, GK20A_PMU_DMAIDX_VIRT); | ||
280 | nvgpu_flcn_copy_to_dmem(pmu->flcn, g->acr.pmu_args, | ||
281 | (u8 *)(g->ops.pmu_ver.get_pmu_cmdline_args_ptr(pmu)), | ||
282 | g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu), 0); | ||
283 | /*disable irqs for hs falcon booting as we will poll for halt*/ | ||
284 | nvgpu_mutex_acquire(&pmu->isr_mutex); | ||
285 | pmu_enable_irq(pmu, false); | ||
286 | pmu->isr_enabled = false; | ||
287 | nvgpu_mutex_release(&pmu->isr_mutex); | ||
288 | /*Clearing mailbox register used to reflect capabilities*/ | ||
289 | gk20a_writel(g, pwr_falcon_mailbox1_r(), 0); | ||
290 | err = bl_bootstrap(pmu, desc, bl_sz); | ||
291 | if (err) | ||
292 | return err; | ||
293 | return 0; | ||
294 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/acr_gv11b.h b/drivers/gpu/nvgpu/gv11b/acr_gv11b.h new file mode 100644 index 00000000..72b3ec35 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/acr_gv11b.h | |||
@@ -0,0 +1,30 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef __ACR_GV11B_H_ | ||
24 | #define __ACR_GV11B_H_ | ||
25 | |||
26 | |||
27 | int gv11b_bootstrap_hs_flcn(struct gk20a *g); | ||
28 | int gv11b_init_pmu_setup_hw1(struct gk20a *g, | ||
29 | void *desc, u32 bl_sz); | ||
30 | #endif /*__PMU_GP106_H_*/ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/ce_gv11b.c b/drivers/gpu/nvgpu/gv11b/ce_gv11b.c new file mode 100644 index 00000000..86518ac7 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/ce_gv11b.c | |||
@@ -0,0 +1,110 @@ | |||
1 | /* | ||
2 | * Volta GPU series Copy Engine. | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include "nvgpu/log.h" | ||
26 | #include "nvgpu/bitops.h" | ||
27 | |||
28 | #include "gk20a/gk20a.h" | ||
29 | |||
30 | #include "gp10b/ce_gp10b.h" | ||
31 | |||
32 | #include "ce_gv11b.h" | ||
33 | |||
34 | #include <nvgpu/hw/gv11b/hw_ce_gv11b.h> | ||
35 | #include <nvgpu/hw/gv11b/hw_top_gv11b.h> | ||
36 | |||
37 | u32 gv11b_ce_get_num_pce(struct gk20a *g) | ||
38 | { | ||
39 | /* register contains a bitmask indicating which physical copy | ||
40 | * engines are present (and not floorswept). | ||
41 | */ | ||
42 | u32 num_pce; | ||
43 | u32 ce_pce_map = gk20a_readl(g, ce_pce_map_r()); | ||
44 | |||
45 | num_pce = hweight32(ce_pce_map); | ||
46 | nvgpu_log_info(g, "num PCE: %d", num_pce); | ||
47 | return num_pce; | ||
48 | } | ||
49 | |||
50 | void gv11b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base) | ||
51 | { | ||
52 | u32 ce_intr = gk20a_readl(g, ce_intr_status_r(inst_id)); | ||
53 | u32 clear_intr = 0; | ||
54 | |||
55 | nvgpu_log(g, gpu_dbg_intr, "ce isr 0x%08x 0x%08x", ce_intr, inst_id); | ||
56 | |||
57 | /* An INVALID_CONFIG interrupt will be generated if a floorswept | ||
58 | * PCE is assigned to a valid LCE in the NV_CE_PCE2LCE_CONFIG | ||
59 | * registers. This is a fatal error and the LCE will have to be | ||
60 | * reset to get back to a working state. | ||
61 | */ | ||
62 | if (ce_intr & ce_intr_status_invalid_config_pending_f()) { | ||
63 | nvgpu_log(g, gpu_dbg_intr, | ||
64 | "ce: inst %d: invalid config", inst_id); | ||
65 | clear_intr |= ce_intr_status_invalid_config_reset_f(); | ||
66 | } | ||
67 | |||
68 | /* A MTHD_BUFFER_FAULT interrupt will be triggered if any access | ||
69 | * to a method buffer during context load or save encounters a fault. | ||
70 | * This is a fatal interrupt and will require at least the LCE to be | ||
71 | * reset before operations can start again, if not the entire GPU. | ||
72 | */ | ||
73 | if (ce_intr & ce_intr_status_mthd_buffer_fault_pending_f()) { | ||
74 | nvgpu_log(g, gpu_dbg_intr, | ||
75 | "ce: inst %d: mthd buffer fault", inst_id); | ||
76 | clear_intr |= ce_intr_status_mthd_buffer_fault_reset_f(); | ||
77 | } | ||
78 | |||
79 | gk20a_writel(g, ce_intr_status_r(inst_id), clear_intr); | ||
80 | |||
81 | gp10b_ce_isr(g, inst_id, pri_base); | ||
82 | } | ||
83 | |||
84 | u32 gv11b_ce_get_num_lce(struct gk20a *g) | ||
85 | { | ||
86 | u32 reg_val, num_lce; | ||
87 | |||
88 | reg_val = gk20a_readl(g, top_num_ces_r()); | ||
89 | num_lce = top_num_ces_value_v(reg_val); | ||
90 | nvgpu_log_info(g, "num LCE: %d", num_lce); | ||
91 | |||
92 | return num_lce; | ||
93 | } | ||
94 | |||
95 | void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g) | ||
96 | { | ||
97 | u32 reg_val, num_lce, lce, clear_intr; | ||
98 | |||
99 | num_lce = gv11b_ce_get_num_lce(g); | ||
100 | |||
101 | for (lce = 0; lce < num_lce; lce++) { | ||
102 | reg_val = gk20a_readl(g, ce_intr_status_r(lce)); | ||
103 | if (reg_val & ce_intr_status_mthd_buffer_fault_pending_f()) { | ||
104 | nvgpu_log(g, gpu_dbg_intr, | ||
105 | "ce: lce %d: mthd buffer fault", lce); | ||
106 | clear_intr = ce_intr_status_mthd_buffer_fault_reset_f(); | ||
107 | gk20a_writel(g, ce_intr_status_r(lce), clear_intr); | ||
108 | } | ||
109 | } | ||
110 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/ce_gv11b.h b/drivers/gpu/nvgpu/gv11b/ce_gv11b.h new file mode 100644 index 00000000..a0c7e0b1 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/ce_gv11b.h | |||
@@ -0,0 +1,35 @@ | |||
1 | /* | ||
2 | * | ||
3 | * Volta GPU series copy engine | ||
4 | * | ||
5 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
6 | * | ||
7 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
8 | * copy of this software and associated documentation files (the "Software"), | ||
9 | * to deal in the Software without restriction, including without limitation | ||
10 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
11 | * and/or sell copies of the Software, and to permit persons to whom the | ||
12 | * Software is furnished to do so, subject to the following conditions: | ||
13 | * | ||
14 | * The above copyright notice and this permission notice shall be included in | ||
15 | * all copies or substantial portions of the Software. | ||
16 | * | ||
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
23 | * DEALINGS IN THE SOFTWARE. | ||
24 | */ | ||
25 | #ifndef __CE_GV11B_H__ | ||
26 | #define __CE_GV11B_H__ | ||
27 | |||
28 | struct gk20a; | ||
29 | |||
30 | void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g); | ||
31 | u32 gv11b_ce_get_num_lce(struct gk20a *g); | ||
32 | u32 gv11b_ce_get_num_pce(struct gk20a *g); | ||
33 | void gv11b_ce_isr(struct gk20a *g, u32 inst_id, u32 pri_base); | ||
34 | |||
35 | #endif /*__CE2_GV11B_H__*/ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c new file mode 100644 index 00000000..2eb45a88 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.c | |||
@@ -0,0 +1,206 @@ | |||
1 | /* | ||
2 | * GV11B Cycle stats snapshots support | ||
3 | * | ||
4 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/dma-mapping.h> | ||
26 | #include <linux/dma-buf.h> | ||
27 | |||
28 | #include <nvgpu/bitops.h> | ||
29 | #include <nvgpu/kmem.h> | ||
30 | #include <nvgpu/lock.h> | ||
31 | #include <nvgpu/dma.h> | ||
32 | #include <nvgpu/mm.h> | ||
33 | |||
34 | #include "gk20a/gk20a.h" | ||
35 | #include "gk20a/css_gr_gk20a.h" | ||
36 | #include "css_gr_gv11b.h" | ||
37 | |||
38 | #include <nvgpu/log.h> | ||
39 | #include <nvgpu/bug.h> | ||
40 | |||
41 | #include <nvgpu/hw/gv11b/hw_perf_gv11b.h> | ||
42 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
43 | |||
44 | |||
45 | /* reports whether the hw queue overflowed */ | ||
46 | static inline bool css_hw_get_overflow_status(struct gk20a *g) | ||
47 | { | ||
48 | const u32 st = perf_pmasys_control_membuf_status_overflowed_f(); | ||
49 | return st == (gk20a_readl(g, perf_pmasys_control_r()) & st); | ||
50 | } | ||
51 | |||
52 | /* returns how many pending snapshot entries are pending */ | ||
53 | static inline u32 css_hw_get_pending_snapshots(struct gk20a *g) | ||
54 | { | ||
55 | return gk20a_readl(g, perf_pmasys_mem_bytes_r()) / | ||
56 | sizeof(struct gk20a_cs_snapshot_fifo_entry); | ||
57 | } | ||
58 | |||
59 | /* informs hw how many snapshots have been processed (frees up fifo space) */ | ||
60 | static inline void gv11b_css_hw_set_handled_snapshots(struct gk20a *g, u32 done) | ||
61 | { | ||
62 | if (done > 0) { | ||
63 | gk20a_writel(g, perf_pmasys_mem_bump_r(), | ||
64 | done * sizeof(struct gk20a_cs_snapshot_fifo_entry)); | ||
65 | } | ||
66 | } | ||
67 | |||
68 | /* disable streaming to memory */ | ||
69 | static void gv11b_css_hw_reset_streaming(struct gk20a *g) | ||
70 | { | ||
71 | u32 engine_status; | ||
72 | |||
73 | /* reset the perfmon */ | ||
74 | g->ops.mc.reset(g, mc_enable_perfmon_enabled_f()); | ||
75 | |||
76 | /* RBUFEMPTY must be set -- otherwise we'll pick up */ | ||
77 | /* snapshot that have been queued up from earlier */ | ||
78 | engine_status = gk20a_readl(g, perf_pmasys_enginestatus_r()); | ||
79 | |||
80 | /* turn off writes */ | ||
81 | gk20a_writel(g, perf_pmasys_control_r(), | ||
82 | perf_pmasys_control_membuf_clear_status_doit_f()); | ||
83 | |||
84 | /* pointing all pending snapshots as handled */ | ||
85 | gv11b_css_hw_set_handled_snapshots(g, css_hw_get_pending_snapshots(g)); | ||
86 | } | ||
87 | |||
88 | int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch, | ||
89 | struct gk20a_cs_snapshot_client *cs_client) | ||
90 | { | ||
91 | struct gk20a *g = ch->g; | ||
92 | struct gr_gk20a *gr = &g->gr; | ||
93 | struct gk20a_cs_snapshot *data = gr->cs_data; | ||
94 | u32 snapshot_size = cs_client->snapshot_size; | ||
95 | int ret; | ||
96 | |||
97 | u32 virt_addr_lo; | ||
98 | u32 virt_addr_hi; | ||
99 | u32 inst_pa_page; | ||
100 | |||
101 | if (data->hw_snapshot) | ||
102 | return 0; | ||
103 | |||
104 | if (snapshot_size < CSS_MIN_HW_SNAPSHOT_SIZE) | ||
105 | snapshot_size = CSS_MIN_HW_SNAPSHOT_SIZE; | ||
106 | |||
107 | ret = nvgpu_dma_alloc_map_sys(g->mm.pmu.vm, snapshot_size, | ||
108 | &data->hw_memdesc); | ||
109 | if (ret) | ||
110 | return ret; | ||
111 | |||
112 | /* perf output buffer may not cross a 4GB boundary - with a separate */ | ||
113 | /* va smaller than that, it won't but check anyway */ | ||
114 | if (!data->hw_memdesc.cpu_va || | ||
115 | data->hw_memdesc.size < snapshot_size || | ||
116 | data->hw_memdesc.gpu_va + u64_lo32(snapshot_size) > SZ_4G) { | ||
117 | ret = -EFAULT; | ||
118 | goto failed_allocation; | ||
119 | } | ||
120 | |||
121 | data->hw_snapshot = | ||
122 | (struct gk20a_cs_snapshot_fifo_entry *)data->hw_memdesc.cpu_va; | ||
123 | data->hw_end = data->hw_snapshot + | ||
124 | snapshot_size / sizeof(struct gk20a_cs_snapshot_fifo_entry); | ||
125 | data->hw_get = data->hw_snapshot; | ||
126 | memset(data->hw_snapshot, 0xff, snapshot_size); | ||
127 | |||
128 | virt_addr_lo = u64_lo32(data->hw_memdesc.gpu_va); | ||
129 | virt_addr_hi = u64_hi32(data->hw_memdesc.gpu_va); | ||
130 | |||
131 | gv11b_css_hw_reset_streaming(g); | ||
132 | |||
133 | gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo); | ||
134 | gk20a_writel(g, perf_pmasys_outbaseupper_r(), | ||
135 | perf_pmasys_outbaseupper_ptr_f(virt_addr_hi)); | ||
136 | gk20a_writel(g, perf_pmasys_outsize_r(), snapshot_size); | ||
137 | |||
138 | /* this field is aligned to 4K */ | ||
139 | inst_pa_page = nvgpu_inst_block_addr(g, &g->mm.hwpm.inst_block) >> 12; | ||
140 | |||
141 | gk20a_writel(g, perf_pmasys_mem_block_r(), | ||
142 | perf_pmasys_mem_block_base_f(inst_pa_page) | | ||
143 | perf_pmasys_mem_block_valid_true_f() | | ||
144 | nvgpu_aperture_mask(g, &g->mm.hwpm.inst_block, | ||
145 | perf_pmasys_mem_block_target_sys_ncoh_f(), | ||
146 | perf_pmasys_mem_block_target_lfb_f())); | ||
147 | |||
148 | |||
149 | gk20a_dbg_info("cyclestats: buffer for hardware snapshots enabled\n"); | ||
150 | |||
151 | return 0; | ||
152 | |||
153 | failed_allocation: | ||
154 | if (data->hw_memdesc.size) { | ||
155 | nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); | ||
156 | memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); | ||
157 | } | ||
158 | data->hw_snapshot = NULL; | ||
159 | |||
160 | return ret; | ||
161 | } | ||
162 | |||
163 | void gv11b_css_hw_disable_snapshot(struct gr_gk20a *gr) | ||
164 | { | ||
165 | struct gk20a *g = gr->g; | ||
166 | struct gk20a_cs_snapshot *data = gr->cs_data; | ||
167 | |||
168 | if (!data->hw_snapshot) | ||
169 | return; | ||
170 | |||
171 | gv11b_css_hw_reset_streaming(g); | ||
172 | |||
173 | gk20a_writel(g, perf_pmasys_outbase_r(), 0); | ||
174 | gk20a_writel(g, perf_pmasys_outbaseupper_r(), | ||
175 | perf_pmasys_outbaseupper_ptr_f(0)); | ||
176 | gk20a_writel(g, perf_pmasys_outsize_r(), 0); | ||
177 | |||
178 | gk20a_writel(g, perf_pmasys_mem_block_r(), | ||
179 | perf_pmasys_mem_block_base_f(0) | | ||
180 | perf_pmasys_mem_block_valid_false_f() | | ||
181 | perf_pmasys_mem_block_target_f(0)); | ||
182 | |||
183 | nvgpu_dma_unmap_free(g->mm.pmu.vm, &data->hw_memdesc); | ||
184 | memset(&data->hw_memdesc, 0, sizeof(data->hw_memdesc)); | ||
185 | data->hw_snapshot = NULL; | ||
186 | |||
187 | gk20a_dbg_info("cyclestats: buffer for hardware snapshots disabled\n"); | ||
188 | } | ||
189 | |||
190 | int gv11b_css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending, | ||
191 | bool *hw_overflow) | ||
192 | { | ||
193 | struct gk20a *g = ch->g; | ||
194 | struct gr_gk20a *gr = &g->gr; | ||
195 | struct gk20a_cs_snapshot *css = gr->cs_data; | ||
196 | |||
197 | if (!css->hw_snapshot) | ||
198 | return -EINVAL; | ||
199 | |||
200 | *pending = css_hw_get_pending_snapshots(g); | ||
201 | if (!*pending) | ||
202 | return 0; | ||
203 | |||
204 | *hw_overflow = css_hw_get_overflow_status(g); | ||
205 | return 0; | ||
206 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h new file mode 100644 index 00000000..6b11a62e --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/css_gr_gv11b.h | |||
@@ -0,0 +1,34 @@ | |||
1 | /* | ||
2 | * GV11B Cycle stats snapshots support | ||
3 | * | ||
4 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef CSS_GR_GV11B_H | ||
26 | #define CSS_GR_GV11B_H | ||
27 | |||
28 | int gv11b_css_hw_enable_snapshot(struct channel_gk20a *ch, | ||
29 | struct gk20a_cs_snapshot_client *cs_client); | ||
30 | void gv11b_css_hw_disable_snapshot(struct gr_gk20a *gr); | ||
31 | int gv11b_css_hw_check_data_available(struct channel_gk20a *ch, u32 *pending, | ||
32 | bool *hw_overflow); | ||
33 | |||
34 | #endif /* CSS_GR_GV11B_H */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c new file mode 100644 index 00000000..a02c2ddd --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.c | |||
@@ -0,0 +1,99 @@ | |||
1 | /* | ||
2 | * Tegra GV11B GPU Debugger/Profiler Driver | ||
3 | * | ||
4 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <uapi/linux/nvgpu.h> | ||
26 | |||
27 | #include <nvgpu/log.h> | ||
28 | #include "gk20a/gk20a.h" | ||
29 | #include <nvgpu/hw/gv11b/hw_perf_gv11b.h> | ||
30 | |||
31 | int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size) | ||
32 | { | ||
33 | struct mm_gk20a *mm = &g->mm; | ||
34 | u32 virt_addr_lo; | ||
35 | u32 virt_addr_hi; | ||
36 | u32 inst_pa_page; | ||
37 | int err; | ||
38 | |||
39 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, ""); | ||
40 | err = gk20a_busy(g); | ||
41 | if (err) { | ||
42 | nvgpu_err(g, "failed to poweron"); | ||
43 | return err; | ||
44 | } | ||
45 | |||
46 | err = gk20a_alloc_inst_block(g, &mm->perfbuf.inst_block); | ||
47 | if (err) | ||
48 | return err; | ||
49 | |||
50 | g->ops.mm.init_inst_block(&mm->perfbuf.inst_block, mm->perfbuf.vm, 0); | ||
51 | |||
52 | virt_addr_lo = u64_lo32(offset); | ||
53 | virt_addr_hi = u64_hi32(offset); | ||
54 | |||
55 | gk20a_writel(g, perf_pmasys_outbase_r(), virt_addr_lo); | ||
56 | gk20a_writel(g, perf_pmasys_outbaseupper_r(), | ||
57 | perf_pmasys_outbaseupper_ptr_f(virt_addr_hi)); | ||
58 | gk20a_writel(g, perf_pmasys_outsize_r(), size); | ||
59 | |||
60 | /* this field is aligned to 4K */ | ||
61 | inst_pa_page = nvgpu_inst_block_addr(g, &mm->perfbuf.inst_block) >> 12; | ||
62 | |||
63 | gk20a_writel(g, perf_pmasys_mem_block_r(), | ||
64 | perf_pmasys_mem_block_base_f(inst_pa_page) | | ||
65 | perf_pmasys_mem_block_valid_true_f() | | ||
66 | nvgpu_aperture_mask(g, &mm->perfbuf.inst_block, | ||
67 | + perf_pmasys_mem_block_target_sys_ncoh_f(), | ||
68 | + perf_pmasys_mem_block_target_lfb_f())); | ||
69 | |||
70 | gk20a_idle(g); | ||
71 | return 0; | ||
72 | } | ||
73 | |||
74 | /* must be called with dbg_sessions_lock held */ | ||
75 | int gv11b_perfbuf_disable_locked(struct gk20a *g) | ||
76 | { | ||
77 | int err; | ||
78 | |||
79 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, ""); | ||
80 | err = gk20a_busy(g); | ||
81 | if (err) { | ||
82 | nvgpu_err(g, "failed to poweron"); | ||
83 | return err; | ||
84 | } | ||
85 | |||
86 | gk20a_writel(g, perf_pmasys_outbase_r(), 0); | ||
87 | gk20a_writel(g, perf_pmasys_outbaseupper_r(), | ||
88 | perf_pmasys_outbaseupper_ptr_f(0)); | ||
89 | gk20a_writel(g, perf_pmasys_outsize_r(), 0); | ||
90 | |||
91 | gk20a_writel(g, perf_pmasys_mem_block_r(), | ||
92 | perf_pmasys_mem_block_base_f(0) | | ||
93 | perf_pmasys_mem_block_valid_false_f() | | ||
94 | perf_pmasys_mem_block_target_f(0)); | ||
95 | |||
96 | gk20a_idle(g); | ||
97 | |||
98 | return 0; | ||
99 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h new file mode 100644 index 00000000..88771a49 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/dbg_gpu_gv11b.h | |||
@@ -0,0 +1,28 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | #ifndef DBG_GPU_GV11B_H | ||
23 | #define DBG_GPU_GV11B_H | ||
24 | |||
25 | int gv11b_perfbuf_enable_locked(struct gk20a *g, u64 offset, u32 size); | ||
26 | int gv11b_perfbuf_disable_locked(struct gk20a *g); | ||
27 | |||
28 | #endif /* DBG_GPU_GV11B_H */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h b/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h new file mode 100644 index 00000000..94b25c02 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/ecc_gv11b.h | |||
@@ -0,0 +1,64 @@ | |||
1 | /* | ||
2 | * GV11B GPU ECC | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_ECC_GV11B_H_ | ||
26 | #define _NVGPU_ECC_GV11B_H_ | ||
27 | |||
28 | struct ecc_gr_t19x { | ||
29 | struct gk20a_ecc_stat sm_l1_tag_corrected_err_count; | ||
30 | struct gk20a_ecc_stat sm_l1_tag_uncorrected_err_count; | ||
31 | struct gk20a_ecc_stat sm_cbu_corrected_err_count; | ||
32 | struct gk20a_ecc_stat sm_cbu_uncorrected_err_count; | ||
33 | struct gk20a_ecc_stat sm_l1_data_corrected_err_count; | ||
34 | struct gk20a_ecc_stat sm_l1_data_uncorrected_err_count; | ||
35 | struct gk20a_ecc_stat sm_icache_corrected_err_count; | ||
36 | struct gk20a_ecc_stat sm_icache_uncorrected_err_count; | ||
37 | struct gk20a_ecc_stat gcc_l15_corrected_err_count; | ||
38 | struct gk20a_ecc_stat gcc_l15_uncorrected_err_count; | ||
39 | struct gk20a_ecc_stat fecs_corrected_err_count; | ||
40 | struct gk20a_ecc_stat fecs_uncorrected_err_count; | ||
41 | struct gk20a_ecc_stat gpccs_corrected_err_count; | ||
42 | struct gk20a_ecc_stat gpccs_uncorrected_err_count; | ||
43 | struct gk20a_ecc_stat mmu_l1tlb_corrected_err_count; | ||
44 | struct gk20a_ecc_stat mmu_l1tlb_uncorrected_err_count; | ||
45 | }; | ||
46 | |||
47 | struct ecc_ltc_t19x { | ||
48 | struct gk20a_ecc_stat l2_cache_corrected_err_count; | ||
49 | struct gk20a_ecc_stat l2_cache_uncorrected_err_count; | ||
50 | }; | ||
51 | |||
52 | /* TODO: PMU and FB ECC features are still under embargo */ | ||
53 | struct ecc_eng_t19x { | ||
54 | /* FB */ | ||
55 | struct gk20a_ecc_stat mmu_l2tlb_corrected_err_count; | ||
56 | struct gk20a_ecc_stat mmu_l2tlb_uncorrected_err_count; | ||
57 | struct gk20a_ecc_stat mmu_hubtlb_corrected_err_count; | ||
58 | struct gk20a_ecc_stat mmu_hubtlb_uncorrected_err_count; | ||
59 | struct gk20a_ecc_stat mmu_fillunit_corrected_err_count; | ||
60 | struct gk20a_ecc_stat mmu_fillunit_uncorrected_err_count; | ||
61 | /* PMU */ | ||
62 | }; | ||
63 | |||
64 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c new file mode 100644 index 00000000..ec487bdf --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c | |||
@@ -0,0 +1,1555 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | |||
27 | #include <nvgpu/dma.h> | ||
28 | #include <nvgpu/log.h> | ||
29 | #include <nvgpu/enabled.h> | ||
30 | #include <nvgpu/gmmu.h> | ||
31 | #include <nvgpu/barrier.h> | ||
32 | #include <nvgpu/soc.h> | ||
33 | |||
34 | #include "gk20a/gk20a.h" | ||
35 | #include "gk20a/mm_gk20a.h" | ||
36 | |||
37 | #include "gp10b/fb_gp10b.h" | ||
38 | |||
39 | #include "gv11b/fifo_gv11b.h" | ||
40 | #include "gv11b/fb_gv11b.h" | ||
41 | #include "gv11b/ce_gv11b.h" | ||
42 | |||
43 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
44 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
45 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
46 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
47 | |||
48 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
49 | struct mmu_fault_info *mmfault); | ||
50 | |||
51 | static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
52 | u32 invalidate_replay_val); | ||
53 | |||
54 | static void gv11b_init_nvlink_soc_credits(struct gk20a *g) | ||
55 | { | ||
56 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
57 | nvgpu_info(g, "nvlink soc credits init done by bpmp"); | ||
58 | } else { | ||
59 | /* MSS_NVLINK_1_BASE */ | ||
60 | void __iomem *soc1 = ioremap(0x01f20010, 4096); | ||
61 | /* MSS_NVLINK_2_BASE */ | ||
62 | void __iomem *soc2 = ioremap(0x01f40010, 4096); | ||
63 | /* MSS_NVLINK_3_BASE */ | ||
64 | void __iomem *soc3 = ioremap(0x01f60010, 4096); | ||
65 | /* MSS_NVLINK_4_BASE */ | ||
66 | void __iomem *soc4 = ioremap(0x01f80010, 4096); | ||
67 | u32 val; | ||
68 | |||
69 | nvgpu_info(g, "init nvlink soc credits"); | ||
70 | |||
71 | val = readl_relaxed(soc1); | ||
72 | writel_relaxed(val, soc1); | ||
73 | val = readl_relaxed(soc1 + 4); | ||
74 | writel_relaxed(val, soc1 + 4); | ||
75 | |||
76 | val = readl_relaxed(soc2); | ||
77 | writel_relaxed(val, soc2); | ||
78 | val = readl_relaxed(soc2 + 4); | ||
79 | writel_relaxed(val, soc2 + 4); | ||
80 | |||
81 | val = readl_relaxed(soc3); | ||
82 | writel_relaxed(val, soc3); | ||
83 | val = readl_relaxed(soc3 + 4); | ||
84 | writel_relaxed(val, soc3 + 4); | ||
85 | |||
86 | val = readl_relaxed(soc4); | ||
87 | writel_relaxed(val, soc4); | ||
88 | val = readl_relaxed(soc4 + 4); | ||
89 | writel_relaxed(val, soc4 + 4); | ||
90 | } | ||
91 | } | ||
92 | |||
93 | void gv11b_fb_init_fs_state(struct gk20a *g) | ||
94 | { | ||
95 | nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); | ||
96 | |||
97 | nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", | ||
98 | gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); | ||
99 | |||
100 | nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", | ||
101 | fb_mmu_num_active_ltcs_count_v( | ||
102 | gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); | ||
103 | } | ||
104 | |||
105 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) | ||
106 | { | ||
107 | u32 max_size = gr->max_comptag_mem; | ||
108 | /* one tag line covers 64KB */ | ||
109 | u32 max_comptag_lines = max_size << 4; | ||
110 | u32 compbit_base_post_divide; | ||
111 | u64 compbit_base_post_multiply64; | ||
112 | u64 compbit_store_iova; | ||
113 | u64 compbit_base_post_divide64; | ||
114 | |||
115 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) | ||
116 | compbit_store_iova = nvgpu_mem_get_phys_addr(g, | ||
117 | &gr->compbit_store.mem); | ||
118 | else | ||
119 | compbit_store_iova = nvgpu_mem_get_addr(g, | ||
120 | &gr->compbit_store.mem); | ||
121 | |||
122 | compbit_base_post_divide64 = compbit_store_iova >> | ||
123 | fb_mmu_cbc_base_address_alignment_shift_v(); | ||
124 | |||
125 | do_div(compbit_base_post_divide64, g->ltc_count); | ||
126 | compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); | ||
127 | |||
128 | compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * | ||
129 | g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); | ||
130 | |||
131 | if (compbit_base_post_multiply64 < compbit_store_iova) | ||
132 | compbit_base_post_divide++; | ||
133 | |||
134 | if (g->ops.ltc.cbc_fix_config) | ||
135 | compbit_base_post_divide = | ||
136 | g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); | ||
137 | |||
138 | gk20a_writel(g, fb_mmu_cbc_base_r(), | ||
139 | fb_mmu_cbc_base_address_f(compbit_base_post_divide)); | ||
140 | |||
141 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, | ||
142 | "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", | ||
143 | (u32)(compbit_store_iova >> 32), | ||
144 | (u32)(compbit_store_iova & 0xffffffff), | ||
145 | compbit_base_post_divide); | ||
146 | nvgpu_log(g, gpu_dbg_fn, "cbc base %x", | ||
147 | gk20a_readl(g, fb_mmu_cbc_base_r())); | ||
148 | |||
149 | gr->compbit_store.base_hw = compbit_base_post_divide; | ||
150 | |||
151 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, | ||
152 | 0, max_comptag_lines - 1); | ||
153 | |||
154 | } | ||
155 | |||
156 | void gv11b_fb_reset(struct gk20a *g) | ||
157 | { | ||
158 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
159 | nvgpu_log(g, gpu_dbg_info, "mc_elpg_enable set by bpmp"); | ||
160 | } else { | ||
161 | u32 mc_elpg_enable_val; | ||
162 | |||
163 | nvgpu_log(g, gpu_dbg_info, "enable xbar, pfb and hub"); | ||
164 | mc_elpg_enable_val = mc_elpg_enable_xbar_enabled_f() | | ||
165 | mc_elpg_enable_pfb_enabled_f() | | ||
166 | mc_elpg_enable_hub_enabled_f(); | ||
167 | mc_elpg_enable_val |= gk20a_readl(g, mc_elpg_enable_r()); | ||
168 | gk20a_writel(g, mc_elpg_enable_r(), mc_elpg_enable_val); | ||
169 | |||
170 | } | ||
171 | /* fs hub should be out of reset by now */ | ||
172 | gv11b_init_nvlink_soc_credits(g); | ||
173 | } | ||
174 | |||
175 | static const char * const invalid_str = "invalid"; | ||
176 | |||
177 | static const char *const fault_type_descs_gv11b[] = { | ||
178 | "invalid pde", | ||
179 | "invalid pde size", | ||
180 | "invalid pte", | ||
181 | "limit violation", | ||
182 | "unbound inst block", | ||
183 | "priv violation", | ||
184 | "write", | ||
185 | "read", | ||
186 | "pitch mask violation", | ||
187 | "work creation", | ||
188 | "unsupported aperture", | ||
189 | "compression failure", | ||
190 | "unsupported kind", | ||
191 | "region violation", | ||
192 | "poison", | ||
193 | "atomic" | ||
194 | }; | ||
195 | |||
196 | static const char *const fault_client_type_descs_gv11b[] = { | ||
197 | "gpc", | ||
198 | "hub", | ||
199 | }; | ||
200 | |||
201 | static const char *const fault_access_type_descs_gv11b[] = { | ||
202 | "virt read", | ||
203 | "virt write", | ||
204 | "virt atomic strong", | ||
205 | "virt prefetch", | ||
206 | "virt atomic weak", | ||
207 | "xxx", | ||
208 | "xxx", | ||
209 | "xxx", | ||
210 | "phys read", | ||
211 | "phys write", | ||
212 | "phys atomic", | ||
213 | "phys prefetch", | ||
214 | }; | ||
215 | |||
216 | static const char *const hub_client_descs_gv11b[] = { | ||
217 | "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", | ||
218 | "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", | ||
219 | "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", | ||
220 | "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", | ||
221 | "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", | ||
222 | "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", | ||
223 | "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", | ||
224 | "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", | ||
225 | "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", | ||
226 | "dont care" | ||
227 | }; | ||
228 | |||
229 | static const char *const gpc_client_descs_gv11b[] = { | ||
230 | "t1 0", "t1 1", "t1 2", "t1 3", | ||
231 | "t1 4", "t1 5", "t1 6", "t1 7", | ||
232 | "pe 0", "pe 1", "pe 2", "pe 3", | ||
233 | "pe 4", "pe 5", "pe 6", "pe 7", | ||
234 | "rast", "gcc", "gpccs", | ||
235 | "prop 0", "prop 1", "prop 2", "prop 3", | ||
236 | "gpm", | ||
237 | "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", | ||
238 | "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", | ||
239 | "utlb", | ||
240 | "t1 8", "t1 9", "t1 10", "t1 11", | ||
241 | "t1 12", "t1 13", "t1 14", "t1 15", | ||
242 | "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", | ||
243 | "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", | ||
244 | "pe 8", "pe 9", "tpccs 8", "tpccs 9", | ||
245 | "t1 16", "t1 17", "t1 18", "t1 19", | ||
246 | "pe 10", "pe 11", "tpccs 10", "tpccs 11", | ||
247 | "t1 20", "t1 21", "t1 22", "t1 23", | ||
248 | "pe 12", "pe 13", "tpccs 12", "tpccs 13", | ||
249 | "t1 24", "t1 25", "t1 26", "t1 27", | ||
250 | "pe 14", "pe 15", "tpccs 14", "tpccs 15", | ||
251 | "t1 28", "t1 29", "t1 30", "t1 31", | ||
252 | "pe 16", "pe 17", "tpccs 16", "tpccs 17", | ||
253 | "t1 32", "t1 33", "t1 34", "t1 35", | ||
254 | "pe 18", "pe 19", "tpccs 18", "tpccs 19", | ||
255 | "t1 36", "t1 37", "t1 38", "t1 39", | ||
256 | }; | ||
257 | |||
258 | u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g, | ||
259 | unsigned int index) | ||
260 | { | ||
261 | u32 reg_val; | ||
262 | |||
263 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
264 | return fb_mmu_fault_buffer_size_enable_v(reg_val); | ||
265 | } | ||
266 | |||
267 | static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, | ||
268 | unsigned int index, u32 next) | ||
269 | { | ||
270 | u32 reg_val; | ||
271 | |||
272 | nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); | ||
273 | |||
274 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
275 | reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), | ||
276 | fb_mmu_fault_buffer_get_ptr_f(next)); | ||
277 | |||
278 | /* while the fault is being handled it is possible for overflow | ||
279 | * to happen, | ||
280 | */ | ||
281 | if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) | ||
282 | reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); | ||
283 | |||
284 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
285 | |||
286 | /* make sure get ptr update is visible to everyone to avoid | ||
287 | * reading already read entry | ||
288 | */ | ||
289 | nvgpu_mb(); | ||
290 | } | ||
291 | |||
292 | static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, | ||
293 | unsigned int index) | ||
294 | { | ||
295 | u32 reg_val; | ||
296 | |||
297 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
298 | return fb_mmu_fault_buffer_get_ptr_v(reg_val); | ||
299 | } | ||
300 | |||
301 | static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, | ||
302 | unsigned int index) | ||
303 | { | ||
304 | u32 reg_val; | ||
305 | |||
306 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_put_r(index)); | ||
307 | return fb_mmu_fault_buffer_put_ptr_v(reg_val); | ||
308 | } | ||
309 | |||
310 | static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, | ||
311 | unsigned int index) | ||
312 | { | ||
313 | u32 reg_val; | ||
314 | |||
315 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
316 | return fb_mmu_fault_buffer_size_val_v(reg_val); | ||
317 | } | ||
318 | |||
319 | static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, | ||
320 | unsigned int index, u32 *get_idx) | ||
321 | { | ||
322 | u32 put_idx; | ||
323 | |||
324 | *get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
325 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
326 | |||
327 | return *get_idx == put_idx; | ||
328 | } | ||
329 | |||
330 | static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, | ||
331 | unsigned int index) | ||
332 | { | ||
333 | u32 get_idx, put_idx, entries; | ||
334 | |||
335 | |||
336 | get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
337 | |||
338 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
339 | |||
340 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
341 | |||
342 | return get_idx == ((put_idx + 1) % entries); | ||
343 | } | ||
344 | |||
345 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
346 | unsigned int index, unsigned int state) | ||
347 | { | ||
348 | u32 fault_status; | ||
349 | u32 reg_val; | ||
350 | |||
351 | nvgpu_log_fn(g, " "); | ||
352 | |||
353 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
354 | if (state) { | ||
355 | if (gv11b_fb_is_fault_buf_enabled(g, index)) { | ||
356 | nvgpu_log_info(g, "fault buffer is already enabled"); | ||
357 | } else { | ||
358 | reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); | ||
359 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), | ||
360 | reg_val); | ||
361 | } | ||
362 | |||
363 | } else { | ||
364 | struct nvgpu_timeout timeout; | ||
365 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
366 | |||
367 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
368 | NVGPU_TIMER_CPU_TIMER); | ||
369 | |||
370 | reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); | ||
371 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); | ||
372 | |||
373 | fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
374 | |||
375 | do { | ||
376 | if (!(fault_status & fb_mmu_fault_status_busy_true_f())) | ||
377 | break; | ||
378 | /* | ||
379 | * Make sure fault buffer is disabled. | ||
380 | * This is to avoid accessing fault buffer by hw | ||
381 | * during the window BAR2 is being unmapped by s/w | ||
382 | */ | ||
383 | nvgpu_log_info(g, "fault status busy set, check again"); | ||
384 | fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
385 | |||
386 | nvgpu_usleep_range(delay, delay * 2); | ||
387 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
388 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
389 | "fault status busy set")); | ||
390 | } | ||
391 | } | ||
392 | |||
393 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index) | ||
394 | { | ||
395 | u32 addr_lo; | ||
396 | u32 addr_hi; | ||
397 | |||
398 | nvgpu_log_fn(g, " "); | ||
399 | |||
400 | gv11b_fb_fault_buf_set_state_hw(g, index, | ||
401 | FAULT_BUF_DISABLED); | ||
402 | addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> | ||
403 | ram_in_base_shift_v()); | ||
404 | addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); | ||
405 | |||
406 | gk20a_writel(g, fb_mmu_fault_buffer_lo_r(index), | ||
407 | fb_mmu_fault_buffer_lo_addr_f(addr_lo)); | ||
408 | |||
409 | gk20a_writel(g, fb_mmu_fault_buffer_hi_r(index), | ||
410 | fb_mmu_fault_buffer_hi_addr_f(addr_hi)); | ||
411 | |||
412 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), | ||
413 | fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | | ||
414 | fb_mmu_fault_buffer_size_overflow_intr_enable_f()); | ||
415 | |||
416 | gv11b_fb_fault_buf_set_state_hw(g, index, FAULT_BUF_ENABLED); | ||
417 | } | ||
418 | |||
419 | static void gv11b_fb_intr_en_set(struct gk20a *g, | ||
420 | unsigned int index, u32 mask) | ||
421 | { | ||
422 | u32 reg_val; | ||
423 | |||
424 | reg_val = gk20a_readl(g, fb_niso_intr_en_set_r(index)); | ||
425 | reg_val |= mask; | ||
426 | gk20a_writel(g, fb_niso_intr_en_set_r(index), reg_val); | ||
427 | } | ||
428 | |||
429 | static void gv11b_fb_intr_en_clr(struct gk20a *g, | ||
430 | unsigned int index, u32 mask) | ||
431 | { | ||
432 | u32 reg_val; | ||
433 | |||
434 | reg_val = gk20a_readl(g, fb_niso_intr_en_clr_r(index)); | ||
435 | reg_val |= mask; | ||
436 | gk20a_writel(g, fb_niso_intr_en_clr_r(index), reg_val); | ||
437 | } | ||
438 | |||
439 | static u32 gv11b_fb_get_hub_intr_clr_mask(struct gk20a *g, | ||
440 | unsigned int intr_type) | ||
441 | { | ||
442 | u32 mask = 0; | ||
443 | |||
444 | if (intr_type & HUB_INTR_TYPE_OTHER) { | ||
445 | mask |= | ||
446 | fb_niso_intr_en_clr_mmu_other_fault_notify_m(); | ||
447 | } | ||
448 | |||
449 | if (intr_type & HUB_INTR_TYPE_NONREPLAY) { | ||
450 | mask |= | ||
451 | fb_niso_intr_en_clr_mmu_nonreplayable_fault_notify_m() | | ||
452 | fb_niso_intr_en_clr_mmu_nonreplayable_fault_overflow_m(); | ||
453 | } | ||
454 | |||
455 | if (intr_type & HUB_INTR_TYPE_REPLAY) { | ||
456 | mask |= | ||
457 | fb_niso_intr_en_clr_mmu_replayable_fault_notify_m() | | ||
458 | fb_niso_intr_en_clr_mmu_replayable_fault_overflow_m(); | ||
459 | } | ||
460 | |||
461 | if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) { | ||
462 | mask |= | ||
463 | fb_niso_intr_en_clr_mmu_ecc_uncorrected_error_notify_m(); | ||
464 | } | ||
465 | |||
466 | if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) { | ||
467 | mask |= | ||
468 | fb_niso_intr_en_clr_hub_access_counter_notify_m() | | ||
469 | fb_niso_intr_en_clr_hub_access_counter_error_m(); | ||
470 | } | ||
471 | |||
472 | return mask; | ||
473 | } | ||
474 | |||
475 | static u32 gv11b_fb_get_hub_intr_en_mask(struct gk20a *g, | ||
476 | unsigned int intr_type) | ||
477 | { | ||
478 | u32 mask = 0; | ||
479 | |||
480 | if (intr_type & HUB_INTR_TYPE_OTHER) { | ||
481 | mask |= | ||
482 | fb_niso_intr_en_set_mmu_other_fault_notify_m(); | ||
483 | } | ||
484 | |||
485 | if (intr_type & HUB_INTR_TYPE_NONREPLAY) { | ||
486 | mask |= | ||
487 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
488 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m(); | ||
489 | } | ||
490 | |||
491 | if (intr_type & HUB_INTR_TYPE_REPLAY) { | ||
492 | mask |= | ||
493 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
494 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); | ||
495 | } | ||
496 | |||
497 | if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) { | ||
498 | mask |= | ||
499 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
500 | } | ||
501 | |||
502 | if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) { | ||
503 | mask |= | ||
504 | fb_niso_intr_en_set_hub_access_counter_notify_m() | | ||
505 | fb_niso_intr_en_set_hub_access_counter_error_m(); | ||
506 | } | ||
507 | |||
508 | return mask; | ||
509 | } | ||
510 | |||
511 | void gv11b_fb_enable_hub_intr(struct gk20a *g, | ||
512 | unsigned int index, unsigned int intr_type) | ||
513 | { | ||
514 | u32 mask = 0; | ||
515 | |||
516 | mask = gv11b_fb_get_hub_intr_en_mask(g, intr_type); | ||
517 | |||
518 | if (mask) | ||
519 | gv11b_fb_intr_en_set(g, index, mask); | ||
520 | } | ||
521 | |||
522 | void gv11b_fb_disable_hub_intr(struct gk20a *g, | ||
523 | unsigned int index, unsigned int intr_type) | ||
524 | { | ||
525 | u32 mask = 0; | ||
526 | |||
527 | mask = gv11b_fb_get_hub_intr_clr_mask(g, intr_type); | ||
528 | |||
529 | if (mask) | ||
530 | gv11b_fb_intr_en_clr(g, index, mask); | ||
531 | } | ||
532 | |||
533 | static void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
534 | { | ||
535 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
536 | u32 corrected_delta, uncorrected_delta; | ||
537 | u32 corrected_overflow, uncorrected_overflow; | ||
538 | |||
539 | ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); | ||
540 | corrected_cnt = gk20a_readl(g, | ||
541 | fb_mmu_l2tlb_ecc_corrected_err_count_r()); | ||
542 | uncorrected_cnt = gk20a_readl(g, | ||
543 | fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); | ||
544 | |||
545 | corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( | ||
546 | corrected_cnt); | ||
547 | uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( | ||
548 | uncorrected_cnt); | ||
549 | corrected_overflow = ecc_status & | ||
550 | fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
551 | |||
552 | uncorrected_overflow = ecc_status & | ||
553 | fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
554 | |||
555 | /* clear the interrupt */ | ||
556 | if ((corrected_delta > 0) || corrected_overflow) | ||
557 | gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); | ||
558 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
559 | gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); | ||
560 | |||
561 | gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), | ||
562 | fb_mmu_l2tlb_ecc_status_reset_clear_f()); | ||
563 | |||
564 | /* Handle overflow */ | ||
565 | if (corrected_overflow) | ||
566 | corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); | ||
567 | if (uncorrected_overflow) | ||
568 | uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); | ||
569 | |||
570 | |||
571 | g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0] += | ||
572 | corrected_delta; | ||
573 | g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0] += | ||
574 | uncorrected_delta; | ||
575 | |||
576 | if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) | ||
577 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
578 | if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) | ||
579 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
580 | if (corrected_overflow || uncorrected_overflow) | ||
581 | nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); | ||
582 | |||
583 | nvgpu_log(g, gpu_dbg_intr, | ||
584 | "ecc error address: 0x%x", ecc_addr); | ||
585 | nvgpu_log(g, gpu_dbg_intr, | ||
586 | "ecc error count corrected: %d, uncorrected %d", | ||
587 | g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0], | ||
588 | g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0]); | ||
589 | } | ||
590 | |||
591 | static void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
592 | { | ||
593 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
594 | u32 corrected_delta, uncorrected_delta; | ||
595 | u32 corrected_overflow, uncorrected_overflow; | ||
596 | |||
597 | ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); | ||
598 | corrected_cnt = gk20a_readl(g, | ||
599 | fb_mmu_hubtlb_ecc_corrected_err_count_r()); | ||
600 | uncorrected_cnt = gk20a_readl(g, | ||
601 | fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); | ||
602 | |||
603 | corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( | ||
604 | corrected_cnt); | ||
605 | uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( | ||
606 | uncorrected_cnt); | ||
607 | corrected_overflow = ecc_status & | ||
608 | fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
609 | |||
610 | uncorrected_overflow = ecc_status & | ||
611 | fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
612 | |||
613 | /* clear the interrupt */ | ||
614 | if ((corrected_delta > 0) || corrected_overflow) | ||
615 | gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); | ||
616 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
617 | gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); | ||
618 | |||
619 | gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), | ||
620 | fb_mmu_hubtlb_ecc_status_reset_clear_f()); | ||
621 | |||
622 | /* Handle overflow */ | ||
623 | if (corrected_overflow) | ||
624 | corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); | ||
625 | if (uncorrected_overflow) | ||
626 | uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); | ||
627 | |||
628 | |||
629 | g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0] += | ||
630 | corrected_delta; | ||
631 | g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0] += | ||
632 | uncorrected_delta; | ||
633 | |||
634 | if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) | ||
635 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
636 | if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) | ||
637 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
638 | if (corrected_overflow || uncorrected_overflow) | ||
639 | nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); | ||
640 | |||
641 | nvgpu_log(g, gpu_dbg_intr, | ||
642 | "ecc error address: 0x%x", ecc_addr); | ||
643 | nvgpu_log(g, gpu_dbg_intr, | ||
644 | "ecc error count corrected: %d, uncorrected %d", | ||
645 | g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0], | ||
646 | g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0]); | ||
647 | } | ||
648 | |||
649 | static void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
650 | { | ||
651 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
652 | u32 corrected_delta, uncorrected_delta; | ||
653 | u32 corrected_overflow, uncorrected_overflow; | ||
654 | |||
655 | ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); | ||
656 | corrected_cnt = gk20a_readl(g, | ||
657 | fb_mmu_fillunit_ecc_corrected_err_count_r()); | ||
658 | uncorrected_cnt = gk20a_readl(g, | ||
659 | fb_mmu_fillunit_ecc_uncorrected_err_count_r()); | ||
660 | |||
661 | corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( | ||
662 | corrected_cnt); | ||
663 | uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( | ||
664 | uncorrected_cnt); | ||
665 | corrected_overflow = ecc_status & | ||
666 | fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); | ||
667 | |||
668 | uncorrected_overflow = ecc_status & | ||
669 | fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
670 | |||
671 | /* clear the interrupt */ | ||
672 | if ((corrected_delta > 0) || corrected_overflow) | ||
673 | gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); | ||
674 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
675 | gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); | ||
676 | |||
677 | gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), | ||
678 | fb_mmu_fillunit_ecc_status_reset_clear_f()); | ||
679 | |||
680 | /* Handle overflow */ | ||
681 | if (corrected_overflow) | ||
682 | corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); | ||
683 | if (uncorrected_overflow) | ||
684 | uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); | ||
685 | |||
686 | |||
687 | g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0] += | ||
688 | corrected_delta; | ||
689 | g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0] += | ||
690 | uncorrected_delta; | ||
691 | |||
692 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) | ||
693 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); | ||
694 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) | ||
695 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); | ||
696 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) | ||
697 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); | ||
698 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) | ||
699 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); | ||
700 | |||
701 | if (corrected_overflow || uncorrected_overflow) | ||
702 | nvgpu_info(g, "mmu fillunit ecc counter overflow!"); | ||
703 | |||
704 | nvgpu_log(g, gpu_dbg_intr, | ||
705 | "ecc error address: 0x%x", ecc_addr); | ||
706 | nvgpu_log(g, gpu_dbg_intr, | ||
707 | "ecc error count corrected: %d, uncorrected %d", | ||
708 | g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0], | ||
709 | g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0]); | ||
710 | } | ||
711 | |||
712 | static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) | ||
713 | { | ||
714 | if (WARN_ON(mmfault->fault_type >= | ||
715 | ARRAY_SIZE(fault_type_descs_gv11b))) | ||
716 | mmfault->fault_type_desc = invalid_str; | ||
717 | else | ||
718 | mmfault->fault_type_desc = | ||
719 | fault_type_descs_gv11b[mmfault->fault_type]; | ||
720 | |||
721 | if (WARN_ON(mmfault->client_type >= | ||
722 | ARRAY_SIZE(fault_client_type_descs_gv11b))) | ||
723 | mmfault->client_type_desc = invalid_str; | ||
724 | else | ||
725 | mmfault->client_type_desc = | ||
726 | fault_client_type_descs_gv11b[mmfault->client_type]; | ||
727 | |||
728 | mmfault->client_id_desc = invalid_str; | ||
729 | if (mmfault->client_type == | ||
730 | gmmu_fault_client_type_hub_v()) { | ||
731 | |||
732 | if (!(WARN_ON(mmfault->client_id >= | ||
733 | ARRAY_SIZE(hub_client_descs_gv11b)))) | ||
734 | mmfault->client_id_desc = | ||
735 | hub_client_descs_gv11b[mmfault->client_id]; | ||
736 | } else if (mmfault->client_type == | ||
737 | gmmu_fault_client_type_gpc_v()) { | ||
738 | if (!(WARN_ON(mmfault->client_id >= | ||
739 | ARRAY_SIZE(gpc_client_descs_gv11b)))) | ||
740 | mmfault->client_id_desc = | ||
741 | gpc_client_descs_gv11b[mmfault->client_id]; | ||
742 | } | ||
743 | |||
744 | } | ||
745 | |||
746 | static void gv11b_fb_print_fault_info(struct gk20a *g, | ||
747 | struct mmu_fault_info *mmfault) | ||
748 | { | ||
749 | if (mmfault && mmfault->valid) { | ||
750 | nvgpu_err(g, "[MMU FAULT] " | ||
751 | "mmu engine id: %d, " | ||
752 | "ch id: %d, " | ||
753 | "fault addr: 0x%llx, " | ||
754 | "fault addr aperture: %d, " | ||
755 | "fault type: %s, " | ||
756 | "access type: %s, ", | ||
757 | mmfault->mmu_engine_id, | ||
758 | mmfault->chid, | ||
759 | mmfault->fault_addr, | ||
760 | mmfault->fault_addr_aperture, | ||
761 | mmfault->fault_type_desc, | ||
762 | fault_access_type_descs_gv11b[mmfault->access_type]); | ||
763 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
764 | "mmu engine id: %d, " | ||
765 | "faulted act eng id if any: 0x%x, " | ||
766 | "faulted veid if any: 0x%x, " | ||
767 | "faulted pbdma id if any: 0x%x, " | ||
768 | "fault addr: 0x%llx, ", | ||
769 | mmfault->mmu_engine_id, | ||
770 | mmfault->faulted_engine, | ||
771 | mmfault->faulted_subid, | ||
772 | mmfault->faulted_pbdma, | ||
773 | mmfault->fault_addr); | ||
774 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
775 | "fault addr aperture: %d, " | ||
776 | "fault type: %s, " | ||
777 | "access type: %s, " | ||
778 | "inst ptr: 0x%llx, " | ||
779 | "inst ptr aperture: %d, ", | ||
780 | mmfault->fault_addr_aperture, | ||
781 | mmfault->fault_type_desc, | ||
782 | fault_access_type_descs_gv11b[mmfault->access_type], | ||
783 | mmfault->inst_ptr, | ||
784 | mmfault->inst_aperture); | ||
785 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
786 | "ch id: %d, " | ||
787 | "timestamp hi:lo 0x%08x:0x%08x, " | ||
788 | "client type: %s, " | ||
789 | "client id: %s, " | ||
790 | "gpc id if client type is gpc: %d, ", | ||
791 | mmfault->chid, | ||
792 | mmfault->timestamp_hi, mmfault->timestamp_lo, | ||
793 | mmfault->client_type_desc, | ||
794 | mmfault->client_id_desc, | ||
795 | mmfault->gpc_id); | ||
796 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
797 | "protected mode: %d, " | ||
798 | "replayable fault: %d, " | ||
799 | "replayable fault en: %d ", | ||
800 | mmfault->protected_mode, | ||
801 | mmfault->replayable_fault, | ||
802 | mmfault->replay_fault_en); | ||
803 | } | ||
804 | } | ||
805 | |||
806 | /* | ||
807 | *Fault buffer format | ||
808 | * | ||
809 | * 31 28 24 23 16 15 8 7 4 0 | ||
810 | *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. | ||
811 | *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| | ||
812 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
813 | *| inst_hi | | ||
814 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
815 | *| addr_31_12 | |AP | | ||
816 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
817 | *| addr_63_32 | | ||
818 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
819 | *| timestamp_lo | | ||
820 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
821 | *| timestamp_hi | | ||
822 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
823 | *| (reserved) | engine_id | | ||
824 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
825 | *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| | ||
826 | */ | ||
827 | |||
828 | static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, | ||
829 | struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) | ||
830 | { | ||
831 | u32 rd32_val; | ||
832 | u32 addr_lo, addr_hi; | ||
833 | u64 inst_ptr; | ||
834 | u32 chid = FIFO_INVAL_CHANNEL_ID; | ||
835 | struct channel_gk20a *refch; | ||
836 | |||
837 | memset(mmfault, 0, sizeof(*mmfault)); | ||
838 | |||
839 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
840 | gmmu_fault_buf_entry_inst_lo_w()); | ||
841 | addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); | ||
842 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
843 | |||
844 | addr_hi = nvgpu_mem_rd32(g, mem, offset + | ||
845 | gmmu_fault_buf_entry_inst_hi_w()); | ||
846 | addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); | ||
847 | |||
848 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
849 | |||
850 | /* refch will be put back after fault is handled */ | ||
851 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
852 | if (refch) | ||
853 | chid = refch->chid; | ||
854 | |||
855 | /* it is ok to continue even if refch is NULL */ | ||
856 | mmfault->refch = refch; | ||
857 | mmfault->chid = chid; | ||
858 | mmfault->inst_ptr = inst_ptr; | ||
859 | mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); | ||
860 | |||
861 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
862 | gmmu_fault_buf_entry_addr_lo_w()); | ||
863 | |||
864 | mmfault->fault_addr_aperture = | ||
865 | gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); | ||
866 | addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); | ||
867 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
868 | |||
869 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
870 | gmmu_fault_buf_entry_addr_hi_w()); | ||
871 | addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); | ||
872 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
873 | |||
874 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
875 | gmmu_fault_buf_entry_timestamp_lo_w()); | ||
876 | mmfault->timestamp_lo = | ||
877 | gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); | ||
878 | |||
879 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
880 | gmmu_fault_buf_entry_timestamp_hi_w()); | ||
881 | mmfault->timestamp_hi = | ||
882 | gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); | ||
883 | |||
884 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
885 | gmmu_fault_buf_entry_engine_id_w()); | ||
886 | |||
887 | mmfault->mmu_engine_id = | ||
888 | gmmu_fault_buf_entry_engine_id_v(rd32_val); | ||
889 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
890 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
891 | &mmfault->faulted_pbdma); | ||
892 | |||
893 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
894 | gmmu_fault_buf_entry_fault_type_w()); | ||
895 | mmfault->client_id = | ||
896 | gmmu_fault_buf_entry_client_v(rd32_val); | ||
897 | mmfault->replayable_fault = | ||
898 | gmmu_fault_buf_entry_replayable_fault_v(rd32_val); | ||
899 | |||
900 | mmfault->fault_type = | ||
901 | gmmu_fault_buf_entry_fault_type_v(rd32_val); | ||
902 | mmfault->access_type = | ||
903 | gmmu_fault_buf_entry_access_type_v(rd32_val); | ||
904 | |||
905 | mmfault->client_type = | ||
906 | gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); | ||
907 | |||
908 | mmfault->gpc_id = | ||
909 | gmmu_fault_buf_entry_gpc_id_v(rd32_val); | ||
910 | mmfault->protected_mode = | ||
911 | gmmu_fault_buf_entry_protected_mode_v(rd32_val); | ||
912 | |||
913 | mmfault->replay_fault_en = | ||
914 | gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); | ||
915 | |||
916 | mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); | ||
917 | |||
918 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
919 | gmmu_fault_buf_entry_fault_type_w()); | ||
920 | rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); | ||
921 | nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), | ||
922 | rd32_val); | ||
923 | |||
924 | gv11b_fb_parse_mmfault(mmfault); | ||
925 | } | ||
926 | |||
927 | static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, | ||
928 | struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) | ||
929 | { | ||
930 | unsigned int id_type; | ||
931 | u32 num_lce, act_eng_bitmask = 0; | ||
932 | int err = 0; | ||
933 | u32 id = ((u32)~0); | ||
934 | |||
935 | if (!mmfault->valid) | ||
936 | return; | ||
937 | |||
938 | gv11b_fb_print_fault_info(g, mmfault); | ||
939 | |||
940 | num_lce = gv11b_ce_get_num_lce(g); | ||
941 | if ((mmfault->mmu_engine_id >= | ||
942 | gmmu_fault_mmu_eng_id_ce0_v()) && | ||
943 | (mmfault->mmu_engine_id < | ||
944 | gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { | ||
945 | /* CE page faults are not reported as replayable */ | ||
946 | nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); | ||
947 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
948 | gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, | ||
949 | mmfault->faulted_pbdma, mmfault->faulted_engine); | ||
950 | if (!err) { | ||
951 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); | ||
952 | *invalidate_replay_val = 0; | ||
953 | /* refch in mmfault is assigned at the time of copying | ||
954 | * fault info from snap reg or bar2 fault buf | ||
955 | */ | ||
956 | gk20a_channel_put(mmfault->refch); | ||
957 | return; | ||
958 | } | ||
959 | /* Do recovery. Channel recovery needs refch */ | ||
960 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); | ||
961 | } | ||
962 | |||
963 | if (!mmfault->replayable_fault) { | ||
964 | if (mmfault->fault_type == | ||
965 | gmmu_fault_type_unbound_inst_block_v()) { | ||
966 | /* | ||
967 | * Bug 1847172: When an engine faults due to an unbound | ||
968 | * instance block, the fault cannot be isolated to a | ||
969 | * single context so we need to reset the entire runlist | ||
970 | */ | ||
971 | id_type = ID_TYPE_UNKNOWN; | ||
972 | nvgpu_log(g, gpu_dbg_intr, "UNBOUND INST BLOCK MMU FAULT"); | ||
973 | |||
974 | } else if (mmfault->refch) { | ||
975 | if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { | ||
976 | id = mmfault->refch->tsgid; | ||
977 | id_type = ID_TYPE_TSG; | ||
978 | } else { | ||
979 | id = mmfault->chid; | ||
980 | id_type = ID_TYPE_CHANNEL; | ||
981 | } | ||
982 | } else { | ||
983 | id_type = ID_TYPE_UNKNOWN; | ||
984 | } | ||
985 | if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
986 | act_eng_bitmask = BIT(mmfault->faulted_engine); | ||
987 | |||
988 | g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, | ||
989 | id, id_type, RC_TYPE_MMU_FAULT, mmfault); | ||
990 | } else { | ||
991 | if (mmfault->fault_type == gmmu_fault_type_pte_v()) { | ||
992 | nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); | ||
993 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
994 | if (err) | ||
995 | *invalidate_replay_val |= | ||
996 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
997 | else | ||
998 | *invalidate_replay_val |= | ||
999 | fb_mmu_invalidate_replay_start_ack_all_f(); | ||
1000 | } else { | ||
1001 | /* cancel faults other than invalid pte */ | ||
1002 | *invalidate_replay_val |= | ||
1003 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
1004 | } | ||
1005 | /* refch in mmfault is assigned at the time of copying | ||
1006 | * fault info from snap reg or bar2 fault buf | ||
1007 | */ | ||
1008 | gk20a_channel_put(mmfault->refch); | ||
1009 | } | ||
1010 | } | ||
1011 | |||
1012 | static void gv11b_fb_replay_or_cancel_faults(struct gk20a *g, | ||
1013 | u32 invalidate_replay_val) | ||
1014 | { | ||
1015 | int err = 0; | ||
1016 | |||
1017 | nvgpu_log_fn(g, " "); | ||
1018 | |||
1019 | if (invalidate_replay_val & | ||
1020 | fb_mmu_invalidate_replay_cancel_global_f()) { | ||
1021 | /* | ||
1022 | * cancel faults so that next time it faults as | ||
1023 | * replayable faults and channel recovery can be done | ||
1024 | */ | ||
1025 | err = gv11b_fb_mmu_invalidate_replay(g, | ||
1026 | fb_mmu_invalidate_replay_cancel_global_f()); | ||
1027 | } else if (invalidate_replay_val & | ||
1028 | fb_mmu_invalidate_replay_start_ack_all_f()) { | ||
1029 | /* pte valid is fixed. replay faulting request */ | ||
1030 | err = gv11b_fb_mmu_invalidate_replay(g, | ||
1031 | fb_mmu_invalidate_replay_start_ack_all_f()); | ||
1032 | } | ||
1033 | } | ||
1034 | |||
1035 | static void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, | ||
1036 | u32 fault_status, unsigned int index) | ||
1037 | { | ||
1038 | u32 get_indx, offset, rd32_val, entries; | ||
1039 | struct nvgpu_mem *mem; | ||
1040 | struct mmu_fault_info *mmfault; | ||
1041 | u32 invalidate_replay_val = 0; | ||
1042 | u64 prev_fault_addr = 0ULL; | ||
1043 | u64 next_fault_addr = 0ULL; | ||
1044 | |||
1045 | if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { | ||
1046 | nvgpu_log(g, gpu_dbg_intr, | ||
1047 | "SPURIOUS mmu fault: reg index:%d", index); | ||
1048 | return; | ||
1049 | } | ||
1050 | nvgpu_info(g, "%s MMU FAULT" , | ||
1051 | index == REPLAY_REG_INDEX ? "REPLAY" : "NON-REPLAY"); | ||
1052 | |||
1053 | nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); | ||
1054 | |||
1055 | mem = &g->mm.hw_fault_buf[index]; | ||
1056 | mmfault = g->mm.fault_info[index]; | ||
1057 | |||
1058 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
1059 | nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); | ||
1060 | |||
1061 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
1062 | nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); | ||
1063 | |||
1064 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
1065 | offset + gmmu_fault_buf_entry_valid_w()); | ||
1066 | nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); | ||
1067 | |||
1068 | while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { | ||
1069 | |||
1070 | nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); | ||
1071 | |||
1072 | gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); | ||
1073 | |||
1074 | get_indx = (get_indx + 1) % entries; | ||
1075 | nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); | ||
1076 | |||
1077 | gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); | ||
1078 | |||
1079 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
1080 | nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); | ||
1081 | |||
1082 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
1083 | offset + gmmu_fault_buf_entry_valid_w()); | ||
1084 | |||
1085 | if (index == REPLAY_REG_INDEX && mmfault->fault_addr != 0ULL) { | ||
1086 | /* fault_addr "0" is not supposed to be fixed ever. | ||
1087 | * For the first time when prev = 0, next = 0 and | ||
1088 | * fault addr is also 0 then handle_mmu_fault_common will | ||
1089 | * not be called. Fix by checking fault_addr not equal to 0 | ||
1090 | */ | ||
1091 | prev_fault_addr = next_fault_addr; | ||
1092 | next_fault_addr = mmfault->fault_addr; | ||
1093 | if (prev_fault_addr == next_fault_addr) { | ||
1094 | nvgpu_log(g, gpu_dbg_intr, "pte is fixed"); | ||
1095 | if (mmfault->refch) | ||
1096 | gk20a_channel_put(mmfault->refch); | ||
1097 | /* pte already fixed for this addr */ | ||
1098 | continue; | ||
1099 | } | ||
1100 | } | ||
1101 | |||
1102 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1103 | &invalidate_replay_val); | ||
1104 | |||
1105 | } | ||
1106 | if (index == REPLAY_REG_INDEX && invalidate_replay_val) | ||
1107 | gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); | ||
1108 | } | ||
1109 | |||
1110 | static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, | ||
1111 | u32 fault_status, struct mmu_fault_info *mmfault) | ||
1112 | { | ||
1113 | u32 reg_val; | ||
1114 | u32 addr_lo, addr_hi; | ||
1115 | u64 inst_ptr; | ||
1116 | int chid = FIFO_INVAL_CHANNEL_ID; | ||
1117 | struct channel_gk20a *refch; | ||
1118 | |||
1119 | memset(mmfault, 0, sizeof(*mmfault)); | ||
1120 | |||
1121 | if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { | ||
1122 | |||
1123 | nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); | ||
1124 | return; | ||
1125 | } | ||
1126 | |||
1127 | reg_val = gk20a_readl(g, fb_mmu_fault_inst_lo_r()); | ||
1128 | addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); | ||
1129 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1130 | |||
1131 | addr_hi = gk20a_readl(g, fb_mmu_fault_inst_hi_r()); | ||
1132 | addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); | ||
1133 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1134 | |||
1135 | /* refch will be put back after fault is handled */ | ||
1136 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
1137 | if (refch) | ||
1138 | chid = refch->chid; | ||
1139 | |||
1140 | /* It is still ok to continue if refch is NULL */ | ||
1141 | mmfault->refch = refch; | ||
1142 | mmfault->chid = chid; | ||
1143 | mmfault->inst_ptr = inst_ptr; | ||
1144 | mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); | ||
1145 | mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); | ||
1146 | |||
1147 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
1148 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
1149 | &mmfault->faulted_pbdma); | ||
1150 | |||
1151 | reg_val = gk20a_readl(g, fb_mmu_fault_addr_lo_r()); | ||
1152 | addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); | ||
1153 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1154 | |||
1155 | mmfault->fault_addr_aperture = | ||
1156 | fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); | ||
1157 | |||
1158 | addr_hi = gk20a_readl(g, fb_mmu_fault_addr_hi_r()); | ||
1159 | addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); | ||
1160 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1161 | |||
1162 | reg_val = gk20a_readl(g, fb_mmu_fault_info_r()); | ||
1163 | mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); | ||
1164 | mmfault->replayable_fault = | ||
1165 | fb_mmu_fault_info_replayable_fault_v(reg_val); | ||
1166 | mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); | ||
1167 | mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); | ||
1168 | mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); | ||
1169 | mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); | ||
1170 | mmfault->protected_mode = | ||
1171 | fb_mmu_fault_info_protected_mode_v(reg_val); | ||
1172 | mmfault->replay_fault_en = | ||
1173 | fb_mmu_fault_info_replayable_fault_en_v(reg_val); | ||
1174 | |||
1175 | mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); | ||
1176 | |||
1177 | fault_status &= ~(fb_mmu_fault_status_valid_m()); | ||
1178 | gk20a_writel(g, fb_mmu_fault_status_r(), fault_status); | ||
1179 | |||
1180 | gv11b_fb_parse_mmfault(mmfault); | ||
1181 | |||
1182 | } | ||
1183 | |||
1184 | static void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, | ||
1185 | u32 fault_status) | ||
1186 | { | ||
1187 | u32 reg_val; | ||
1188 | unsigned int index = REPLAY_REG_INDEX; | ||
1189 | |||
1190 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1191 | |||
1192 | if (fault_status & | ||
1193 | fb_mmu_fault_status_replayable_getptr_corrupted_m()) { | ||
1194 | |||
1195 | nvgpu_err(g, "replayable getptr corrupted set"); | ||
1196 | |||
1197 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1198 | |||
1199 | reg_val = set_field(reg_val, | ||
1200 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1201 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1202 | } | ||
1203 | |||
1204 | if (fault_status & | ||
1205 | fb_mmu_fault_status_replayable_overflow_m()) { | ||
1206 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1207 | |||
1208 | nvgpu_err(g, "replayable overflow: buffer full:%s", | ||
1209 | buffer_full?"true":"false"); | ||
1210 | |||
1211 | reg_val = set_field(reg_val, | ||
1212 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1213 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1214 | } | ||
1215 | |||
1216 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1217 | } | ||
1218 | |||
1219 | static void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, | ||
1220 | u32 fault_status) | ||
1221 | { | ||
1222 | u32 reg_val; | ||
1223 | unsigned int index = NONREPLAY_REG_INDEX; | ||
1224 | |||
1225 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1226 | |||
1227 | if (fault_status & | ||
1228 | fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { | ||
1229 | |||
1230 | nvgpu_err(g, "non replayable getptr corrupted set"); | ||
1231 | |||
1232 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1233 | |||
1234 | reg_val = set_field(reg_val, | ||
1235 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1236 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1237 | } | ||
1238 | |||
1239 | if (fault_status & | ||
1240 | fb_mmu_fault_status_non_replayable_overflow_m()) { | ||
1241 | |||
1242 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1243 | |||
1244 | nvgpu_err(g, "non replayable overflow: buffer full:%s", | ||
1245 | buffer_full?"true":"false"); | ||
1246 | |||
1247 | reg_val = set_field(reg_val, | ||
1248 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1249 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1250 | } | ||
1251 | |||
1252 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1253 | } | ||
1254 | |||
1255 | static void gv11b_fb_handle_bar2_fault(struct gk20a *g, | ||
1256 | struct mmu_fault_info *mmfault, u32 fault_status) | ||
1257 | { | ||
1258 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, | ||
1259 | HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY); | ||
1260 | |||
1261 | |||
1262 | if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { | ||
1263 | if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) | ||
1264 | gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX); | ||
1265 | } | ||
1266 | |||
1267 | if (fault_status & fb_mmu_fault_status_replayable_error_m()) { | ||
1268 | if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) | ||
1269 | gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX); | ||
1270 | } | ||
1271 | gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); | ||
1272 | |||
1273 | g->ops.mm.init_bar2_mm_hw_setup(g); | ||
1274 | |||
1275 | if (mmfault->refch) { | ||
1276 | gk20a_channel_put(mmfault->refch); | ||
1277 | mmfault->refch = NULL; | ||
1278 | } | ||
1279 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, | ||
1280 | HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY); | ||
1281 | } | ||
1282 | |||
1283 | static void gv11b_fb_handle_other_fault_notify(struct gk20a *g, | ||
1284 | u32 fault_status) | ||
1285 | { | ||
1286 | struct mmu_fault_info *mmfault; | ||
1287 | u32 invalidate_replay_val = 0; | ||
1288 | |||
1289 | mmfault = g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY]; | ||
1290 | |||
1291 | gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); | ||
1292 | |||
1293 | /* BAR2/Physical faults will not be snapped in hw fault buf */ | ||
1294 | if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { | ||
1295 | nvgpu_err(g, "BAR2 MMU FAULT"); | ||
1296 | gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); | ||
1297 | |||
1298 | } else if (mmfault->mmu_engine_id == | ||
1299 | gmmu_fault_mmu_eng_id_physical_v()) { | ||
1300 | /* usually means VPR or out of bounds physical accesses */ | ||
1301 | nvgpu_err(g, "PHYSICAL MMU FAULT"); | ||
1302 | |||
1303 | } else { | ||
1304 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1305 | &invalidate_replay_val); | ||
1306 | |||
1307 | if (invalidate_replay_val) | ||
1308 | gv11b_fb_replay_or_cancel_faults(g, | ||
1309 | invalidate_replay_val); | ||
1310 | } | ||
1311 | } | ||
1312 | |||
1313 | static void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) | ||
1314 | { | ||
1315 | u32 dropped_faults = 0; | ||
1316 | |||
1317 | dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | | ||
1318 | fb_mmu_fault_status_dropped_bar1_virt_set_f() | | ||
1319 | fb_mmu_fault_status_dropped_bar2_phys_set_f() | | ||
1320 | fb_mmu_fault_status_dropped_bar2_virt_set_f() | | ||
1321 | fb_mmu_fault_status_dropped_ifb_phys_set_f() | | ||
1322 | fb_mmu_fault_status_dropped_ifb_virt_set_f() | | ||
1323 | fb_mmu_fault_status_dropped_other_phys_set_f()| | ||
1324 | fb_mmu_fault_status_dropped_other_virt_set_f(); | ||
1325 | |||
1326 | if (fault_status & dropped_faults) { | ||
1327 | nvgpu_err(g, "dropped mmu fault (0x%08x)", | ||
1328 | fault_status & dropped_faults); | ||
1329 | gk20a_writel(g, fb_mmu_fault_status_r(), dropped_faults); | ||
1330 | } | ||
1331 | } | ||
1332 | |||
1333 | |||
1334 | static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) | ||
1335 | { | ||
1336 | u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
1337 | |||
1338 | nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); | ||
1339 | |||
1340 | if (niso_intr & | ||
1341 | fb_niso_intr_mmu_other_fault_notify_m()) { | ||
1342 | |||
1343 | gv11b_fb_handle_dropped_mmu_fault(g, fault_status); | ||
1344 | |||
1345 | gv11b_fb_handle_other_fault_notify(g, fault_status); | ||
1346 | } | ||
1347 | |||
1348 | if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) { | ||
1349 | |||
1350 | if (niso_intr & | ||
1351 | fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { | ||
1352 | |||
1353 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1354 | fault_status, NONREPLAY_REG_INDEX); | ||
1355 | |||
1356 | /* | ||
1357 | * When all the faults are processed, | ||
1358 | * GET and PUT will have same value and mmu fault status | ||
1359 | * bit will be reset by HW | ||
1360 | */ | ||
1361 | } | ||
1362 | if (niso_intr & | ||
1363 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { | ||
1364 | |||
1365 | gv11b_fb_handle_nonreplay_fault_overflow(g, | ||
1366 | fault_status); | ||
1367 | } | ||
1368 | |||
1369 | } | ||
1370 | |||
1371 | if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) { | ||
1372 | |||
1373 | if (niso_intr & | ||
1374 | fb_niso_intr_mmu_replayable_fault_notify_m()) { | ||
1375 | |||
1376 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1377 | fault_status, REPLAY_REG_INDEX); | ||
1378 | } | ||
1379 | if (niso_intr & | ||
1380 | fb_niso_intr_mmu_replayable_fault_overflow_m()) { | ||
1381 | |||
1382 | gv11b_fb_handle_replay_fault_overflow(g, | ||
1383 | fault_status); | ||
1384 | } | ||
1385 | |||
1386 | } | ||
1387 | |||
1388 | nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); | ||
1389 | gk20a_writel(g, fb_mmu_fault_status_r(), | ||
1390 | fb_mmu_fault_status_valid_clear_f()); | ||
1391 | } | ||
1392 | |||
1393 | void gv11b_fb_hub_isr(struct gk20a *g) | ||
1394 | { | ||
1395 | u32 status, niso_intr; | ||
1396 | |||
1397 | nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); | ||
1398 | |||
1399 | niso_intr = gk20a_readl(g, fb_niso_intr_r()); | ||
1400 | |||
1401 | nvgpu_info(g, "enter hub isr, niso_intr = 0x%08x", niso_intr); | ||
1402 | |||
1403 | if (niso_intr & | ||
1404 | (fb_niso_intr_hub_access_counter_notify_m() | | ||
1405 | fb_niso_intr_hub_access_counter_error_m())) { | ||
1406 | |||
1407 | nvgpu_info(g, "hub access counter notify/error"); | ||
1408 | } | ||
1409 | if (niso_intr & | ||
1410 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { | ||
1411 | |||
1412 | nvgpu_info(g, "ecc uncorrected error notify"); | ||
1413 | |||
1414 | /* disable interrupts during handling */ | ||
1415 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, | ||
1416 | HUB_INTR_TYPE_ECC_UNCORRECTED); | ||
1417 | |||
1418 | status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); | ||
1419 | if (status) | ||
1420 | gv11b_handle_l2tlb_ecc_isr(g, status); | ||
1421 | |||
1422 | status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); | ||
1423 | if (status) | ||
1424 | gv11b_handle_hubtlb_ecc_isr(g, status); | ||
1425 | |||
1426 | status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); | ||
1427 | if (status) | ||
1428 | gv11b_handle_fillunit_ecc_isr(g, status); | ||
1429 | |||
1430 | /* re-enable interrupts after handling */ | ||
1431 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, | ||
1432 | HUB_INTR_TYPE_ECC_UNCORRECTED); | ||
1433 | |||
1434 | } | ||
1435 | if (niso_intr & | ||
1436 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1437 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1438 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1439 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1440 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { | ||
1441 | |||
1442 | nvgpu_info(g, "MMU Fault"); | ||
1443 | gv11b_fb_handle_mmu_fault(g, niso_intr); | ||
1444 | } | ||
1445 | |||
1446 | nvgpu_mutex_release(&g->mm.hub_isr_mutex); | ||
1447 | } | ||
1448 | |||
1449 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g) | ||
1450 | { | ||
1451 | if (gk20a_readl(g, fb_niso_intr_r()) & | ||
1452 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1453 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | | ||
1454 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1455 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1456 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1457 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) | ||
1458 | return true; | ||
1459 | |||
1460 | return false; | ||
1461 | } | ||
1462 | |||
1463 | static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
1464 | u32 invalidate_replay_val) | ||
1465 | { | ||
1466 | int err = -ETIMEDOUT; | ||
1467 | u32 reg_val; | ||
1468 | struct nvgpu_timeout timeout; | ||
1469 | |||
1470 | gk20a_dbg_fn(""); | ||
1471 | |||
1472 | nvgpu_mutex_acquire(&g->mm.tlb_lock); | ||
1473 | |||
1474 | reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); | ||
1475 | |||
1476 | reg_val |= fb_mmu_invalidate_all_va_true_f() | | ||
1477 | fb_mmu_invalidate_all_pdb_true_f() | | ||
1478 | invalidate_replay_val | | ||
1479 | fb_mmu_invalidate_trigger_true_f(); | ||
1480 | |||
1481 | gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); | ||
1482 | |||
1483 | /* retry 200 times */ | ||
1484 | nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); | ||
1485 | do { | ||
1486 | reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
1487 | if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != | ||
1488 | fb_mmu_ctrl_pri_fifo_empty_false_f()) { | ||
1489 | err = 0; | ||
1490 | break; | ||
1491 | } | ||
1492 | nvgpu_udelay(5); | ||
1493 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
1494 | "invalidate replay failed on 0x%llx")); | ||
1495 | if (err) | ||
1496 | nvgpu_err(g, "invalidate replay timedout"); | ||
1497 | |||
1498 | nvgpu_mutex_release(&g->mm.tlb_lock); | ||
1499 | |||
1500 | return err; | ||
1501 | } | ||
1502 | |||
1503 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
1504 | struct mmu_fault_info *mmfault) | ||
1505 | { | ||
1506 | int err = 0; | ||
1507 | u32 pte[2]; | ||
1508 | |||
1509 | if (mmfault->refch == NULL) { | ||
1510 | nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); | ||
1511 | return -EINVAL; | ||
1512 | } | ||
1513 | |||
1514 | err = __nvgpu_get_pte(g, | ||
1515 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1516 | if (err) { | ||
1517 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); | ||
1518 | return err; | ||
1519 | } | ||
1520 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1521 | "pte: %#08x %#08x", pte[1], pte[0]); | ||
1522 | |||
1523 | if (pte[0] == 0x0 && pte[1] == 0x0) { | ||
1524 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1525 | "pte all zeros, do not set valid"); | ||
1526 | return -1; | ||
1527 | } | ||
1528 | if (pte[0] & gmmu_new_pte_valid_true_f()) { | ||
1529 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1530 | "pte valid already set"); | ||
1531 | return -1; | ||
1532 | } | ||
1533 | |||
1534 | pte[0] |= gmmu_new_pte_valid_true_f(); | ||
1535 | if (pte[0] & gmmu_new_pte_read_only_true_f()) | ||
1536 | pte[0] &= ~(gmmu_new_pte_read_only_true_f()); | ||
1537 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1538 | "new pte: %#08x %#08x", pte[1], pte[0]); | ||
1539 | |||
1540 | err = __nvgpu_set_pte(g, | ||
1541 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1542 | if (err) { | ||
1543 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); | ||
1544 | return err; | ||
1545 | } | ||
1546 | /* invalidate tlb so that GMMU does not use old cached translation */ | ||
1547 | g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); | ||
1548 | |||
1549 | err = __nvgpu_get_pte(g, | ||
1550 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1551 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1552 | "pte after tlb invalidate: %#08x %#08x", | ||
1553 | pte[1], pte[0]); | ||
1554 | return err; | ||
1555 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.h b/drivers/gpu/nvgpu/gv11b/fb_gv11b.h new file mode 100644 index 00000000..d9a6fa77 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.h | |||
@@ -0,0 +1,72 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_GV11B_FB | ||
26 | #define _NVGPU_GV11B_FB | ||
27 | |||
28 | #define STALL_REG_INDEX 0 | ||
29 | #define NONSTALL_REG_INDEX 1 | ||
30 | |||
31 | #define NONREPLAY_REG_INDEX 0 | ||
32 | #define REPLAY_REG_INDEX 1 | ||
33 | |||
34 | #define FAULT_BUF_DISABLED 0 | ||
35 | #define FAULT_BUF_ENABLED 1 | ||
36 | |||
37 | #define FAULT_BUF_INVALID 0 | ||
38 | #define FAULT_BUF_VALID 1 | ||
39 | |||
40 | #define HUB_INTR_TYPE_OTHER 1 /* bit 0 */ | ||
41 | #define HUB_INTR_TYPE_NONREPLAY 2 /* bit 1 */ | ||
42 | #define HUB_INTR_TYPE_REPLAY 4 /* bit 2 */ | ||
43 | #define HUB_INTR_TYPE_ECC_UNCORRECTED 8 /* bit 3 */ | ||
44 | #define HUB_INTR_TYPE_ACCESS_COUNTER 16 /* bit 4 */ | ||
45 | #define HUB_INTR_TYPE_ALL (HUB_INTR_TYPE_OTHER | \ | ||
46 | HUB_INTR_TYPE_NONREPLAY | \ | ||
47 | HUB_INTR_TYPE_REPLAY | \ | ||
48 | HUB_INTR_TYPE_ECC_UNCORRECTED | \ | ||
49 | HUB_INTR_TYPE_ACCESS_COUNTER) | ||
50 | |||
51 | #define FAULT_TYPE_OTHER_AND_NONREPLAY 0 | ||
52 | #define FAULT_TYPE_REPLAY 1 | ||
53 | |||
54 | struct gk20a; | ||
55 | |||
56 | void gv11b_fb_init_fs_state(struct gk20a *g); | ||
57 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr); | ||
58 | void gv11b_fb_reset(struct gk20a *g); | ||
59 | void gv11b_fb_hub_isr(struct gk20a *g); | ||
60 | |||
61 | u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g, | ||
62 | unsigned int index); | ||
63 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
64 | unsigned int index, unsigned int state); | ||
65 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index); | ||
66 | void gv11b_fb_enable_hub_intr(struct gk20a *g, | ||
67 | unsigned int index, unsigned int intr_type); | ||
68 | void gv11b_fb_disable_hub_intr(struct gk20a *g, | ||
69 | unsigned int index, unsigned int intr_type); | ||
70 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g); | ||
71 | |||
72 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c new file mode 100644 index 00000000..f87c6dea --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | |||
@@ -0,0 +1,1907 @@ | |||
1 | /* | ||
2 | * GV11B fifo | ||
3 | * | ||
4 | * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | #include <linux/delay.h> | ||
25 | #include <linux/types.h> | ||
26 | |||
27 | #include <nvgpu/semaphore.h> | ||
28 | #include <nvgpu/timers.h> | ||
29 | #include <nvgpu/log.h> | ||
30 | #include <nvgpu/dma.h> | ||
31 | #include <nvgpu/nvgpu_mem.h> | ||
32 | #include <nvgpu/gmmu.h> | ||
33 | #include <nvgpu/soc.h> | ||
34 | #include <nvgpu/debug.h> | ||
35 | #include <nvgpu/nvhost_t19x.h> | ||
36 | #include <nvgpu/barrier.h> | ||
37 | #include <nvgpu/mm.h> | ||
38 | #include <nvgpu/ctxsw_trace.h> | ||
39 | |||
40 | #include "gk20a/gk20a.h" | ||
41 | #include "gk20a/fifo_gk20a.h" | ||
42 | #include "gk20a/channel_gk20a.h" | ||
43 | |||
44 | #include "gp10b/fifo_gp10b.h" | ||
45 | |||
46 | #include <nvgpu/hw/gv11b/hw_pbdma_gv11b.h> | ||
47 | #include <nvgpu/hw/gv11b/hw_fifo_gv11b.h> | ||
48 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
49 | #include <nvgpu/hw/gv11b/hw_ccsr_gv11b.h> | ||
50 | #include <nvgpu/hw/gv11b/hw_usermode_gv11b.h> | ||
51 | #include <nvgpu/hw/gv11b/hw_top_gv11b.h> | ||
52 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
53 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
54 | #include <nvgpu/hw/gv11b/hw_gr_gv11b.h> | ||
55 | |||
56 | #include "fifo_gv11b.h" | ||
57 | #include "subctx_gv11b.h" | ||
58 | #include "gr_gv11b.h" | ||
59 | #include "mc_gv11b.h" | ||
60 | |||
61 | #define PBDMA_SUBDEVICE_ID 1 | ||
62 | |||
63 | static void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g, | ||
64 | struct channel_gk20a *ch, struct nvgpu_mem *mem); | ||
65 | |||
66 | void gv11b_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist) | ||
67 | { | ||
68 | |||
69 | u32 runlist_entry_0 = ram_rl_entry_type_tsg_v(); | ||
70 | |||
71 | if (tsg->timeslice_timeout) | ||
72 | runlist_entry_0 |= | ||
73 | ram_rl_entry_tsg_timeslice_scale_f(tsg->timeslice_scale) | | ||
74 | ram_rl_entry_tsg_timeslice_timeout_f(tsg->timeslice_timeout); | ||
75 | else | ||
76 | runlist_entry_0 |= | ||
77 | ram_rl_entry_tsg_timeslice_scale_f( | ||
78 | ram_rl_entry_tsg_timeslice_scale_3_v()) | | ||
79 | ram_rl_entry_tsg_timeslice_timeout_f( | ||
80 | ram_rl_entry_tsg_timeslice_timeout_128_v()); | ||
81 | |||
82 | runlist[0] = runlist_entry_0; | ||
83 | runlist[1] = ram_rl_entry_tsg_length_f(tsg->num_active_channels); | ||
84 | runlist[2] = ram_rl_entry_tsg_tsgid_f(tsg->tsgid); | ||
85 | runlist[3] = 0; | ||
86 | |||
87 | gk20a_dbg_info("gv11b tsg runlist [0] %x [1] %x [2] %x [3] %x\n", | ||
88 | runlist[0], runlist[1], runlist[2], runlist[3]); | ||
89 | |||
90 | } | ||
91 | |||
92 | void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist) | ||
93 | { | ||
94 | struct gk20a *g = c->g; | ||
95 | u32 addr_lo, addr_hi; | ||
96 | u32 runlist_entry; | ||
97 | |||
98 | /* Time being use 0 pbdma sequencer */ | ||
99 | runlist_entry = ram_rl_entry_type_channel_v() | | ||
100 | ram_rl_entry_chan_runqueue_selector_f( | ||
101 | c->t19x.runqueue_sel) | | ||
102 | ram_rl_entry_chan_userd_target_f( | ||
103 | nvgpu_aperture_mask(g, &g->fifo.userd, | ||
104 | ram_rl_entry_chan_userd_target_sys_mem_ncoh_v(), | ||
105 | ram_rl_entry_chan_userd_target_vid_mem_v())) | | ||
106 | ram_rl_entry_chan_inst_target_f( | ||
107 | nvgpu_aperture_mask(g, &c->inst_block, | ||
108 | ram_rl_entry_chan_inst_target_sys_mem_ncoh_v(), | ||
109 | ram_rl_entry_chan_inst_target_vid_mem_v())); | ||
110 | |||
111 | addr_lo = u64_lo32(c->userd_iova) >> | ||
112 | ram_rl_entry_chan_userd_ptr_align_shift_v(); | ||
113 | addr_hi = u64_hi32(c->userd_iova); | ||
114 | runlist[0] = runlist_entry | ram_rl_entry_chan_userd_ptr_lo_f(addr_lo); | ||
115 | runlist[1] = ram_rl_entry_chan_userd_ptr_hi_f(addr_hi); | ||
116 | |||
117 | addr_lo = u64_lo32(nvgpu_inst_block_addr(g, &c->inst_block)) >> | ||
118 | ram_rl_entry_chan_inst_ptr_align_shift_v(); | ||
119 | addr_hi = u64_hi32(nvgpu_inst_block_addr(g, &c->inst_block)); | ||
120 | |||
121 | runlist[2] = ram_rl_entry_chan_inst_ptr_lo_f(addr_lo) | | ||
122 | ram_rl_entry_chid_f(c->chid); | ||
123 | runlist[3] = ram_rl_entry_chan_inst_ptr_hi_f(addr_hi); | ||
124 | |||
125 | gk20a_dbg_info("gv11b channel runlist [0] %x [1] %x [2] %x [3] %x\n", | ||
126 | runlist[0], runlist[1], runlist[2], runlist[3]); | ||
127 | } | ||
128 | |||
129 | static void gv11b_userd_writeback_config(struct gk20a *g) | ||
130 | { | ||
131 | gk20a_writel(g, fifo_userd_writeback_r(), fifo_userd_writeback_timer_f( | ||
132 | fifo_userd_writeback_timer_100us_v())); | ||
133 | |||
134 | |||
135 | } | ||
136 | |||
137 | int channel_gv11b_setup_ramfc(struct channel_gk20a *c, | ||
138 | u64 gpfifo_base, u32 gpfifo_entries, | ||
139 | unsigned long acquire_timeout, u32 flags) | ||
140 | { | ||
141 | struct gk20a *g = c->g; | ||
142 | struct nvgpu_mem *mem = &c->inst_block; | ||
143 | u32 data; | ||
144 | |||
145 | gk20a_dbg_fn(""); | ||
146 | |||
147 | nvgpu_memset(g, mem, 0, 0, ram_fc_size_val_v()); | ||
148 | |||
149 | nvgpu_mem_wr32(g, mem, ram_fc_gp_base_w(), | ||
150 | pbdma_gp_base_offset_f( | ||
151 | u64_lo32(gpfifo_base >> pbdma_gp_base_rsvd_s()))); | ||
152 | |||
153 | nvgpu_mem_wr32(g, mem, ram_fc_gp_base_hi_w(), | ||
154 | pbdma_gp_base_hi_offset_f(u64_hi32(gpfifo_base)) | | ||
155 | pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries))); | ||
156 | |||
157 | nvgpu_mem_wr32(g, mem, ram_fc_signature_w(), | ||
158 | c->g->ops.fifo.get_pbdma_signature(c->g)); | ||
159 | |||
160 | nvgpu_mem_wr32(g, mem, ram_fc_pb_header_w(), | ||
161 | pbdma_pb_header_priv_user_f() | | ||
162 | pbdma_pb_header_method_zero_f() | | ||
163 | pbdma_pb_header_subchannel_zero_f() | | ||
164 | pbdma_pb_header_level_main_f() | | ||
165 | pbdma_pb_header_first_true_f() | | ||
166 | pbdma_pb_header_type_inc_f()); | ||
167 | |||
168 | nvgpu_mem_wr32(g, mem, ram_fc_subdevice_w(), | ||
169 | pbdma_subdevice_id_f(PBDMA_SUBDEVICE_ID) | | ||
170 | pbdma_subdevice_status_active_f() | | ||
171 | pbdma_subdevice_channel_dma_enable_f()); | ||
172 | |||
173 | nvgpu_mem_wr32(g, mem, ram_fc_target_w(), | ||
174 | pbdma_target_eng_ctx_valid_true_f() | | ||
175 | pbdma_target_ce_ctx_valid_true_f() | | ||
176 | pbdma_target_engine_sw_f()); | ||
177 | |||
178 | nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(), | ||
179 | g->ops.fifo.pbdma_acquire_val(acquire_timeout)); | ||
180 | |||
181 | nvgpu_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(), | ||
182 | pbdma_runlist_timeslice_timeout_128_f() | | ||
183 | pbdma_runlist_timeslice_timescale_3_f() | | ||
184 | pbdma_runlist_timeslice_enable_true_f()); | ||
185 | |||
186 | |||
187 | nvgpu_mem_wr32(g, mem, ram_fc_chid_w(), ram_fc_chid_id_f(c->chid)); | ||
188 | |||
189 | if (c->t19x.subctx_id == CHANNEL_INFO_VEID0) | ||
190 | nvgpu_mem_wr32(g, mem, ram_fc_set_channel_info_w(), | ||
191 | pbdma_set_channel_info_scg_type_graphics_compute0_f() | | ||
192 | pbdma_set_channel_info_veid_f(c->t19x.subctx_id)); | ||
193 | else | ||
194 | nvgpu_mem_wr32(g, mem, ram_fc_set_channel_info_w(), | ||
195 | pbdma_set_channel_info_scg_type_compute1_f() | | ||
196 | pbdma_set_channel_info_veid_f(c->t19x.subctx_id)); | ||
197 | |||
198 | gv11b_fifo_init_ramfc_eng_method_buffer(g, c, mem); | ||
199 | |||
200 | if (c->is_privileged_channel) { | ||
201 | /* Set privilege level for channel */ | ||
202 | nvgpu_mem_wr32(g, mem, ram_fc_config_w(), | ||
203 | pbdma_config_auth_level_privileged_f()); | ||
204 | |||
205 | gk20a_fifo_setup_ramfc_for_privileged_channel(c); | ||
206 | } | ||
207 | |||
208 | /* Enable userd writeback */ | ||
209 | data = nvgpu_mem_rd32(g, mem, ram_fc_config_w()); | ||
210 | data = data | pbdma_config_userd_writeback_enable_f(); | ||
211 | nvgpu_mem_wr32(g, mem, ram_fc_config_w(),data); | ||
212 | |||
213 | gv11b_userd_writeback_config(g); | ||
214 | |||
215 | return channel_gp10b_commit_userd(c); | ||
216 | } | ||
217 | |||
218 | |||
219 | static void gv11b_ring_channel_doorbell(struct channel_gk20a *c) | ||
220 | { | ||
221 | struct fifo_gk20a *f = &c->g->fifo; | ||
222 | u32 hw_chid = f->channel_base + c->chid; | ||
223 | |||
224 | gk20a_dbg_info("channel ring door bell %d\n", c->chid); | ||
225 | |||
226 | gv11b_usermode_writel(c->g, usermode_notify_channel_pending_r(), | ||
227 | usermode_notify_channel_pending_id_f(hw_chid)); | ||
228 | } | ||
229 | |||
230 | u32 gv11b_userd_gp_get(struct gk20a *g, struct channel_gk20a *c) | ||
231 | { | ||
232 | struct nvgpu_mem *userd_mem = &g->fifo.userd; | ||
233 | u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32)); | ||
234 | |||
235 | return nvgpu_mem_rd32(g, userd_mem, | ||
236 | offset + ram_userd_gp_get_w()); | ||
237 | } | ||
238 | |||
239 | u64 gv11b_userd_pb_get(struct gk20a *g, struct channel_gk20a *c) | ||
240 | { | ||
241 | struct nvgpu_mem *userd_mem = &g->fifo.userd; | ||
242 | u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32)); | ||
243 | u32 lo = nvgpu_mem_rd32(g, userd_mem, offset + ram_userd_get_w()); | ||
244 | u32 hi = nvgpu_mem_rd32(g, userd_mem, offset + ram_userd_get_hi_w()); | ||
245 | |||
246 | return ((u64)hi << 32) | lo; | ||
247 | } | ||
248 | |||
249 | void gv11b_userd_gp_put(struct gk20a *g, struct channel_gk20a *c) | ||
250 | { | ||
251 | struct nvgpu_mem *userd_mem = &g->fifo.userd; | ||
252 | u32 offset = c->chid * (g->fifo.userd_entry_size / sizeof(u32)); | ||
253 | |||
254 | nvgpu_mem_wr32(g, userd_mem, offset + ram_userd_gp_put_w(), | ||
255 | c->gpfifo.put); | ||
256 | /* commit everything to cpu */ | ||
257 | nvgpu_smp_mb(); | ||
258 | |||
259 | gv11b_ring_channel_doorbell(c); | ||
260 | } | ||
261 | |||
262 | void channel_gv11b_unbind(struct channel_gk20a *ch) | ||
263 | { | ||
264 | struct gk20a *g = ch->g; | ||
265 | |||
266 | gk20a_dbg_fn(""); | ||
267 | |||
268 | if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) { | ||
269 | gk20a_writel(g, ccsr_channel_inst_r(ch->chid), | ||
270 | ccsr_channel_inst_ptr_f(0) | | ||
271 | ccsr_channel_inst_bind_false_f()); | ||
272 | |||
273 | gk20a_writel(g, ccsr_channel_r(ch->chid), | ||
274 | ccsr_channel_enable_clr_true_f() | | ||
275 | ccsr_channel_pbdma_faulted_reset_f() | | ||
276 | ccsr_channel_eng_faulted_reset_f()); | ||
277 | } | ||
278 | } | ||
279 | |||
280 | u32 gv11b_fifo_get_num_fifos(struct gk20a *g) | ||
281 | { | ||
282 | return ccsr_channel__size_1_v(); | ||
283 | } | ||
284 | |||
285 | bool gv11b_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid) | ||
286 | { | ||
287 | return (engine_subid == gmmu_fault_client_type_gpc_v()); | ||
288 | } | ||
289 | |||
290 | void gv11b_dump_channel_status_ramfc(struct gk20a *g, | ||
291 | struct gk20a_debug_output *o, | ||
292 | u32 chid, | ||
293 | struct ch_state *ch_state) | ||
294 | { | ||
295 | u32 channel = gk20a_readl(g, ccsr_channel_r(chid)); | ||
296 | u32 status = ccsr_channel_status_v(channel); | ||
297 | u32 *inst_mem; | ||
298 | struct channel_gk20a *c = g->fifo.channel + chid; | ||
299 | struct nvgpu_semaphore_int *hw_sema = NULL; | ||
300 | |||
301 | if (c->hw_sema) | ||
302 | hw_sema = c->hw_sema; | ||
303 | |||
304 | if (!ch_state) | ||
305 | return; | ||
306 | |||
307 | inst_mem = &ch_state->inst_block[0]; | ||
308 | |||
309 | gk20a_debug_output(o, "%d-%s, pid %d, refs: %d: ", chid, | ||
310 | g->name, | ||
311 | ch_state->pid, | ||
312 | ch_state->refs); | ||
313 | gk20a_debug_output(o, "channel status: %s in use %s %s\n", | ||
314 | ccsr_channel_enable_v(channel) ? "" : "not", | ||
315 | gk20a_decode_ccsr_chan_status(status), | ||
316 | ccsr_channel_busy_v(channel) ? "busy" : "not busy"); | ||
317 | gk20a_debug_output(o, "RAMFC : TOP: %016llx PUT: %016llx GET: %016llx " | ||
318 | "FETCH: %016llx\nHEADER: %08x COUNT: %08x\n" | ||
319 | "SEMAPHORE: addr hi: %08x addr lo: %08x\n" | ||
320 | "payload %08x execute %08x\n", | ||
321 | (u64)inst_mem[ram_fc_pb_top_level_get_w()] + | ||
322 | ((u64)inst_mem[ram_fc_pb_top_level_get_hi_w()] << 32ULL), | ||
323 | (u64)inst_mem[ram_fc_pb_put_w()] + | ||
324 | ((u64)inst_mem[ram_fc_pb_put_hi_w()] << 32ULL), | ||
325 | (u64)inst_mem[ram_fc_pb_get_w()] + | ||
326 | ((u64)inst_mem[ram_fc_pb_get_hi_w()] << 32ULL), | ||
327 | (u64)inst_mem[ram_fc_pb_fetch_w()] + | ||
328 | ((u64)inst_mem[ram_fc_pb_fetch_hi_w()] << 32ULL), | ||
329 | inst_mem[ram_fc_pb_header_w()], | ||
330 | inst_mem[ram_fc_pb_count_w()], | ||
331 | inst_mem[ram_fc_sem_addr_hi_w()], | ||
332 | inst_mem[ram_fc_sem_addr_lo_w()], | ||
333 | inst_mem[ram_fc_sem_payload_lo_w()], | ||
334 | inst_mem[ram_fc_sem_execute_w()]); | ||
335 | if (hw_sema) | ||
336 | gk20a_debug_output(o, "SEMA STATE: value: 0x%08x next_val: 0x%08x addr: 0x%010llx\n", | ||
337 | __nvgpu_semaphore_read(hw_sema), | ||
338 | nvgpu_atomic_read(&hw_sema->next_value), | ||
339 | nvgpu_hw_sema_addr(hw_sema)); | ||
340 | gk20a_debug_output(o, "\n"); | ||
341 | } | ||
342 | |||
343 | void gv11b_dump_eng_status(struct gk20a *g, | ||
344 | struct gk20a_debug_output *o) | ||
345 | { | ||
346 | u32 i, host_num_engines; | ||
347 | |||
348 | host_num_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES); | ||
349 | |||
350 | for (i = 0; i < host_num_engines; i++) { | ||
351 | u32 status = gk20a_readl(g, fifo_engine_status_r(i)); | ||
352 | u32 ctx_status = fifo_engine_status_ctx_status_v(status); | ||
353 | |||
354 | gk20a_debug_output(o, "%s eng %d: ", g->name, i); | ||
355 | gk20a_debug_output(o, | ||
356 | "id: %d (%s), next_id: %d (%s), ctx status: %s ", | ||
357 | fifo_engine_status_id_v(status), | ||
358 | fifo_engine_status_id_type_v(status) ? | ||
359 | "tsg" : "channel", | ||
360 | fifo_engine_status_next_id_v(status), | ||
361 | fifo_engine_status_next_id_type_v(status) ? | ||
362 | "tsg" : "channel", | ||
363 | gk20a_decode_pbdma_chan_eng_ctx_status(ctx_status)); | ||
364 | |||
365 | if (fifo_engine_status_eng_reload_v(status)) | ||
366 | gk20a_debug_output(o, "ctx_reload "); | ||
367 | if (fifo_engine_status_faulted_v(status)) | ||
368 | gk20a_debug_output(o, "faulted "); | ||
369 | if (fifo_engine_status_engine_v(status)) | ||
370 | gk20a_debug_output(o, "busy "); | ||
371 | gk20a_debug_output(o, "\n"); | ||
372 | } | ||
373 | gk20a_debug_output(o, "\n"); | ||
374 | } | ||
375 | |||
376 | u32 gv11b_fifo_intr_0_error_mask(struct gk20a *g) | ||
377 | { | ||
378 | u32 intr_0_error_mask = | ||
379 | fifo_intr_0_bind_error_pending_f() | | ||
380 | fifo_intr_0_sched_error_pending_f() | | ||
381 | fifo_intr_0_chsw_error_pending_f() | | ||
382 | fifo_intr_0_fb_flush_timeout_pending_f() | | ||
383 | fifo_intr_0_lb_error_pending_f(); | ||
384 | |||
385 | return intr_0_error_mask; | ||
386 | } | ||
387 | |||
388 | u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g) | ||
389 | { | ||
390 | return gk20a_get_gr_idle_timeout(g); | ||
391 | } | ||
392 | |||
393 | static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id, | ||
394 | u32 pbdma_id, unsigned int timeout_rc_type) | ||
395 | { | ||
396 | struct nvgpu_timeout timeout; | ||
397 | unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */ | ||
398 | u32 pbdma_stat; | ||
399 | u32 chan_stat; | ||
400 | int ret = -EBUSY; | ||
401 | |||
402 | /* timeout in milli seconds */ | ||
403 | nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), | ||
404 | NVGPU_TIMER_CPU_TIMER); | ||
405 | |||
406 | nvgpu_log(g, gpu_dbg_info, "wait preempt pbdma %d", pbdma_id); | ||
407 | /* Verify that ch/tsg is no longer on the pbdma */ | ||
408 | do { | ||
409 | /* | ||
410 | * If the PBDMA has a stalling interrupt and receives a NACK, | ||
411 | * the PBDMA won't save out until the STALLING interrupt is | ||
412 | * cleared. Stalling interrupt need not be directly addressed, | ||
413 | * as simply clearing of the interrupt bit will be sufficient | ||
414 | * to allow the PBDMA to save out. If the stalling interrupt | ||
415 | * was due to a SW method or another deterministic failure, | ||
416 | * the PBDMA will assert it when the channel is reloaded | ||
417 | * or resumed. Note that the fault will still be | ||
418 | * reported to SW. | ||
419 | */ | ||
420 | |||
421 | gk20a_fifo_handle_pbdma_intr(g, &g->fifo, pbdma_id, RC_NO); | ||
422 | |||
423 | pbdma_stat = gk20a_readl(g, fifo_pbdma_status_r(pbdma_id)); | ||
424 | chan_stat = fifo_pbdma_status_chan_status_v(pbdma_stat); | ||
425 | |||
426 | if (chan_stat == | ||
427 | fifo_pbdma_status_chan_status_valid_v() || | ||
428 | chan_stat == | ||
429 | fifo_pbdma_status_chan_status_chsw_save_v()) { | ||
430 | |||
431 | if (id != fifo_pbdma_status_id_v(pbdma_stat)) { | ||
432 | ret = 0; | ||
433 | break; | ||
434 | } | ||
435 | |||
436 | } else if (chan_stat == | ||
437 | fifo_pbdma_status_chan_status_chsw_load_v()) { | ||
438 | |||
439 | if (id != fifo_pbdma_status_next_id_v(pbdma_stat)) { | ||
440 | ret = 0; | ||
441 | break; | ||
442 | } | ||
443 | |||
444 | } else if (chan_stat == | ||
445 | fifo_pbdma_status_chan_status_chsw_switch_v()) { | ||
446 | |||
447 | if ((id != fifo_pbdma_status_next_id_v(pbdma_stat)) && | ||
448 | (id != fifo_pbdma_status_id_v(pbdma_stat))) { | ||
449 | ret = 0; | ||
450 | break; | ||
451 | } | ||
452 | } else { | ||
453 | /* pbdma status is invalid i.e. it is not loaded */ | ||
454 | ret = 0; | ||
455 | break; | ||
456 | } | ||
457 | |||
458 | usleep_range(delay, delay * 2); | ||
459 | delay = min_t(unsigned long, | ||
460 | delay << 1, GR_IDLE_CHECK_MAX); | ||
461 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
462 | "preempt timeout pbdma")); | ||
463 | return ret; | ||
464 | } | ||
465 | |||
466 | static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, | ||
467 | u32 act_eng_id, u32 *reset_eng_bitmask, | ||
468 | unsigned int timeout_rc_type) | ||
469 | { | ||
470 | struct nvgpu_timeout timeout; | ||
471 | unsigned long delay = GR_IDLE_CHECK_DEFAULT; /* in micro seconds */ | ||
472 | u32 eng_stat; | ||
473 | u32 ctx_stat; | ||
474 | int ret = -EBUSY; | ||
475 | bool stall_intr = false; | ||
476 | |||
477 | /* timeout in milli seconds */ | ||
478 | nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), | ||
479 | NVGPU_TIMER_CPU_TIMER); | ||
480 | |||
481 | nvgpu_log(g, gpu_dbg_info, "wait preempt act engine id: %u", | ||
482 | act_eng_id); | ||
483 | /* Check if ch/tsg has saved off the engine or if ctxsw is hung */ | ||
484 | do { | ||
485 | eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); | ||
486 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); | ||
487 | |||
488 | if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) { | ||
489 | stall_intr = true; | ||
490 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, | ||
491 | "stall intr set, " | ||
492 | "preemption will not finish"); | ||
493 | } | ||
494 | if (ctx_stat == | ||
495 | fifo_engine_status_ctx_status_ctxsw_switch_v()) { | ||
496 | /* Eng save hasn't started yet. Continue polling */ | ||
497 | |||
498 | } else if (ctx_stat == | ||
499 | fifo_engine_status_ctx_status_valid_v() || | ||
500 | ctx_stat == | ||
501 | fifo_engine_status_ctx_status_ctxsw_save_v()) { | ||
502 | |||
503 | if (id == fifo_engine_status_id_v(eng_stat)) { | ||
504 | if (stall_intr || | ||
505 | timeout_rc_type == PREEMPT_TIMEOUT_NORC) { | ||
506 | /* preemption will not finish */ | ||
507 | *reset_eng_bitmask |= BIT(act_eng_id); | ||
508 | ret = 0; | ||
509 | break; | ||
510 | } | ||
511 | } else { | ||
512 | /* context is not running on the engine */ | ||
513 | ret = 0; | ||
514 | break; | ||
515 | } | ||
516 | |||
517 | } else if (ctx_stat == | ||
518 | fifo_engine_status_ctx_status_ctxsw_load_v()) { | ||
519 | |||
520 | if (id == fifo_engine_status_next_id_v(eng_stat)) { | ||
521 | |||
522 | if (stall_intr || | ||
523 | timeout_rc_type == PREEMPT_TIMEOUT_NORC) { | ||
524 | /* preemption will not finish */ | ||
525 | *reset_eng_bitmask |= BIT(act_eng_id); | ||
526 | ret = 0; | ||
527 | break; | ||
528 | } | ||
529 | } else { | ||
530 | /* context is not running on the engine */ | ||
531 | ret = 0; | ||
532 | break; | ||
533 | } | ||
534 | |||
535 | } else { | ||
536 | /* Preempt should be finished */ | ||
537 | ret = 0; | ||
538 | break; | ||
539 | } | ||
540 | nvgpu_usleep_range(delay, delay * 2); | ||
541 | delay = min_t(unsigned long, | ||
542 | delay << 1, GR_IDLE_CHECK_MAX); | ||
543 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
544 | "preempt timeout eng")); | ||
545 | return ret; | ||
546 | } | ||
547 | |||
548 | static void gv11b_reset_eng_faulted_ch(struct gk20a *g, u32 chid) | ||
549 | { | ||
550 | u32 reg_val; | ||
551 | |||
552 | reg_val = gk20a_readl(g, ccsr_channel_r(chid)); | ||
553 | reg_val |= ccsr_channel_eng_faulted_reset_f(); | ||
554 | gk20a_writel(g, ccsr_channel_r(chid), reg_val); | ||
555 | } | ||
556 | |||
557 | static void gv11b_reset_eng_faulted_tsg(struct tsg_gk20a *tsg) | ||
558 | { | ||
559 | struct gk20a *g = tsg->g; | ||
560 | struct channel_gk20a *ch; | ||
561 | |||
562 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
563 | list_for_each_entry(ch, &tsg->ch_list, ch_entry) { | ||
564 | gv11b_reset_eng_faulted_ch(g, ch->chid); | ||
565 | } | ||
566 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
567 | } | ||
568 | |||
569 | static void gv11b_reset_pbdma_faulted_ch(struct gk20a *g, u32 chid) | ||
570 | { | ||
571 | u32 reg_val; | ||
572 | |||
573 | reg_val = gk20a_readl(g, ccsr_channel_r(chid)); | ||
574 | reg_val |= ccsr_channel_pbdma_faulted_reset_f(); | ||
575 | gk20a_writel(g, ccsr_channel_r(chid), reg_val); | ||
576 | } | ||
577 | |||
578 | static void gv11b_reset_pbdma_faulted_tsg(struct tsg_gk20a *tsg) | ||
579 | { | ||
580 | struct gk20a *g = tsg->g; | ||
581 | struct channel_gk20a *ch; | ||
582 | |||
583 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
584 | list_for_each_entry(ch, &tsg->ch_list, ch_entry) { | ||
585 | gv11b_reset_pbdma_faulted_ch(g, ch->chid); | ||
586 | } | ||
587 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
588 | } | ||
589 | |||
590 | void gv11b_fifo_reset_pbdma_and_eng_faulted(struct gk20a *g, | ||
591 | struct channel_gk20a *refch, | ||
592 | u32 faulted_pbdma, u32 faulted_engine) | ||
593 | { | ||
594 | struct tsg_gk20a *tsg; | ||
595 | |||
596 | nvgpu_log(g, gpu_dbg_intr, "reset faulted pbdma:0x%x eng:0x%x", | ||
597 | faulted_pbdma, faulted_engine); | ||
598 | |||
599 | if (gk20a_is_channel_marked_as_tsg(refch)) { | ||
600 | tsg = &g->fifo.tsg[refch->tsgid]; | ||
601 | if (faulted_pbdma != FIFO_INVAL_PBDMA_ID) | ||
602 | gv11b_reset_pbdma_faulted_tsg(tsg); | ||
603 | if (faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
604 | gv11b_reset_eng_faulted_tsg(tsg); | ||
605 | } else { | ||
606 | if (faulted_pbdma != FIFO_INVAL_PBDMA_ID) | ||
607 | gv11b_reset_pbdma_faulted_ch(g, refch->chid); | ||
608 | if (faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
609 | gv11b_reset_eng_faulted_ch(g, refch->chid); | ||
610 | } | ||
611 | } | ||
612 | |||
613 | static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, | ||
614 | u32 id, unsigned int id_type, unsigned int rc_type, | ||
615 | struct mmu_fault_info *mmfault) | ||
616 | { | ||
617 | u32 runlists_mask = 0; | ||
618 | struct fifo_gk20a *f = &g->fifo; | ||
619 | struct fifo_runlist_info_gk20a *runlist; | ||
620 | u32 pbdma_bitmask = 0; | ||
621 | |||
622 | if (id_type != ID_TYPE_UNKNOWN) { | ||
623 | if (id_type == ID_TYPE_TSG) | ||
624 | runlists_mask |= fifo_sched_disable_runlist_m( | ||
625 | f->tsg[id].runlist_id); | ||
626 | else | ||
627 | runlists_mask |= fifo_sched_disable_runlist_m( | ||
628 | f->channel[id].runlist_id); | ||
629 | } | ||
630 | |||
631 | if (rc_type == RC_TYPE_MMU_FAULT && mmfault) { | ||
632 | if (mmfault->faulted_pbdma != FIFO_INVAL_PBDMA_ID) | ||
633 | pbdma_bitmask = BIT(mmfault->faulted_pbdma); | ||
634 | |||
635 | for (id = 0; id < f->max_runlists; id++) { | ||
636 | |||
637 | runlist = &f->runlist_info[id]; | ||
638 | |||
639 | if (runlist->eng_bitmask & act_eng_bitmask) | ||
640 | runlists_mask |= | ||
641 | fifo_sched_disable_runlist_m(id); | ||
642 | |||
643 | if (runlist->pbdma_bitmask & pbdma_bitmask) | ||
644 | runlists_mask |= | ||
645 | fifo_sched_disable_runlist_m(id); | ||
646 | } | ||
647 | } | ||
648 | |||
649 | if (id_type == ID_TYPE_UNKNOWN) { | ||
650 | for (id = 0; id < f->max_runlists; id++) { | ||
651 | if (act_eng_bitmask) { | ||
652 | /* eng ids are known */ | ||
653 | runlist = &f->runlist_info[id]; | ||
654 | if (runlist->eng_bitmask & act_eng_bitmask) | ||
655 | runlists_mask |= | ||
656 | fifo_sched_disable_runlist_m(id); | ||
657 | } else { | ||
658 | runlists_mask |= | ||
659 | fifo_sched_disable_runlist_m(id); | ||
660 | } | ||
661 | } | ||
662 | } | ||
663 | gk20a_dbg_info("runlists_mask = %08x", runlists_mask); | ||
664 | return runlists_mask; | ||
665 | } | ||
666 | |||
667 | static void gv11b_fifo_runlist_event_intr_disable(struct gk20a *g) | ||
668 | { | ||
669 | u32 reg_val; | ||
670 | |||
671 | reg_val = gk20a_readl(g, fifo_intr_en_0_r()); | ||
672 | reg_val &= fifo_intr_0_runlist_event_pending_f(); | ||
673 | gk20a_writel(g, fifo_intr_en_0_r(), reg_val); | ||
674 | } | ||
675 | |||
676 | static void gv11b_fifo_runlist_event_intr_enable(struct gk20a *g) | ||
677 | { | ||
678 | u32 reg_val; | ||
679 | |||
680 | reg_val = gk20a_readl(g, fifo_intr_en_0_r()); | ||
681 | reg_val |= fifo_intr_0_runlist_event_pending_f(); | ||
682 | gk20a_writel(g, fifo_intr_en_0_r(), reg_val); | ||
683 | } | ||
684 | |||
685 | static void gv11b_fifo_issue_runlist_preempt(struct gk20a *g, | ||
686 | u32 runlists_mask) | ||
687 | { | ||
688 | u32 reg_val; | ||
689 | |||
690 | /* issue runlist preempt */ | ||
691 | reg_val = gk20a_readl(g, fifo_runlist_preempt_r()); | ||
692 | reg_val |= runlists_mask; | ||
693 | gk20a_writel(g, fifo_runlist_preempt_r(), reg_val); | ||
694 | } | ||
695 | |||
696 | static int gv11b_fifo_poll_runlist_preempt_pending(struct gk20a *g, | ||
697 | u32 runlists_mask) | ||
698 | { | ||
699 | struct nvgpu_timeout timeout; | ||
700 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
701 | int ret = -EBUSY; | ||
702 | |||
703 | nvgpu_timeout_init(g, &timeout, g->ops.fifo.get_preempt_timeout(g), | ||
704 | NVGPU_TIMER_CPU_TIMER); | ||
705 | do { | ||
706 | if (!((gk20a_readl(g, fifo_runlist_preempt_r())) & | ||
707 | runlists_mask)) { | ||
708 | ret = 0; | ||
709 | break; | ||
710 | } | ||
711 | |||
712 | nvgpu_usleep_range(delay, delay * 2); | ||
713 | delay = min_t(unsigned long, | ||
714 | delay << 1, GR_IDLE_CHECK_MAX); | ||
715 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
716 | "runlist preempt timeout")); | ||
717 | return ret; | ||
718 | } | ||
719 | |||
720 | int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, | ||
721 | unsigned int id_type, unsigned int timeout_rc_type) | ||
722 | { | ||
723 | struct fifo_gk20a *f = &g->fifo; | ||
724 | unsigned long runlist_served_pbdmas; | ||
725 | unsigned long runlist_served_engines; | ||
726 | u32 pbdma_id; | ||
727 | u32 act_eng_id; | ||
728 | u32 runlist_id; | ||
729 | int func_ret; | ||
730 | int ret = 0; | ||
731 | u32 tsgid; | ||
732 | |||
733 | if (id_type == ID_TYPE_TSG) { | ||
734 | runlist_id = f->tsg[id].runlist_id; | ||
735 | tsgid = id; | ||
736 | } else { | ||
737 | runlist_id = f->channel[id].runlist_id; | ||
738 | tsgid = f->channel[id].tsgid; | ||
739 | } | ||
740 | |||
741 | nvgpu_log_info(g, "Check preempt pending for tsgid = %u", tsgid); | ||
742 | |||
743 | runlist_served_pbdmas = f->runlist_info[runlist_id].pbdma_bitmask; | ||
744 | runlist_served_engines = f->runlist_info[runlist_id].eng_bitmask; | ||
745 | |||
746 | for_each_set_bit(pbdma_id, &runlist_served_pbdmas, f->num_pbdma) { | ||
747 | |||
748 | func_ret = gv11b_fifo_poll_pbdma_chan_status(g, tsgid, pbdma_id, | ||
749 | timeout_rc_type); | ||
750 | if (func_ret != 0) { | ||
751 | gk20a_dbg_info("preempt timeout pbdma %d", pbdma_id); | ||
752 | ret |= func_ret; | ||
753 | } | ||
754 | } | ||
755 | |||
756 | f->runlist_info[runlist_id].reset_eng_bitmask = 0; | ||
757 | |||
758 | for_each_set_bit(act_eng_id, &runlist_served_engines, f->max_engines) { | ||
759 | |||
760 | func_ret = gv11b_fifo_poll_eng_ctx_status(g, tsgid, act_eng_id, | ||
761 | &f->runlist_info[runlist_id].reset_eng_bitmask, | ||
762 | timeout_rc_type); | ||
763 | |||
764 | if (func_ret != 0) { | ||
765 | gk20a_dbg_info("preempt timeout engine %d", act_eng_id); | ||
766 | ret |= func_ret; | ||
767 | } | ||
768 | } | ||
769 | |||
770 | return ret; | ||
771 | } | ||
772 | |||
773 | int gv11b_fifo_preempt_channel(struct gk20a *g, u32 chid) | ||
774 | { | ||
775 | struct fifo_gk20a *f = &g->fifo; | ||
776 | u32 tsgid; | ||
777 | |||
778 | tsgid = f->channel[chid].tsgid; | ||
779 | nvgpu_log_info(g, "chid:%d tsgid:%d", chid, tsgid); | ||
780 | |||
781 | /* Preempt tsg. Channel preempt is NOOP */ | ||
782 | return g->ops.fifo.preempt_tsg(g, tsgid); | ||
783 | } | ||
784 | |||
785 | static int __locked_fifo_preempt_runlists(struct gk20a *g, u32 runlists_mask) | ||
786 | { | ||
787 | int ret; | ||
788 | |||
789 | /* | ||
790 | * Disable runlist event interrupt as it will get | ||
791 | * triggered after runlist preempt finishes | ||
792 | */ | ||
793 | gv11b_fifo_runlist_event_intr_disable(g); | ||
794 | |||
795 | /* issue runlist preempt */ | ||
796 | gv11b_fifo_issue_runlist_preempt(g, runlists_mask); | ||
797 | |||
798 | /* poll for runlist preempt done */ | ||
799 | ret = gv11b_fifo_poll_runlist_preempt_pending(g, runlists_mask); | ||
800 | |||
801 | /* Clear outstanding runlist event */ | ||
802 | gk20a_fifo_handle_runlist_event(g); | ||
803 | |||
804 | /* Enable runlist event interrupt*/ | ||
805 | gv11b_fifo_runlist_event_intr_enable(g); | ||
806 | |||
807 | return ret; | ||
808 | } | ||
809 | |||
810 | /* TSG enable sequence applicable for Volta and onwards */ | ||
811 | int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg) | ||
812 | { | ||
813 | struct gk20a *g = tsg->g; | ||
814 | struct channel_gk20a *ch; | ||
815 | |||
816 | nvgpu_rwsem_down_read(&tsg->ch_list_lock); | ||
817 | nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { | ||
818 | g->ops.fifo.enable_channel(ch); | ||
819 | } | ||
820 | nvgpu_rwsem_up_read(&tsg->ch_list_lock); | ||
821 | |||
822 | return 0; | ||
823 | } | ||
824 | |||
825 | int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid) | ||
826 | { | ||
827 | struct fifo_gk20a *f = &g->fifo; | ||
828 | u32 ret = 0; | ||
829 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
830 | u32 mutex_ret = 0; | ||
831 | u32 runlist_id; | ||
832 | |||
833 | gk20a_dbg_fn("%d", tsgid); | ||
834 | |||
835 | runlist_id = f->tsg[tsgid].runlist_id; | ||
836 | gk20a_dbg_fn("runlist_id %d", runlist_id); | ||
837 | |||
838 | nvgpu_mutex_acquire(&f->runlist_info[runlist_id].mutex); | ||
839 | |||
840 | mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
841 | |||
842 | ret = __locked_fifo_preempt(g, tsgid, true); | ||
843 | |||
844 | if (!mutex_ret) | ||
845 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
846 | |||
847 | nvgpu_mutex_release(&f->runlist_info[runlist_id].mutex); | ||
848 | |||
849 | return ret; | ||
850 | } | ||
851 | |||
852 | |||
853 | static int gv11b_fifo_preempt_runlists(struct gk20a *g, u32 runlists_mask) | ||
854 | { | ||
855 | int ret = 0; | ||
856 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
857 | u32 mutex_ret = 0; | ||
858 | u32 runlist_id; | ||
859 | |||
860 | gk20a_dbg_fn(""); | ||
861 | |||
862 | for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { | ||
863 | if (runlists_mask & fifo_runlist_preempt_runlist_m(runlist_id)) | ||
864 | nvgpu_mutex_acquire(&g->fifo. | ||
865 | runlist_info[runlist_id].mutex); | ||
866 | } | ||
867 | |||
868 | mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
869 | |||
870 | ret = __locked_fifo_preempt_runlists(g, runlists_mask); | ||
871 | |||
872 | if (!mutex_ret) | ||
873 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
874 | |||
875 | for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { | ||
876 | if (runlists_mask & fifo_runlist_preempt_runlist_m(runlist_id)) | ||
877 | nvgpu_mutex_release(&g->fifo. | ||
878 | runlist_info[runlist_id].mutex); | ||
879 | } | ||
880 | |||
881 | return ret; | ||
882 | } | ||
883 | |||
884 | static int __locked_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, | ||
885 | unsigned int id_type, unsigned int timeout_rc_type) | ||
886 | { | ||
887 | int ret; | ||
888 | struct fifo_gk20a *f = &g->fifo; | ||
889 | |||
890 | nvgpu_log_fn(g, "id:%d id_type:%d", id, id_type); | ||
891 | |||
892 | /* Issue tsg preempt. Channel preempt is noop */ | ||
893 | if (id_type == ID_TYPE_CHANNEL) | ||
894 | gk20a_fifo_issue_preempt(g, f->channel[id].tsgid, true); | ||
895 | else | ||
896 | gk20a_fifo_issue_preempt(g, id, true); | ||
897 | |||
898 | /* wait for preempt */ | ||
899 | ret = g->ops.fifo.is_preempt_pending(g, id, id_type, | ||
900 | timeout_rc_type); | ||
901 | |||
902 | if (ret && (timeout_rc_type == PREEMPT_TIMEOUT_RC)) | ||
903 | __locked_fifo_preempt_timeout_rc(g, id, id_type); | ||
904 | |||
905 | return ret; | ||
906 | } | ||
907 | |||
908 | |||
909 | int gv11b_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, | ||
910 | unsigned int id_type, unsigned int timeout_rc_type) | ||
911 | { | ||
912 | struct fifo_gk20a *f = &g->fifo; | ||
913 | u32 ret = 0; | ||
914 | u32 token = PMU_INVALID_MUTEX_OWNER_ID; | ||
915 | u32 mutex_ret = 0; | ||
916 | u32 runlist_id; | ||
917 | |||
918 | if (id_type == ID_TYPE_TSG) | ||
919 | runlist_id = f->tsg[id].runlist_id; | ||
920 | else if (id_type == ID_TYPE_CHANNEL) | ||
921 | runlist_id = f->channel[id].runlist_id; | ||
922 | else | ||
923 | return -EINVAL; | ||
924 | |||
925 | if (runlist_id >= g->fifo.max_runlists) { | ||
926 | gk20a_dbg_info("runlist_id = %d", runlist_id); | ||
927 | return -EINVAL; | ||
928 | } | ||
929 | |||
930 | gk20a_dbg_fn("preempt id = %d, runlist_id = %d", id, runlist_id); | ||
931 | |||
932 | nvgpu_mutex_acquire(&f->runlist_info[runlist_id].mutex); | ||
933 | |||
934 | mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
935 | |||
936 | ret = __locked_fifo_preempt_ch_tsg(g, id, id_type, timeout_rc_type); | ||
937 | |||
938 | if (!mutex_ret) | ||
939 | nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); | ||
940 | |||
941 | nvgpu_mutex_release(&f->runlist_info[runlist_id].mutex); | ||
942 | |||
943 | return ret; | ||
944 | |||
945 | } | ||
946 | |||
947 | void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, | ||
948 | u32 id, unsigned int id_type, unsigned int rc_type, | ||
949 | struct mmu_fault_info *mmfault) | ||
950 | { | ||
951 | bool verbose = false; | ||
952 | struct tsg_gk20a *tsg = NULL; | ||
953 | struct channel_gk20a *refch = NULL; | ||
954 | u32 runlists_mask, runlist_id; | ||
955 | struct fifo_runlist_info_gk20a *runlist = NULL; | ||
956 | u32 engine_id, client_type = ~0; | ||
957 | |||
958 | gk20a_dbg_info("active engine ids bitmask =0x%x", act_eng_bitmask); | ||
959 | gk20a_dbg_info("hw id =%d", id); | ||
960 | gk20a_dbg_info("id_type =%d", id_type); | ||
961 | gk20a_dbg_info("rc_type =%d", rc_type); | ||
962 | gk20a_dbg_info("mmu_fault =0x%p", mmfault); | ||
963 | |||
964 | runlists_mask = gv11b_fifo_get_runlists_mask(g, act_eng_bitmask, id, | ||
965 | id_type, rc_type, mmfault); | ||
966 | |||
967 | gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_DISABLED, | ||
968 | !RUNLIST_INFO_MUTEX_LOCKED); | ||
969 | |||
970 | g->fifo.deferred_reset_pending = false; | ||
971 | |||
972 | /* Disable power management */ | ||
973 | if (g->support_pmu && g->elpg_enabled) { | ||
974 | if (nvgpu_pmu_disable_elpg(g)) | ||
975 | nvgpu_err(g, "failed to set disable elpg"); | ||
976 | } | ||
977 | if (g->ops.clock_gating.slcg_gr_load_gating_prod) | ||
978 | g->ops.clock_gating.slcg_gr_load_gating_prod(g, | ||
979 | false); | ||
980 | if (g->ops.clock_gating.slcg_perf_load_gating_prod) | ||
981 | g->ops.clock_gating.slcg_perf_load_gating_prod(g, | ||
982 | false); | ||
983 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) | ||
984 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, | ||
985 | false); | ||
986 | |||
987 | gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); | ||
988 | |||
989 | if (rc_type == RC_TYPE_MMU_FAULT) | ||
990 | gk20a_debug_dump(g); | ||
991 | |||
992 | /* get the channel/TSG */ | ||
993 | if (rc_type == RC_TYPE_MMU_FAULT && mmfault && mmfault->refch) { | ||
994 | refch = mmfault->refch; | ||
995 | client_type = mmfault->client_type; | ||
996 | if (gk20a_is_channel_marked_as_tsg(refch)) | ||
997 | tsg = &g->fifo.tsg[refch->tsgid]; | ||
998 | gv11b_fifo_reset_pbdma_and_eng_faulted(g, refch, | ||
999 | mmfault->faulted_pbdma, | ||
1000 | mmfault->faulted_engine); | ||
1001 | } else { | ||
1002 | if (id_type == ID_TYPE_TSG) | ||
1003 | tsg = &g->fifo.tsg[id]; | ||
1004 | else if (id_type == ID_TYPE_CHANNEL) | ||
1005 | refch = gk20a_channel_get(&g->fifo.channel[id]); | ||
1006 | } | ||
1007 | |||
1008 | if (id_type == ID_TYPE_TSG || id_type == ID_TYPE_CHANNEL) { | ||
1009 | g->ops.fifo.preempt_ch_tsg(g, id, id_type, | ||
1010 | PREEMPT_TIMEOUT_NORC); | ||
1011 | } else { | ||
1012 | gv11b_fifo_preempt_runlists(g, runlists_mask); | ||
1013 | } | ||
1014 | |||
1015 | if (tsg) { | ||
1016 | if (!g->fifo.deferred_reset_pending) { | ||
1017 | if (rc_type == RC_TYPE_MMU_FAULT) { | ||
1018 | gk20a_fifo_set_ctx_mmu_error_tsg(g, tsg); | ||
1019 | verbose = gk20a_fifo_error_tsg(g, tsg); | ||
1020 | } | ||
1021 | } | ||
1022 | gk20a_fifo_abort_tsg(g, tsg->tsgid, false); | ||
1023 | if (refch) | ||
1024 | gk20a_channel_put(refch); | ||
1025 | } else if (refch) { | ||
1026 | if (!g->fifo.deferred_reset_pending) { | ||
1027 | if (rc_type == RC_TYPE_MMU_FAULT) { | ||
1028 | gk20a_fifo_set_ctx_mmu_error_ch(g, refch); | ||
1029 | verbose = gk20a_fifo_error_ch(g, refch); | ||
1030 | } | ||
1031 | } | ||
1032 | gk20a_channel_abort(refch, false); | ||
1033 | gk20a_channel_put(refch); | ||
1034 | } else { | ||
1035 | nvgpu_err(g, "id unknown, abort runlist"); | ||
1036 | for (runlist_id = 0; runlist_id < g->fifo.max_runlists; | ||
1037 | runlist_id++) { | ||
1038 | if (runlists_mask & BIT(runlist_id)) | ||
1039 | g->ops.fifo.update_runlist(g, runlist_id, | ||
1040 | FIFO_INVAL_CHANNEL_ID, false, true); | ||
1041 | } | ||
1042 | } | ||
1043 | |||
1044 | /* check if engine reset should be deferred */ | ||
1045 | for (runlist_id = 0; runlist_id < g->fifo.max_runlists; runlist_id++) { | ||
1046 | |||
1047 | runlist = &g->fifo.runlist_info[runlist_id]; | ||
1048 | if ((runlists_mask & BIT(runlist_id)) && | ||
1049 | runlist->reset_eng_bitmask) { | ||
1050 | |||
1051 | unsigned long __reset_eng_bitmask = | ||
1052 | runlist->reset_eng_bitmask; | ||
1053 | |||
1054 | for_each_set_bit(engine_id, &__reset_eng_bitmask, 32) { | ||
1055 | if ((refch || tsg) && | ||
1056 | gk20a_fifo_should_defer_engine_reset(g, | ||
1057 | engine_id, client_type, false)) { | ||
1058 | |||
1059 | g->fifo.deferred_fault_engines |= | ||
1060 | BIT(engine_id); | ||
1061 | |||
1062 | /* handled during channel free */ | ||
1063 | g->fifo.deferred_reset_pending = true; | ||
1064 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
1065 | "sm debugger attached," | ||
1066 | " deferring channel recovery to channel free"); | ||
1067 | } else { | ||
1068 | /* | ||
1069 | * if lock is already taken, a reset is | ||
1070 | * taking place so no need to repeat | ||
1071 | */ | ||
1072 | if (nvgpu_mutex_tryacquire( | ||
1073 | &g->fifo.gr_reset_mutex)) { | ||
1074 | |||
1075 | gk20a_fifo_reset_engine(g, | ||
1076 | engine_id); | ||
1077 | |||
1078 | nvgpu_mutex_release( | ||
1079 | &g->fifo.gr_reset_mutex); | ||
1080 | } | ||
1081 | } | ||
1082 | } | ||
1083 | } | ||
1084 | } | ||
1085 | |||
1086 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
1087 | if (refch) | ||
1088 | gk20a_ctxsw_trace_channel_reset(g, refch); | ||
1089 | else if (tsg) | ||
1090 | gk20a_ctxsw_trace_tsg_reset(g, tsg); | ||
1091 | #endif | ||
1092 | |||
1093 | gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_ENABLED, | ||
1094 | !RUNLIST_INFO_MUTEX_LOCKED); | ||
1095 | |||
1096 | /* It is safe to enable ELPG again. */ | ||
1097 | if (g->support_pmu && g->elpg_enabled) | ||
1098 | nvgpu_pmu_enable_elpg(g); | ||
1099 | } | ||
1100 | |||
1101 | void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f) | ||
1102 | { | ||
1103 | /* | ||
1104 | * These are all errors which indicate something really wrong | ||
1105 | * going on in the device | ||
1106 | */ | ||
1107 | f->intr.pbdma.device_fatal_0 = | ||
1108 | pbdma_intr_0_memreq_pending_f() | | ||
1109 | pbdma_intr_0_memack_timeout_pending_f() | | ||
1110 | pbdma_intr_0_memack_extra_pending_f() | | ||
1111 | pbdma_intr_0_memdat_timeout_pending_f() | | ||
1112 | pbdma_intr_0_memdat_extra_pending_f() | | ||
1113 | pbdma_intr_0_memflush_pending_f() | | ||
1114 | pbdma_intr_0_memop_pending_f() | | ||
1115 | pbdma_intr_0_lbconnect_pending_f() | | ||
1116 | pbdma_intr_0_lback_timeout_pending_f() | | ||
1117 | pbdma_intr_0_lback_extra_pending_f() | | ||
1118 | pbdma_intr_0_lbdat_timeout_pending_f() | | ||
1119 | pbdma_intr_0_lbdat_extra_pending_f() | | ||
1120 | pbdma_intr_0_pri_pending_f(); | ||
1121 | |||
1122 | /* | ||
1123 | * These are data parsing, framing errors or others which can be | ||
1124 | * recovered from with intervention... or just resetting the | ||
1125 | * channel | ||
1126 | */ | ||
1127 | f->intr.pbdma.channel_fatal_0 = | ||
1128 | pbdma_intr_0_gpfifo_pending_f() | | ||
1129 | pbdma_intr_0_gpptr_pending_f() | | ||
1130 | pbdma_intr_0_gpentry_pending_f() | | ||
1131 | pbdma_intr_0_gpcrc_pending_f() | | ||
1132 | pbdma_intr_0_pbptr_pending_f() | | ||
1133 | pbdma_intr_0_pbentry_pending_f() | | ||
1134 | pbdma_intr_0_pbcrc_pending_f() | | ||
1135 | pbdma_intr_0_method_pending_f() | | ||
1136 | pbdma_intr_0_methodcrc_pending_f() | | ||
1137 | pbdma_intr_0_pbseg_pending_f() | | ||
1138 | pbdma_intr_0_clear_faulted_error_pending_f() | | ||
1139 | pbdma_intr_0_eng_reset_pending_f() | | ||
1140 | pbdma_intr_0_semaphore_pending_f() | | ||
1141 | pbdma_intr_0_signature_pending_f(); | ||
1142 | |||
1143 | /* Can be used for sw-methods, or represents a recoverable timeout. */ | ||
1144 | f->intr.pbdma.restartable_0 = | ||
1145 | pbdma_intr_0_device_pending_f(); | ||
1146 | } | ||
1147 | |||
1148 | static u32 gv11b_fifo_intr_0_en_mask(struct gk20a *g) | ||
1149 | { | ||
1150 | u32 intr_0_en_mask; | ||
1151 | |||
1152 | intr_0_en_mask = g->ops.fifo.intr_0_error_mask(g); | ||
1153 | |||
1154 | intr_0_en_mask |= fifo_intr_0_runlist_event_pending_f() | | ||
1155 | fifo_intr_0_pbdma_intr_pending_f() | | ||
1156 | fifo_intr_0_ctxsw_timeout_pending_f(); | ||
1157 | |||
1158 | return intr_0_en_mask; | ||
1159 | } | ||
1160 | |||
1161 | int gv11b_init_fifo_reset_enable_hw(struct gk20a *g) | ||
1162 | { | ||
1163 | u32 intr_stall; | ||
1164 | u32 mask; | ||
1165 | u32 timeout; | ||
1166 | unsigned int i; | ||
1167 | u32 host_num_pbdma = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA); | ||
1168 | |||
1169 | gk20a_dbg_fn(""); | ||
1170 | |||
1171 | /* enable pmc pfifo */ | ||
1172 | g->ops.mc.reset(g, mc_enable_pfifo_enabled_f()); | ||
1173 | |||
1174 | if (g->ops.clock_gating.slcg_ce2_load_gating_prod) | ||
1175 | g->ops.clock_gating.slcg_ce2_load_gating_prod(g, | ||
1176 | g->slcg_enabled); | ||
1177 | if (g->ops.clock_gating.slcg_fifo_load_gating_prod) | ||
1178 | g->ops.clock_gating.slcg_fifo_load_gating_prod(g, | ||
1179 | g->slcg_enabled); | ||
1180 | if (g->ops.clock_gating.blcg_fifo_load_gating_prod) | ||
1181 | g->ops.clock_gating.blcg_fifo_load_gating_prod(g, | ||
1182 | g->blcg_enabled); | ||
1183 | |||
1184 | /* enable pbdma */ | ||
1185 | mask = 0; | ||
1186 | for (i = 0; i < host_num_pbdma; ++i) | ||
1187 | mask |= mc_enable_pb_sel_f(mc_enable_pb_0_enabled_v(), i); | ||
1188 | gk20a_writel(g, mc_enable_pb_r(), mask); | ||
1189 | |||
1190 | |||
1191 | timeout = gk20a_readl(g, fifo_fb_timeout_r()); | ||
1192 | nvgpu_log_info(g, "fifo_fb_timeout reg val = 0x%08x", timeout); | ||
1193 | if (!nvgpu_platform_is_silicon(g)) { | ||
1194 | timeout = set_field(timeout, fifo_fb_timeout_period_m(), | ||
1195 | fifo_fb_timeout_period_max_f()); | ||
1196 | timeout = set_field(timeout, fifo_fb_timeout_detection_m(), | ||
1197 | fifo_fb_timeout_detection_disabled_f()); | ||
1198 | nvgpu_log_info(g, "new fifo_fb_timeout reg val = 0x%08x", | ||
1199 | timeout); | ||
1200 | gk20a_writel(g, fifo_fb_timeout_r(), timeout); | ||
1201 | } | ||
1202 | |||
1203 | for (i = 0; i < host_num_pbdma; i++) { | ||
1204 | timeout = gk20a_readl(g, pbdma_timeout_r(i)); | ||
1205 | nvgpu_log_info(g, "pbdma_timeout reg val = 0x%08x", | ||
1206 | timeout); | ||
1207 | if (!nvgpu_platform_is_silicon(g)) { | ||
1208 | timeout = set_field(timeout, pbdma_timeout_period_m(), | ||
1209 | pbdma_timeout_period_max_f()); | ||
1210 | nvgpu_log_info(g, "new pbdma_timeout reg val = 0x%08x", | ||
1211 | timeout); | ||
1212 | gk20a_writel(g, pbdma_timeout_r(i), timeout); | ||
1213 | } | ||
1214 | } | ||
1215 | |||
1216 | /* clear and enable pbdma interrupt */ | ||
1217 | for (i = 0; i < host_num_pbdma; i++) { | ||
1218 | gk20a_writel(g, pbdma_intr_0_r(i), 0xFFFFFFFF); | ||
1219 | gk20a_writel(g, pbdma_intr_1_r(i), 0xFFFFFFFF); | ||
1220 | |||
1221 | intr_stall = gk20a_readl(g, pbdma_intr_stall_r(i)); | ||
1222 | gk20a_dbg_info("pbdma id:%u, intr_en_0 0x%08x", i, intr_stall); | ||
1223 | gk20a_writel(g, pbdma_intr_en_0_r(i), intr_stall); | ||
1224 | |||
1225 | intr_stall = gk20a_readl(g, pbdma_intr_stall_1_r(i)); | ||
1226 | gk20a_dbg_info("pbdma id:%u, intr_en_1 0x%08x", i, intr_stall); | ||
1227 | gk20a_writel(g, pbdma_intr_en_1_r(i), intr_stall); | ||
1228 | } | ||
1229 | |||
1230 | /* clear ctxsw timeout interrupts */ | ||
1231 | gk20a_writel(g, fifo_intr_ctxsw_timeout_r(), ~0); | ||
1232 | |||
1233 | if (nvgpu_platform_is_silicon(g)) { | ||
1234 | /* enable ctxsw timeout */ | ||
1235 | timeout = GRFIFO_TIMEOUT_CHECK_PERIOD_US; | ||
1236 | timeout = scale_ptimer(timeout, | ||
1237 | ptimer_scalingfactor10x(g->ptimer_src_freq)); | ||
1238 | timeout |= fifo_eng_ctxsw_timeout_detection_enabled_f(); | ||
1239 | gk20a_writel(g, fifo_eng_ctxsw_timeout_r(), timeout); | ||
1240 | } else { | ||
1241 | timeout = gk20a_readl(g, fifo_eng_ctxsw_timeout_r()); | ||
1242 | nvgpu_log_info(g, "fifo_eng_ctxsw_timeout reg val = 0x%08x", | ||
1243 | timeout); | ||
1244 | timeout = set_field(timeout, fifo_eng_ctxsw_timeout_period_m(), | ||
1245 | fifo_eng_ctxsw_timeout_period_max_f()); | ||
1246 | timeout = set_field(timeout, | ||
1247 | fifo_eng_ctxsw_timeout_detection_m(), | ||
1248 | fifo_eng_ctxsw_timeout_detection_disabled_f()); | ||
1249 | nvgpu_log_info(g, "new fifo_eng_ctxsw_timeout reg val = 0x%08x", | ||
1250 | timeout); | ||
1251 | gk20a_writel(g, fifo_eng_ctxsw_timeout_r(), timeout); | ||
1252 | } | ||
1253 | |||
1254 | /* clear runlist interrupts */ | ||
1255 | gk20a_writel(g, fifo_intr_runlist_r(), ~0); | ||
1256 | |||
1257 | /* clear and enable pfifo interrupt */ | ||
1258 | gk20a_writel(g, fifo_intr_0_r(), 0xFFFFFFFF); | ||
1259 | mask = gv11b_fifo_intr_0_en_mask(g); | ||
1260 | gk20a_dbg_info("fifo_intr_en_0 0x%08x", mask); | ||
1261 | gk20a_writel(g, fifo_intr_en_0_r(), mask); | ||
1262 | gk20a_dbg_info("fifo_intr_en_1 = 0x80000000"); | ||
1263 | gk20a_writel(g, fifo_intr_en_1_r(), 0x80000000); | ||
1264 | |||
1265 | gk20a_dbg_fn("done"); | ||
1266 | |||
1267 | return 0; | ||
1268 | } | ||
1269 | |||
1270 | static const char *const gv11b_sched_error_str[] = { | ||
1271 | "xxx-0", | ||
1272 | "xxx-1", | ||
1273 | "xxx-2", | ||
1274 | "xxx-3", | ||
1275 | "xxx-4", | ||
1276 | "engine_reset", | ||
1277 | "rl_ack_timeout", | ||
1278 | "rl_ack_extra", | ||
1279 | "rl_rdat_timeout", | ||
1280 | "rl_rdat_extra", | ||
1281 | "xxx-a", | ||
1282 | "xxx-b", | ||
1283 | "rl_req_timeout", | ||
1284 | "new_runlist", | ||
1285 | "code_config_while_busy", | ||
1286 | "xxx-f", | ||
1287 | "xxx-0x10", | ||
1288 | "xxx-0x11", | ||
1289 | "xxx-0x12", | ||
1290 | "xxx-0x13", | ||
1291 | "xxx-0x14", | ||
1292 | "xxx-0x15", | ||
1293 | "xxx-0x16", | ||
1294 | "xxx-0x17", | ||
1295 | "xxx-0x18", | ||
1296 | "xxx-0x19", | ||
1297 | "xxx-0x1a", | ||
1298 | "xxx-0x1b", | ||
1299 | "xxx-0x1c", | ||
1300 | "xxx-0x1d", | ||
1301 | "xxx-0x1e", | ||
1302 | "xxx-0x1f", | ||
1303 | "bad_tsg", | ||
1304 | }; | ||
1305 | |||
1306 | bool gv11b_fifo_handle_sched_error(struct gk20a *g) | ||
1307 | { | ||
1308 | u32 sched_error; | ||
1309 | |||
1310 | sched_error = gk20a_readl(g, fifo_intr_sched_error_r()); | ||
1311 | |||
1312 | if (sched_error < ARRAY_SIZE(gv11b_sched_error_str)) | ||
1313 | nvgpu_err(g, "fifo sched error :%s", | ||
1314 | gv11b_sched_error_str[sched_error]); | ||
1315 | else | ||
1316 | nvgpu_err(g, "fifo sched error code not supported"); | ||
1317 | |||
1318 | if (sched_error == SCHED_ERROR_CODE_BAD_TSG ) { | ||
1319 | /* id is unknown, preempt all runlists and do recovery */ | ||
1320 | gk20a_fifo_recover(g, 0, 0, false, false, false); | ||
1321 | } | ||
1322 | |||
1323 | return false; | ||
1324 | } | ||
1325 | |||
1326 | static u32 gv11b_fifo_ctxsw_timeout_info(struct gk20a *g, u32 active_eng_id) | ||
1327 | { | ||
1328 | u32 tsgid = FIFO_INVAL_TSG_ID; | ||
1329 | u32 timeout_info; | ||
1330 | u32 ctx_status, info_status; | ||
1331 | |||
1332 | timeout_info = gk20a_readl(g, | ||
1333 | fifo_intr_ctxsw_timeout_info_r(active_eng_id)); | ||
1334 | |||
1335 | /* | ||
1336 | * ctxsw_state and tsgid are snapped at the point of the timeout and | ||
1337 | * will not change while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit | ||
1338 | * is PENDING. | ||
1339 | */ | ||
1340 | ctx_status = fifo_intr_ctxsw_timeout_info_ctxsw_state_v(timeout_info); | ||
1341 | if (ctx_status == | ||
1342 | fifo_intr_ctxsw_timeout_info_ctxsw_state_load_v()) { | ||
1343 | |||
1344 | tsgid = fifo_intr_ctxsw_timeout_info_next_tsgid_v(timeout_info); | ||
1345 | |||
1346 | } else if (ctx_status == | ||
1347 | fifo_intr_ctxsw_timeout_info_ctxsw_state_switch_v() || | ||
1348 | ctx_status == | ||
1349 | fifo_intr_ctxsw_timeout_info_ctxsw_state_save_v()) { | ||
1350 | |||
1351 | tsgid = fifo_intr_ctxsw_timeout_info_prev_tsgid_v(timeout_info); | ||
1352 | } | ||
1353 | gk20a_dbg_info("ctxsw timeout info: tsgid = %d", tsgid); | ||
1354 | |||
1355 | /* | ||
1356 | * STATUS indicates whether the context request ack was eventually | ||
1357 | * received and whether a subsequent request timed out. This field is | ||
1358 | * updated live while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit | ||
1359 | * is PENDING. STATUS starts in AWAITING_ACK, and progresses to | ||
1360 | * ACK_RECEIVED and finally ends with DROPPED_TIMEOUT. | ||
1361 | * | ||
1362 | * AWAITING_ACK - context request ack still not returned from engine. | ||
1363 | * ENG_WAS_RESET - The engine was reset via a PRI write to NV_PMC_ENABLE | ||
1364 | * or NV_PMC_ELPG_ENABLE prior to receiving the ack. Host will not | ||
1365 | * expect ctx ack to return, but if it is already in flight, STATUS will | ||
1366 | * transition shortly to ACK_RECEIVED unless the interrupt is cleared | ||
1367 | * first. Once the engine is reset, additional context switches can | ||
1368 | * occur; if one times out, STATUS will transition to DROPPED_TIMEOUT | ||
1369 | * if the interrupt isn't cleared first. | ||
1370 | * ACK_RECEIVED - The ack for the timed-out context request was | ||
1371 | * received between the point of the timeout and this register being | ||
1372 | * read. Note this STATUS can be reported during the load stage of the | ||
1373 | * same context switch that timed out if the timeout occurred during the | ||
1374 | * save half of a context switch. Additional context requests may have | ||
1375 | * completed or may be outstanding, but no further context timeout has | ||
1376 | * occurred. This simplifies checking for spurious context switch | ||
1377 | * timeouts. | ||
1378 | * DROPPED_TIMEOUT - The originally timed-out context request acked, | ||
1379 | * but a subsequent context request then timed out. | ||
1380 | * Information about the subsequent timeout is not stored; in fact, that | ||
1381 | * context request may also have already been acked by the time SW | ||
1382 | * SW reads this register. If not, there is a chance SW can get the | ||
1383 | * dropped information by clearing the corresponding | ||
1384 | * INTR_CTXSW_TIMEOUT_ENGINE bit and waiting for the timeout to occur | ||
1385 | * again. Note, however, that if the engine does time out again, | ||
1386 | * it may not be from the original request that caused the | ||
1387 | * DROPPED_TIMEOUT state, as that request may | ||
1388 | * be acked in the interim. | ||
1389 | */ | ||
1390 | info_status = fifo_intr_ctxsw_timeout_info_status_v(timeout_info); | ||
1391 | if (info_status == | ||
1392 | fifo_intr_ctxsw_timeout_info_status_awaiting_ack_v()) { | ||
1393 | |||
1394 | gk20a_dbg_info("ctxsw timeout info : awaiting ack"); | ||
1395 | |||
1396 | } else if (info_status == | ||
1397 | fifo_intr_ctxsw_timeout_info_status_eng_was_reset_v()) { | ||
1398 | |||
1399 | gk20a_dbg_info("ctxsw timeout info : eng was reset"); | ||
1400 | |||
1401 | } else if (info_status == | ||
1402 | fifo_intr_ctxsw_timeout_info_status_ack_received_v()) { | ||
1403 | |||
1404 | gk20a_dbg_info("ctxsw timeout info : ack received"); | ||
1405 | /* no need to recover */ | ||
1406 | tsgid = FIFO_INVAL_TSG_ID; | ||
1407 | |||
1408 | } else if (info_status == | ||
1409 | fifo_intr_ctxsw_timeout_info_status_dropped_timeout_v()) { | ||
1410 | |||
1411 | gk20a_dbg_info("ctxsw timeout info : dropped timeout"); | ||
1412 | /* no need to recover */ | ||
1413 | tsgid = FIFO_INVAL_TSG_ID; | ||
1414 | |||
1415 | } else { | ||
1416 | gk20a_dbg_info("ctxsw timeout info status = %u", info_status); | ||
1417 | } | ||
1418 | |||
1419 | return tsgid; | ||
1420 | } | ||
1421 | |||
1422 | bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr) | ||
1423 | { | ||
1424 | bool ret = false; | ||
1425 | u32 tsgid = FIFO_INVAL_TSG_ID; | ||
1426 | u32 engine_id, active_eng_id; | ||
1427 | u32 timeout_val, ctxsw_timeout_engines; | ||
1428 | |||
1429 | |||
1430 | if (!(fifo_intr & fifo_intr_0_ctxsw_timeout_pending_f())) | ||
1431 | return ret; | ||
1432 | |||
1433 | /* get ctxsw timedout engines */ | ||
1434 | ctxsw_timeout_engines = gk20a_readl(g, fifo_intr_ctxsw_timeout_r()); | ||
1435 | if (ctxsw_timeout_engines == 0) { | ||
1436 | nvgpu_err(g, "no eng ctxsw timeout pending"); | ||
1437 | return ret; | ||
1438 | } | ||
1439 | |||
1440 | timeout_val = gk20a_readl(g, fifo_eng_ctxsw_timeout_r()); | ||
1441 | timeout_val = fifo_eng_ctxsw_timeout_period_v(timeout_val); | ||
1442 | |||
1443 | gk20a_dbg_info("eng ctxsw timeout period = 0x%x", timeout_val); | ||
1444 | |||
1445 | for (engine_id = 0; engine_id < g->fifo.num_engines; engine_id++) { | ||
1446 | active_eng_id = g->fifo.active_engines_list[engine_id]; | ||
1447 | |||
1448 | if (ctxsw_timeout_engines & | ||
1449 | fifo_intr_ctxsw_timeout_engine_pending_f( | ||
1450 | active_eng_id)) { | ||
1451 | |||
1452 | struct fifo_gk20a *f = &g->fifo; | ||
1453 | u32 ms = 0; | ||
1454 | bool verbose = false; | ||
1455 | |||
1456 | tsgid = gv11b_fifo_ctxsw_timeout_info(g, active_eng_id); | ||
1457 | |||
1458 | if (tsgid == FIFO_INVAL_TSG_ID) | ||
1459 | continue; | ||
1460 | |||
1461 | if (gk20a_fifo_check_tsg_ctxsw_timeout( | ||
1462 | &f->tsg[tsgid], &verbose, &ms)) { | ||
1463 | ret = true; | ||
1464 | nvgpu_err(g, | ||
1465 | "ctxsw timeout error:" | ||
1466 | "active engine id =%u, %s=%d, ms=%u", | ||
1467 | active_eng_id, "tsg", tsgid, ms); | ||
1468 | |||
1469 | /* Cancel all channels' timeout */ | ||
1470 | gk20a_channel_timeout_restart_all_channels(g); | ||
1471 | gk20a_fifo_recover(g, BIT(active_eng_id), tsgid, | ||
1472 | true, true, verbose); | ||
1473 | } else { | ||
1474 | gk20a_dbg_info( | ||
1475 | "fifo is waiting for ctx switch: " | ||
1476 | "for %d ms, %s=%d", ms, "tsg", tsgid); | ||
1477 | } | ||
1478 | } | ||
1479 | } | ||
1480 | /* clear interrupt */ | ||
1481 | gk20a_writel(g, fifo_intr_ctxsw_timeout_r(), ctxsw_timeout_engines); | ||
1482 | return ret; | ||
1483 | } | ||
1484 | |||
1485 | unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g, | ||
1486 | u32 pbdma_id, u32 pbdma_intr_0, | ||
1487 | u32 *handled, u32 *error_notifier) | ||
1488 | { | ||
1489 | unsigned int rc_type = RC_TYPE_NO_RC; | ||
1490 | |||
1491 | rc_type = gk20a_fifo_handle_pbdma_intr_0(g, pbdma_id, | ||
1492 | pbdma_intr_0, handled, error_notifier); | ||
1493 | |||
1494 | if (pbdma_intr_0 & pbdma_intr_0_clear_faulted_error_pending_f()) { | ||
1495 | gk20a_dbg(gpu_dbg_intr, "clear faulted error on pbdma id %d", | ||
1496 | pbdma_id); | ||
1497 | gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0); | ||
1498 | *handled |= pbdma_intr_0_clear_faulted_error_pending_f(); | ||
1499 | rc_type = RC_TYPE_PBDMA_FAULT; | ||
1500 | } | ||
1501 | |||
1502 | if (pbdma_intr_0 & pbdma_intr_0_eng_reset_pending_f()) { | ||
1503 | gk20a_dbg(gpu_dbg_intr, "eng reset intr on pbdma id %d", | ||
1504 | pbdma_id); | ||
1505 | *handled |= pbdma_intr_0_eng_reset_pending_f(); | ||
1506 | rc_type = RC_TYPE_PBDMA_FAULT; | ||
1507 | } | ||
1508 | |||
1509 | return rc_type; | ||
1510 | } | ||
1511 | |||
1512 | /* | ||
1513 | * Pbdma which encountered the ctxnotvalid interrupt will stall and | ||
1514 | * prevent the channel which was loaded at the time the interrupt fired | ||
1515 | * from being swapped out until the interrupt is cleared. | ||
1516 | * CTXNOTVALID pbdma interrupt indicates error conditions related | ||
1517 | * to the *_CTX_VALID fields for a channel. The following | ||
1518 | * conditions trigger the interrupt: | ||
1519 | * * CTX_VALID bit for the targeted engine is FALSE | ||
1520 | * * At channel start/resume, all preemptible eng have CTX_VALID FALSE but: | ||
1521 | * - CTX_RELOAD is set in CCSR_CHANNEL_STATUS, | ||
1522 | * - PBDMA_TARGET_SHOULD_SEND_HOST_TSG_EVENT is TRUE, or | ||
1523 | * - PBDMA_TARGET_NEEDS_HOST_TSG_EVENT is TRUE | ||
1524 | * The field is left NOT_PENDING and the interrupt is not raised if the PBDMA is | ||
1525 | * currently halted. This allows SW to unblock the PBDMA and recover. | ||
1526 | * SW may read METHOD0, CHANNEL_STATUS and TARGET to determine whether the | ||
1527 | * interrupt was due to an engine method, CTX_RELOAD, SHOULD_SEND_HOST_TSG_EVENT | ||
1528 | * or NEEDS_HOST_TSG_EVENT. If METHOD0 VALID is TRUE, lazy context creation | ||
1529 | * can be used or the TSG may be destroyed. | ||
1530 | * If METHOD0 VALID is FALSE, the error is likely a bug in SW, and the TSG | ||
1531 | * will have to be destroyed. | ||
1532 | */ | ||
1533 | |||
1534 | unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g, | ||
1535 | u32 pbdma_id, u32 pbdma_intr_1, | ||
1536 | u32 *handled, u32 *error_notifier) | ||
1537 | { | ||
1538 | unsigned int rc_type = RC_TYPE_PBDMA_FAULT; | ||
1539 | u32 pbdma_intr_1_current = gk20a_readl(g, pbdma_intr_1_r(pbdma_id)); | ||
1540 | |||
1541 | /* minimize race with the gpu clearing the pending interrupt */ | ||
1542 | if (!(pbdma_intr_1_current & | ||
1543 | pbdma_intr_1_ctxnotvalid_pending_f())) | ||
1544 | pbdma_intr_1 &= ~pbdma_intr_1_ctxnotvalid_pending_f(); | ||
1545 | |||
1546 | if (pbdma_intr_1 == 0) | ||
1547 | return RC_TYPE_NO_RC; | ||
1548 | |||
1549 | if (pbdma_intr_1 & pbdma_intr_1_ctxnotvalid_pending_f()) { | ||
1550 | gk20a_dbg(gpu_dbg_intr, "ctxnotvalid intr on pbdma id %d", | ||
1551 | pbdma_id); | ||
1552 | nvgpu_err(g, "pbdma_intr_1(%d)= 0x%08x ", | ||
1553 | pbdma_id, pbdma_intr_1); | ||
1554 | *handled |= pbdma_intr_1_ctxnotvalid_pending_f(); | ||
1555 | } else{ | ||
1556 | /* | ||
1557 | * rest of the interrupts in _intr_1 are "host copy engine" | ||
1558 | * related, which is not supported. For now just make them | ||
1559 | * channel fatal. | ||
1560 | */ | ||
1561 | nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x", | ||
1562 | pbdma_id, pbdma_intr_1); | ||
1563 | *handled |= pbdma_intr_1; | ||
1564 | } | ||
1565 | |||
1566 | return rc_type; | ||
1567 | } | ||
1568 | |||
1569 | static void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g, | ||
1570 | struct channel_gk20a *ch, struct nvgpu_mem *mem) | ||
1571 | { | ||
1572 | struct tsg_gk20a *tsg; | ||
1573 | struct nvgpu_mem *method_buffer_per_runque; | ||
1574 | |||
1575 | tsg = tsg_gk20a_from_ch(ch); | ||
1576 | if (tsg == NULL) { | ||
1577 | nvgpu_err(g, "channel is not part of tsg"); | ||
1578 | return; | ||
1579 | } | ||
1580 | if (tsg->eng_method_buffers == NULL) { | ||
1581 | nvgpu_log_info(g, "eng method buffer NULL"); | ||
1582 | return; | ||
1583 | } | ||
1584 | if (tsg->runlist_id == gk20a_fifo_get_fast_ce_runlist_id(g)) | ||
1585 | method_buffer_per_runque = | ||
1586 | &tsg->eng_method_buffers[ASYNC_CE_RUNQUE]; | ||
1587 | else | ||
1588 | method_buffer_per_runque = | ||
1589 | &tsg->eng_method_buffers[GR_RUNQUE]; | ||
1590 | |||
1591 | nvgpu_mem_wr32(g, mem, ram_in_eng_method_buffer_addr_lo_w(), | ||
1592 | u64_lo32(method_buffer_per_runque->gpu_va)); | ||
1593 | nvgpu_mem_wr32(g, mem, ram_in_eng_method_buffer_addr_hi_w(), | ||
1594 | u64_hi32(method_buffer_per_runque->gpu_va)); | ||
1595 | |||
1596 | nvgpu_log_info(g, "init ramfc with method buffer"); | ||
1597 | } | ||
1598 | |||
1599 | unsigned int gv11b_fifo_get_eng_method_buffer_size(struct gk20a *g) | ||
1600 | { | ||
1601 | unsigned int buffer_size; | ||
1602 | |||
1603 | buffer_size = ((9 + 1 + 3) * g->ops.ce2.get_num_pce(g)) + 2; | ||
1604 | buffer_size = (27 * 5 * buffer_size); | ||
1605 | buffer_size = roundup(buffer_size, PAGE_SIZE); | ||
1606 | nvgpu_log_info(g, "method buffer size in bytes %d", buffer_size); | ||
1607 | |||
1608 | return buffer_size; | ||
1609 | } | ||
1610 | |||
1611 | void gv11b_fifo_init_eng_method_buffers(struct gk20a *g, | ||
1612 | struct tsg_gk20a *tsg) | ||
1613 | { | ||
1614 | struct vm_gk20a *vm = g->mm.bar2.vm; | ||
1615 | int err = 0; | ||
1616 | int i; | ||
1617 | unsigned int runque, method_buffer_size; | ||
1618 | unsigned int num_pbdma = g->fifo.num_pbdma; | ||
1619 | |||
1620 | if (tsg->eng_method_buffers != NULL) | ||
1621 | return; | ||
1622 | |||
1623 | method_buffer_size = gv11b_fifo_get_eng_method_buffer_size(g); | ||
1624 | if (method_buffer_size == 0) { | ||
1625 | nvgpu_info(g, "ce will hit MTHD_BUFFER_FAULT"); | ||
1626 | return; | ||
1627 | } | ||
1628 | |||
1629 | tsg->eng_method_buffers = nvgpu_kzalloc(g, | ||
1630 | num_pbdma * sizeof(struct nvgpu_mem)); | ||
1631 | |||
1632 | for (runque = 0; runque < num_pbdma; runque++) { | ||
1633 | err = nvgpu_dma_alloc_map_sys(vm, method_buffer_size, | ||
1634 | &tsg->eng_method_buffers[runque]); | ||
1635 | if (err) | ||
1636 | break; | ||
1637 | } | ||
1638 | if (err) { | ||
1639 | for (i = (runque - 1); i >= 0; i--) | ||
1640 | nvgpu_dma_unmap_free(vm, | ||
1641 | &tsg->eng_method_buffers[i]); | ||
1642 | |||
1643 | nvgpu_kfree(g, tsg->eng_method_buffers); | ||
1644 | tsg->eng_method_buffers = NULL; | ||
1645 | nvgpu_err(g, "could not alloc eng method buffers"); | ||
1646 | return; | ||
1647 | } | ||
1648 | nvgpu_log_info(g, "eng method buffers allocated"); | ||
1649 | |||
1650 | } | ||
1651 | |||
1652 | void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g, | ||
1653 | struct tsg_gk20a *tsg) | ||
1654 | { | ||
1655 | struct vm_gk20a *vm = g->mm.bar2.vm; | ||
1656 | unsigned int runque; | ||
1657 | |||
1658 | if (tsg->eng_method_buffers == NULL) | ||
1659 | return; | ||
1660 | |||
1661 | for (runque = 0; runque < g->fifo.num_pbdma; runque++) | ||
1662 | nvgpu_dma_unmap_free(vm, &tsg->eng_method_buffers[runque]); | ||
1663 | |||
1664 | nvgpu_kfree(g, tsg->eng_method_buffers); | ||
1665 | tsg->eng_method_buffers = NULL; | ||
1666 | |||
1667 | nvgpu_log_info(g, "eng method buffers de-allocated"); | ||
1668 | } | ||
1669 | |||
1670 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
1671 | int gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c, | ||
1672 | u32 syncpt_id, struct nvgpu_mem *syncpt_buf) | ||
1673 | { | ||
1674 | u32 nr_pages; | ||
1675 | int err = 0; | ||
1676 | struct gk20a *g = c->g; | ||
1677 | struct vm_gk20a *vm = c->vm; | ||
1678 | |||
1679 | /* | ||
1680 | * Add ro map for complete sync point shim range in vm | ||
1681 | * All channels sharing same vm will share same ro mapping. | ||
1682 | * Create rw map for current channel sync point | ||
1683 | */ | ||
1684 | if (!vm->syncpt_ro_map_gpu_va) { | ||
1685 | vm->syncpt_ro_map_gpu_va = nvgpu_gmmu_map(c->vm, | ||
1686 | &g->syncpt_mem, g->syncpt_unit_size, | ||
1687 | 0, gk20a_mem_flag_read_only, | ||
1688 | false, APERTURE_SYSMEM); | ||
1689 | |||
1690 | if (!vm->syncpt_ro_map_gpu_va) { | ||
1691 | nvgpu_err(g, "failed to ro map syncpt buffer"); | ||
1692 | nvgpu_dma_free(g, &g->syncpt_mem); | ||
1693 | err = -ENOMEM; | ||
1694 | } | ||
1695 | } | ||
1696 | |||
1697 | nr_pages = DIV_ROUND_UP(g->syncpt_size, PAGE_SIZE); | ||
1698 | __nvgpu_mem_create_from_phys(g, syncpt_buf, | ||
1699 | (g->syncpt_unit_base + | ||
1700 | nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(syncpt_id)), | ||
1701 | nr_pages); | ||
1702 | syncpt_buf->gpu_va = nvgpu_gmmu_map(c->vm, syncpt_buf, | ||
1703 | g->syncpt_size, 0, gk20a_mem_flag_none, | ||
1704 | false, APERTURE_SYSMEM); | ||
1705 | |||
1706 | if (!syncpt_buf->gpu_va) { | ||
1707 | nvgpu_err(g, "failed to map syncpt buffer"); | ||
1708 | nvgpu_dma_free(g, syncpt_buf); | ||
1709 | err = -ENOMEM; | ||
1710 | } | ||
1711 | return err; | ||
1712 | } | ||
1713 | |||
1714 | void gv11b_fifo_free_syncpt_buf(struct channel_gk20a *c, | ||
1715 | struct nvgpu_mem *syncpt_buf) | ||
1716 | { | ||
1717 | nvgpu_gmmu_unmap(c->vm, syncpt_buf, syncpt_buf->gpu_va); | ||
1718 | nvgpu_dma_free(c->g, syncpt_buf); | ||
1719 | } | ||
1720 | |||
1721 | void gv11b_fifo_add_syncpt_wait_cmd(struct gk20a *g, | ||
1722 | struct priv_cmd_entry *cmd, u32 off, | ||
1723 | u32 id, u32 thresh, u64 gpu_va_base) | ||
1724 | { | ||
1725 | u64 gpu_va = gpu_va_base + | ||
1726 | nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(id); | ||
1727 | |||
1728 | gk20a_dbg_fn(""); | ||
1729 | |||
1730 | off = cmd->off + off; | ||
1731 | |||
1732 | /* semaphore_a */ | ||
1733 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010004); | ||
1734 | nvgpu_mem_wr32(g, cmd->mem, off++, | ||
1735 | (gpu_va >> 32) & 0xff); | ||
1736 | /* semaphore_b */ | ||
1737 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010005); | ||
1738 | /* offset */ | ||
1739 | nvgpu_mem_wr32(g, cmd->mem, off++, gpu_va & 0xffffffff); | ||
1740 | |||
1741 | /* semaphore_c */ | ||
1742 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010006); | ||
1743 | /* payload */ | ||
1744 | nvgpu_mem_wr32(g, cmd->mem, off++, thresh); | ||
1745 | /* semaphore_d */ | ||
1746 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010007); | ||
1747 | /* operation: acq_geq, switch_en */ | ||
1748 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x4 | (0x1 << 12)); | ||
1749 | } | ||
1750 | |||
1751 | u32 gv11b_fifo_get_syncpt_wait_cmd_size(void) | ||
1752 | { | ||
1753 | return 8; | ||
1754 | } | ||
1755 | |||
1756 | void gv11b_fifo_add_syncpt_incr_cmd(struct gk20a *g, | ||
1757 | bool wfi_cmd, struct priv_cmd_entry *cmd, | ||
1758 | u32 id, u64 gpu_va) | ||
1759 | { | ||
1760 | u32 off = cmd->off; | ||
1761 | |||
1762 | gk20a_dbg_fn(""); | ||
1763 | |||
1764 | /* semaphore_a */ | ||
1765 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010004); | ||
1766 | nvgpu_mem_wr32(g, cmd->mem, off++, | ||
1767 | (gpu_va >> 32) & 0xff); | ||
1768 | /* semaphore_b */ | ||
1769 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010005); | ||
1770 | /* offset */ | ||
1771 | nvgpu_mem_wr32(g, cmd->mem, off++, gpu_va & 0xffffffff); | ||
1772 | |||
1773 | /* semaphore_c */ | ||
1774 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010006); | ||
1775 | /* payload */ | ||
1776 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x0); | ||
1777 | /* semaphore_d */ | ||
1778 | nvgpu_mem_wr32(g, cmd->mem, off++, 0x20010007); | ||
1779 | |||
1780 | /* operation: release, wfi */ | ||
1781 | nvgpu_mem_wr32(g, cmd->mem, off++, | ||
1782 | 0x2 | ((wfi_cmd ? 0x0 : 0x1) << 20)); | ||
1783 | /* ignored */ | ||
1784 | nvgpu_mem_wr32(g, cmd->mem, off++, 0); | ||
1785 | } | ||
1786 | |||
1787 | u32 gv11b_fifo_get_syncpt_incr_cmd_size(bool wfi_cmd) | ||
1788 | { | ||
1789 | return 9; | ||
1790 | } | ||
1791 | #endif /* CONFIG_TEGRA_GK20A_NVHOST */ | ||
1792 | |||
1793 | int gv11b_init_fifo_setup_hw(struct gk20a *g) | ||
1794 | { | ||
1795 | struct fifo_gk20a *f = &g->fifo; | ||
1796 | |||
1797 | f->t19x.max_subctx_count = | ||
1798 | gr_pri_fe_chip_def_info_max_veid_count_init_v(); | ||
1799 | return 0; | ||
1800 | } | ||
1801 | |||
1802 | static u32 gv11b_mmu_fault_id_to_gr_veid(struct gk20a *g, u32 gr_eng_fault_id, | ||
1803 | u32 mmu_fault_id) | ||
1804 | { | ||
1805 | struct fifo_gk20a *f = &g->fifo; | ||
1806 | u32 num_subctx; | ||
1807 | u32 veid = FIFO_INVAL_VEID; | ||
1808 | |||
1809 | num_subctx = f->t19x.max_subctx_count; | ||
1810 | |||
1811 | if (mmu_fault_id >= gr_eng_fault_id && | ||
1812 | mmu_fault_id < (gr_eng_fault_id + num_subctx)) | ||
1813 | veid = mmu_fault_id - gr_eng_fault_id; | ||
1814 | |||
1815 | return veid; | ||
1816 | } | ||
1817 | |||
1818 | static u32 gv11b_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g, | ||
1819 | u32 mmu_fault_id, u32 *veid) | ||
1820 | { | ||
1821 | u32 engine_id; | ||
1822 | u32 active_engine_id; | ||
1823 | struct fifo_engine_info_gk20a *engine_info; | ||
1824 | struct fifo_gk20a *f = &g->fifo; | ||
1825 | |||
1826 | |||
1827 | for (engine_id = 0; engine_id < f->num_engines; engine_id++) { | ||
1828 | active_engine_id = f->active_engines_list[engine_id]; | ||
1829 | engine_info = &g->fifo.engine_info[active_engine_id]; | ||
1830 | |||
1831 | if (active_engine_id == ENGINE_GR_GK20A) { | ||
1832 | /* get faulted subctx id */ | ||
1833 | *veid = gv11b_mmu_fault_id_to_gr_veid(g, | ||
1834 | engine_info->fault_id, mmu_fault_id); | ||
1835 | if (*veid != FIFO_INVAL_VEID) | ||
1836 | break; | ||
1837 | } else { | ||
1838 | if (engine_info->fault_id == mmu_fault_id) | ||
1839 | break; | ||
1840 | } | ||
1841 | |||
1842 | active_engine_id = FIFO_INVAL_ENGINE_ID; | ||
1843 | } | ||
1844 | return active_engine_id; | ||
1845 | } | ||
1846 | |||
1847 | static u32 gv11b_mmu_fault_id_to_pbdma_id(struct gk20a *g, u32 mmu_fault_id) | ||
1848 | { | ||
1849 | u32 num_pbdma, reg_val, fault_id_pbdma0; | ||
1850 | |||
1851 | reg_val = gk20a_readl(g, fifo_cfg0_r()); | ||
1852 | num_pbdma = fifo_cfg0_num_pbdma_v(reg_val); | ||
1853 | fault_id_pbdma0 = fifo_cfg0_pbdma_fault_id_v(reg_val); | ||
1854 | |||
1855 | if (mmu_fault_id >= fault_id_pbdma0 && | ||
1856 | mmu_fault_id <= fault_id_pbdma0 + num_pbdma - 1) | ||
1857 | return mmu_fault_id - fault_id_pbdma0; | ||
1858 | |||
1859 | return FIFO_INVAL_PBDMA_ID; | ||
1860 | } | ||
1861 | |||
1862 | void gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(struct gk20a *g, | ||
1863 | u32 mmu_fault_id, u32 *active_engine_id, u32 *veid, u32 *pbdma_id) | ||
1864 | { | ||
1865 | *active_engine_id = gv11b_mmu_fault_id_to_eng_id_and_veid(g, | ||
1866 | mmu_fault_id, veid); | ||
1867 | |||
1868 | if (*active_engine_id == FIFO_INVAL_ENGINE_ID) | ||
1869 | *pbdma_id = gv11b_mmu_fault_id_to_pbdma_id(g, mmu_fault_id); | ||
1870 | else | ||
1871 | *pbdma_id = FIFO_INVAL_PBDMA_ID; | ||
1872 | } | ||
1873 | |||
1874 | static bool gk20a_fifo_channel_status_is_eng_faulted(struct gk20a *g, u32 chid) | ||
1875 | { | ||
1876 | u32 channel = gk20a_readl(g, ccsr_channel_r(chid)); | ||
1877 | |||
1878 | return ccsr_channel_eng_faulted_v(channel) == | ||
1879 | ccsr_channel_eng_faulted_true_v(); | ||
1880 | } | ||
1881 | |||
1882 | void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch) | ||
1883 | { | ||
1884 | struct gk20a *g = ch->g; | ||
1885 | struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid]; | ||
1886 | |||
1887 | /* | ||
1888 | * If channel has FAULTED set, clear the CE method buffer | ||
1889 | * if saved out channel is same as faulted channel | ||
1890 | */ | ||
1891 | if (!gk20a_fifo_channel_status_is_eng_faulted(g, ch->chid)) | ||
1892 | return; | ||
1893 | |||
1894 | if (tsg->eng_method_buffers == NULL) | ||
1895 | return; | ||
1896 | |||
1897 | /* | ||
1898 | * CE method buffer format : | ||
1899 | * DWord0 = method count | ||
1900 | * DWord1 = channel id | ||
1901 | * | ||
1902 | * It is sufficient to write 0 to method count to invalidate | ||
1903 | */ | ||
1904 | if ((u32)ch->chid == | ||
1905 | nvgpu_mem_rd32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 1)) | ||
1906 | nvgpu_mem_wr32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 0, 0); | ||
1907 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h new file mode 100644 index 00000000..fc1ddf83 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h | |||
@@ -0,0 +1,117 @@ | |||
1 | /* | ||
2 | * GV11B Fifo | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef FIFO_GV11B_H | ||
26 | #define FIFO_GV11B_H | ||
27 | |||
28 | #define FIFO_INVAL_PBDMA_ID ((u32)~0) | ||
29 | #define FIFO_INVAL_VEID ((u32)~0) | ||
30 | |||
31 | /* engine context-switch request occurred while the engine was in reset */ | ||
32 | #define SCHED_ERROR_CODE_ENGINE_RESET 0x00000005 | ||
33 | |||
34 | /* | ||
35 | * ERROR_CODE_BAD_TSG indicates that Host encountered a badly formed TSG header | ||
36 | * or a badly formed channel type runlist entry in the runlist. This is typically | ||
37 | * caused by encountering a new TSG entry in the middle of a TSG definition. | ||
38 | * A channel type entry having wrong runqueue selector can also cause this. | ||
39 | * Additionally this error code can indicate when a channel is encountered on | ||
40 | * the runlist which is outside of a TSG. | ||
41 | */ | ||
42 | #define SCHED_ERROR_CODE_BAD_TSG 0x00000020 | ||
43 | |||
44 | /* can be removed after runque support is added */ | ||
45 | |||
46 | #define GR_RUNQUE 0 /* pbdma 0 */ | ||
47 | #define ASYNC_CE_RUNQUE 2 /* pbdma 2 */ | ||
48 | |||
49 | #define CHANNEL_INFO_VEID0 0 | ||
50 | |||
51 | struct gpu_ops; | ||
52 | |||
53 | void gv11b_fifo_reset_pbdma_and_eng_faulted(struct gk20a *g, | ||
54 | struct channel_gk20a *refch, | ||
55 | u32 faulted_pbdma, u32 faulted_engine); | ||
56 | void gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(struct gk20a *g, | ||
57 | u32 mmu_fault_id, u32 *active_engine_id, u32 *veid, u32 *pbdma_id); | ||
58 | |||
59 | void gv11b_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist); | ||
60 | void gv11b_get_ch_runlist_entry(struct channel_gk20a *c, u32 *runlist); | ||
61 | int channel_gv11b_setup_ramfc(struct channel_gk20a *c, | ||
62 | u64 gpfifo_base, u32 gpfifo_entries, | ||
63 | unsigned long acquire_timeout, u32 flags); | ||
64 | u32 gv11b_userd_gp_get(struct gk20a *g, struct channel_gk20a *c); | ||
65 | u64 gv11b_userd_pb_get(struct gk20a *g, struct channel_gk20a *c); | ||
66 | void gv11b_userd_gp_put(struct gk20a *g, struct channel_gk20a *c); | ||
67 | void channel_gv11b_unbind(struct channel_gk20a *ch); | ||
68 | u32 gv11b_fifo_get_num_fifos(struct gk20a *g); | ||
69 | bool gv11b_is_fault_engine_subid_gpc(struct gk20a *g, u32 engine_subid); | ||
70 | void gv11b_dump_channel_status_ramfc(struct gk20a *g, | ||
71 | struct gk20a_debug_output *o, | ||
72 | u32 chid, | ||
73 | struct ch_state *ch_state); | ||
74 | void gv11b_dump_eng_status(struct gk20a *g, | ||
75 | struct gk20a_debug_output *o); | ||
76 | u32 gv11b_fifo_intr_0_error_mask(struct gk20a *g); | ||
77 | int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, | ||
78 | unsigned int id_type, unsigned int timeout_rc_type); | ||
79 | int gv11b_fifo_preempt_channel(struct gk20a *g, u32 chid); | ||
80 | int gv11b_fifo_preempt_tsg(struct gk20a *g, u32 tsgid); | ||
81 | int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg); | ||
82 | int gv11b_fifo_preempt_ch_tsg(struct gk20a *g, u32 id, | ||
83 | unsigned int id_type, unsigned int timeout_rc_type); | ||
84 | void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, | ||
85 | u32 id, unsigned int id_type, unsigned int rc_type, | ||
86 | struct mmu_fault_info *mmfault); | ||
87 | void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f); | ||
88 | int gv11b_init_fifo_reset_enable_hw(struct gk20a *g); | ||
89 | bool gv11b_fifo_handle_sched_error(struct gk20a *g); | ||
90 | bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr); | ||
91 | unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g, | ||
92 | u32 pbdma_id, u32 pbdma_intr_0, | ||
93 | u32 *handled, u32 *error_notifier); | ||
94 | unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g, | ||
95 | u32 pbdma_id, u32 pbdma_intr_1, | ||
96 | u32 *handled, u32 *error_notifier); | ||
97 | void gv11b_fifo_init_eng_method_buffers(struct gk20a *g, | ||
98 | struct tsg_gk20a *tsg); | ||
99 | void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g, | ||
100 | struct tsg_gk20a *tsg); | ||
101 | int gv11b_fifo_alloc_syncpt_buf(struct channel_gk20a *c, | ||
102 | u32 syncpt_id, struct nvgpu_mem *syncpt_buf); | ||
103 | void gv11b_fifo_free_syncpt_buf(struct channel_gk20a *c, | ||
104 | struct nvgpu_mem *syncpt_buf); | ||
105 | void gv11b_fifo_add_syncpt_wait_cmd(struct gk20a *g, | ||
106 | struct priv_cmd_entry *cmd, u32 off, | ||
107 | u32 id, u32 thresh, u64 gpu_va_base); | ||
108 | u32 gv11b_fifo_get_syncpt_wait_cmd_size(void); | ||
109 | void gv11b_fifo_add_syncpt_incr_cmd(struct gk20a *g, | ||
110 | bool wfi_cmd, struct priv_cmd_entry *cmd, | ||
111 | u32 id, u64 gpu_va_base); | ||
112 | u32 gv11b_fifo_get_syncpt_incr_cmd_size(bool wfi_cmd); | ||
113 | int gv11b_init_fifo_setup_hw(struct gk20a *g); | ||
114 | |||
115 | void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch); | ||
116 | u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g); | ||
117 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c new file mode 100644 index 00000000..514aadb1 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.c | |||
@@ -0,0 +1,72 @@ | |||
1 | /* | ||
2 | * | ||
3 | * GV11B Graphics Context | ||
4 | * | ||
5 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
6 | * | ||
7 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
8 | * copy of this software and associated documentation files (the "Software"), | ||
9 | * to deal in the Software without restriction, including without limitation | ||
10 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
11 | * and/or sell copies of the Software, and to permit persons to whom the | ||
12 | * Software is furnished to do so, subject to the following conditions: | ||
13 | * | ||
14 | * The above copyright notice and this permission notice shall be included in | ||
15 | * all copies or substantial portions of the Software. | ||
16 | * | ||
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
23 | * DEALINGS IN THE SOFTWARE. | ||
24 | */ | ||
25 | |||
26 | #include "gk20a/gk20a.h" | ||
27 | |||
28 | #include "gr_ctx_gv11b.h" | ||
29 | |||
30 | int gr_gv11b_get_netlist_name(struct gk20a *g, int index, char *name) | ||
31 | { | ||
32 | switch (index) { | ||
33 | #ifdef GV11B_NETLIST_IMAGE_FW_NAME | ||
34 | case NETLIST_FINAL: | ||
35 | sprintf(name, GV11B_NETLIST_IMAGE_FW_NAME); | ||
36 | return 0; | ||
37 | #endif | ||
38 | #ifdef GK20A_NETLIST_IMAGE_A | ||
39 | case NETLIST_SLOT_A: | ||
40 | sprintf(name, GK20A_NETLIST_IMAGE_A); | ||
41 | return 0; | ||
42 | #endif | ||
43 | #ifdef GK20A_NETLIST_IMAGE_B | ||
44 | case NETLIST_SLOT_B: | ||
45 | sprintf(name, GK20A_NETLIST_IMAGE_B); | ||
46 | return 0; | ||
47 | #endif | ||
48 | #ifdef GK20A_NETLIST_IMAGE_C | ||
49 | case NETLIST_SLOT_C: | ||
50 | sprintf(name, GK20A_NETLIST_IMAGE_C); | ||
51 | return 0; | ||
52 | #endif | ||
53 | #ifdef GK20A_NETLIST_IMAGE_D | ||
54 | case NETLIST_SLOT_D: | ||
55 | sprintf(name, GK20A_NETLIST_IMAGE_D); | ||
56 | return 0; | ||
57 | #endif | ||
58 | default: | ||
59 | return -1; | ||
60 | } | ||
61 | |||
62 | return -1; | ||
63 | } | ||
64 | |||
65 | bool gr_gv11b_is_firmware_defined(void) | ||
66 | { | ||
67 | #ifdef GV11B_NETLIST_IMAGE_FW_NAME | ||
68 | return true; | ||
69 | #else | ||
70 | return false; | ||
71 | #endif | ||
72 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h new file mode 100644 index 00000000..0a95ab11 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gr_ctx_gv11b.h | |||
@@ -0,0 +1,36 @@ | |||
1 | /* | ||
2 | * GV11B Graphics Context | ||
3 | * | ||
4 | * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | #ifndef __GR_CTX_GV11B_H__ | ||
25 | #define __GR_CTX_GV11B_H__ | ||
26 | |||
27 | #include "gk20a/gr_ctx_gk20a.h" | ||
28 | |||
29 | /* Define netlist for silicon only */ | ||
30 | |||
31 | #define GV11B_NETLIST_IMAGE_FW_NAME GK20A_NETLIST_IMAGE_D | ||
32 | |||
33 | int gr_gv11b_get_netlist_name(struct gk20a *g, int index, char *name); | ||
34 | bool gr_gv11b_is_firmware_defined(void); | ||
35 | |||
36 | #endif /*__GR_CTX_GV11B_H__*/ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.c b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c new file mode 100644 index 00000000..3d817d7e --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.c | |||
@@ -0,0 +1,3639 @@ | |||
1 | /* | ||
2 | * GV11b GPU GR | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/delay.h> | ||
26 | #include <linux/version.h> | ||
27 | #include <linux/vmalloc.h> | ||
28 | #include <linux/tegra_gpu_t19x.h> | ||
29 | #include <uapi/linux/nvgpu.h> | ||
30 | |||
31 | #include <soc/tegra/fuse.h> | ||
32 | |||
33 | #include <nvgpu/timers.h> | ||
34 | #include <nvgpu/gmmu.h> | ||
35 | #include <nvgpu/dma.h> | ||
36 | #include <nvgpu/log.h> | ||
37 | #include <nvgpu/debug.h> | ||
38 | #include <nvgpu/enabled.h> | ||
39 | |||
40 | #include "gk20a/gk20a.h" | ||
41 | #include "gk20a/gr_gk20a.h" | ||
42 | #include "gk20a/dbg_gpu_gk20a.h" | ||
43 | #include "gk20a/regops_gk20a.h" | ||
44 | #include "gk20a/gr_pri_gk20a.h" | ||
45 | |||
46 | #include "gm20b/gr_gm20b.h" | ||
47 | |||
48 | #include "gp10b/gr_gp10b.h" | ||
49 | |||
50 | #include "gv11b/gr_gv11b.h" | ||
51 | #include "gv11b/mm_gv11b.h" | ||
52 | #include "gv11b/subctx_gv11b.h" | ||
53 | |||
54 | #include <nvgpu/hw/gv11b/hw_gr_gv11b.h> | ||
55 | #include <nvgpu/hw/gv11b/hw_fifo_gv11b.h> | ||
56 | #include <nvgpu/hw/gv11b/hw_proj_gv11b.h> | ||
57 | #include <nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h> | ||
58 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
59 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
60 | #include <nvgpu/hw/gv11b/hw_pbdma_gv11b.h> | ||
61 | #include <nvgpu/hw/gv11b/hw_therm_gv11b.h> | ||
62 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
63 | |||
64 | bool gr_gv11b_is_valid_class(struct gk20a *g, u32 class_num) | ||
65 | { | ||
66 | bool valid = false; | ||
67 | |||
68 | switch (class_num) { | ||
69 | case VOLTA_COMPUTE_A: | ||
70 | case VOLTA_A: | ||
71 | case VOLTA_DMA_COPY_A: | ||
72 | valid = true; | ||
73 | break; | ||
74 | |||
75 | case MAXWELL_COMPUTE_B: | ||
76 | case MAXWELL_B: | ||
77 | case FERMI_TWOD_A: | ||
78 | case KEPLER_DMA_COPY_A: | ||
79 | case MAXWELL_DMA_COPY_A: | ||
80 | case PASCAL_COMPUTE_A: | ||
81 | case PASCAL_A: | ||
82 | case PASCAL_DMA_COPY_A: | ||
83 | valid = true; | ||
84 | break; | ||
85 | |||
86 | default: | ||
87 | break; | ||
88 | } | ||
89 | gk20a_dbg_info("class=0x%x valid=%d", class_num, valid); | ||
90 | return valid; | ||
91 | } | ||
92 | |||
93 | bool gr_gv11b_is_valid_gfx_class(struct gk20a *g, u32 class_num) | ||
94 | { | ||
95 | bool valid = false; | ||
96 | |||
97 | switch (class_num) { | ||
98 | case VOLTA_A: | ||
99 | case PASCAL_A: | ||
100 | case MAXWELL_B: | ||
101 | valid = true; | ||
102 | break; | ||
103 | |||
104 | default: | ||
105 | break; | ||
106 | } | ||
107 | return valid; | ||
108 | } | ||
109 | |||
110 | bool gr_gv11b_is_valid_compute_class(struct gk20a *g, u32 class_num) | ||
111 | { | ||
112 | bool valid = false; | ||
113 | |||
114 | switch (class_num) { | ||
115 | case VOLTA_COMPUTE_A: | ||
116 | case PASCAL_COMPUTE_A: | ||
117 | case MAXWELL_COMPUTE_B: | ||
118 | valid = true; | ||
119 | break; | ||
120 | |||
121 | default: | ||
122 | break; | ||
123 | } | ||
124 | return valid; | ||
125 | } | ||
126 | |||
127 | static u32 gv11b_gr_sm_offset(struct gk20a *g, u32 sm) | ||
128 | { | ||
129 | |||
130 | u32 sm_pri_stride = nvgpu_get_litter_value(g, GPU_LIT_SM_PRI_STRIDE); | ||
131 | u32 sm_offset = sm_pri_stride * sm; | ||
132 | |||
133 | return sm_offset; | ||
134 | } | ||
135 | |||
136 | static int gr_gv11b_handle_l1_tag_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
137 | bool *post_event, struct channel_gk20a *fault_ch, | ||
138 | u32 *hww_global_esr) | ||
139 | { | ||
140 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
141 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); | ||
142 | u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; | ||
143 | u32 l1_tag_ecc_status, l1_tag_ecc_corrected_err_status = 0; | ||
144 | u32 l1_tag_ecc_uncorrected_err_status = 0; | ||
145 | u32 l1_tag_corrected_err_count_delta = 0; | ||
146 | u32 l1_tag_uncorrected_err_count_delta = 0; | ||
147 | bool is_l1_tag_ecc_corrected_total_err_overflow = 0; | ||
148 | bool is_l1_tag_ecc_uncorrected_total_err_overflow = 0; | ||
149 | |||
150 | /* Check for L1 tag ECC errors. */ | ||
151 | l1_tag_ecc_status = gk20a_readl(g, | ||
152 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_r() + offset); | ||
153 | l1_tag_ecc_corrected_err_status = l1_tag_ecc_status & | ||
154 | (gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_el1_0_m() | | ||
155 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_el1_1_m() | | ||
156 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_pixrpf_m() | | ||
157 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_miss_fifo_m()); | ||
158 | l1_tag_ecc_uncorrected_err_status = l1_tag_ecc_status & | ||
159 | (gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_el1_0_m() | | ||
160 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_el1_1_m() | | ||
161 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_pixrpf_m() | | ||
162 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_miss_fifo_m()); | ||
163 | |||
164 | if ((l1_tag_ecc_corrected_err_status == 0) && (l1_tag_ecc_uncorrected_err_status == 0)) | ||
165 | return 0; | ||
166 | |||
167 | l1_tag_corrected_err_count_delta = | ||
168 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_v( | ||
169 | gk20a_readl(g, | ||
170 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_r() + | ||
171 | offset)); | ||
172 | l1_tag_uncorrected_err_count_delta = | ||
173 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_v( | ||
174 | gk20a_readl(g, | ||
175 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_r() + | ||
176 | offset)); | ||
177 | is_l1_tag_ecc_corrected_total_err_overflow = | ||
178 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_corrected_err_total_counter_overflow_v(l1_tag_ecc_status); | ||
179 | is_l1_tag_ecc_uncorrected_total_err_overflow = | ||
180 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_uncorrected_err_total_counter_overflow_v(l1_tag_ecc_status); | ||
181 | |||
182 | if ((l1_tag_corrected_err_count_delta > 0) || is_l1_tag_ecc_corrected_total_err_overflow) { | ||
183 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
184 | "corrected error (SBE) detected in SM L1 tag! err_mask [%08x] is_overf [%d]", | ||
185 | l1_tag_ecc_corrected_err_status, is_l1_tag_ecc_corrected_total_err_overflow); | ||
186 | |||
187 | /* HW uses 16-bits counter */ | ||
188 | l1_tag_corrected_err_count_delta += | ||
189 | (is_l1_tag_ecc_corrected_total_err_overflow << | ||
190 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_total_s()); | ||
191 | g->ecc.gr.t19x.sm_l1_tag_corrected_err_count.counters[tpc] += | ||
192 | l1_tag_corrected_err_count_delta; | ||
193 | gk20a_writel(g, | ||
194 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_corrected_err_count_r() + offset, | ||
195 | 0); | ||
196 | } | ||
197 | if ((l1_tag_uncorrected_err_count_delta > 0) || is_l1_tag_ecc_uncorrected_total_err_overflow) { | ||
198 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
199 | "Uncorrected error (DBE) detected in SM L1 tag! err_mask [%08x] is_overf [%d]", | ||
200 | l1_tag_ecc_uncorrected_err_status, is_l1_tag_ecc_uncorrected_total_err_overflow); | ||
201 | |||
202 | /* HW uses 16-bits counter */ | ||
203 | l1_tag_uncorrected_err_count_delta += | ||
204 | (is_l1_tag_ecc_uncorrected_total_err_overflow << | ||
205 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_total_s()); | ||
206 | g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count.counters[tpc] += | ||
207 | l1_tag_uncorrected_err_count_delta; | ||
208 | gk20a_writel(g, | ||
209 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_uncorrected_err_count_r() + offset, | ||
210 | 0); | ||
211 | } | ||
212 | |||
213 | gk20a_writel(g, gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_r() + offset, | ||
214 | gr_pri_gpc0_tpc0_sm_l1_tag_ecc_status_reset_task_f()); | ||
215 | |||
216 | return 0; | ||
217 | |||
218 | } | ||
219 | |||
220 | static int gr_gv11b_handle_lrf_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
221 | bool *post_event, struct channel_gk20a *fault_ch, | ||
222 | u32 *hww_global_esr) | ||
223 | { | ||
224 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
225 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); | ||
226 | u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; | ||
227 | u32 lrf_ecc_status, lrf_ecc_corrected_err_status = 0; | ||
228 | u32 lrf_ecc_uncorrected_err_status = 0; | ||
229 | u32 lrf_corrected_err_count_delta = 0; | ||
230 | u32 lrf_uncorrected_err_count_delta = 0; | ||
231 | bool is_lrf_ecc_corrected_total_err_overflow = 0; | ||
232 | bool is_lrf_ecc_uncorrected_total_err_overflow = 0; | ||
233 | |||
234 | /* Check for LRF ECC errors. */ | ||
235 | lrf_ecc_status = gk20a_readl(g, | ||
236 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_r() + offset); | ||
237 | lrf_ecc_corrected_err_status = lrf_ecc_status & | ||
238 | (gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp0_m() | | ||
239 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp1_m() | | ||
240 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp2_m() | | ||
241 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp3_m() | | ||
242 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp4_m() | | ||
243 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp5_m() | | ||
244 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp6_m() | | ||
245 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_qrfdp7_m()); | ||
246 | lrf_ecc_uncorrected_err_status = lrf_ecc_status & | ||
247 | (gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp0_m() | | ||
248 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp1_m() | | ||
249 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp2_m() | | ||
250 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp3_m() | | ||
251 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp4_m() | | ||
252 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp5_m() | | ||
253 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp6_m() | | ||
254 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_qrfdp7_m()); | ||
255 | |||
256 | if ((lrf_ecc_corrected_err_status == 0) && (lrf_ecc_uncorrected_err_status == 0)) | ||
257 | return 0; | ||
258 | |||
259 | lrf_corrected_err_count_delta = | ||
260 | gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_v( | ||
261 | gk20a_readl(g, | ||
262 | gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_r() + | ||
263 | offset)); | ||
264 | lrf_uncorrected_err_count_delta = | ||
265 | gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_v( | ||
266 | gk20a_readl(g, | ||
267 | gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_r() + | ||
268 | offset)); | ||
269 | is_lrf_ecc_corrected_total_err_overflow = | ||
270 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_corrected_err_total_counter_overflow_v(lrf_ecc_status); | ||
271 | is_lrf_ecc_uncorrected_total_err_overflow = | ||
272 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_uncorrected_err_total_counter_overflow_v(lrf_ecc_status); | ||
273 | |||
274 | if ((lrf_corrected_err_count_delta > 0) || is_lrf_ecc_corrected_total_err_overflow) { | ||
275 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
276 | "corrected error (SBE) detected in SM LRF! err_mask [%08x] is_overf [%d]", | ||
277 | lrf_ecc_corrected_err_status, is_lrf_ecc_corrected_total_err_overflow); | ||
278 | |||
279 | /* HW uses 16-bits counter */ | ||
280 | lrf_corrected_err_count_delta += | ||
281 | (is_lrf_ecc_corrected_total_err_overflow << | ||
282 | gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_total_s()); | ||
283 | g->ecc.gr.t18x.sm_lrf_single_err_count.counters[tpc] += | ||
284 | lrf_corrected_err_count_delta; | ||
285 | gk20a_writel(g, | ||
286 | gr_pri_gpc0_tpc0_sm_lrf_ecc_corrected_err_count_r() + offset, | ||
287 | 0); | ||
288 | } | ||
289 | if ((lrf_uncorrected_err_count_delta > 0) || is_lrf_ecc_uncorrected_total_err_overflow) { | ||
290 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
291 | "Uncorrected error (DBE) detected in SM LRF! err_mask [%08x] is_overf [%d]", | ||
292 | lrf_ecc_uncorrected_err_status, is_lrf_ecc_uncorrected_total_err_overflow); | ||
293 | |||
294 | /* HW uses 16-bits counter */ | ||
295 | lrf_uncorrected_err_count_delta += | ||
296 | (is_lrf_ecc_uncorrected_total_err_overflow << | ||
297 | gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_total_s()); | ||
298 | g->ecc.gr.t18x.sm_lrf_double_err_count.counters[tpc] += | ||
299 | lrf_uncorrected_err_count_delta; | ||
300 | gk20a_writel(g, | ||
301 | gr_pri_gpc0_tpc0_sm_lrf_ecc_uncorrected_err_count_r() + offset, | ||
302 | 0); | ||
303 | } | ||
304 | |||
305 | gk20a_writel(g, gr_pri_gpc0_tpc0_sm_lrf_ecc_status_r() + offset, | ||
306 | gr_pri_gpc0_tpc0_sm_lrf_ecc_status_reset_task_f()); | ||
307 | |||
308 | return 0; | ||
309 | |||
310 | } | ||
311 | |||
312 | void gr_gv11b_enable_hww_exceptions(struct gk20a *g) | ||
313 | { | ||
314 | /* enable exceptions */ | ||
315 | gk20a_writel(g, gr_fe_hww_esr_r(), | ||
316 | gr_fe_hww_esr_en_enable_f() | | ||
317 | gr_fe_hww_esr_reset_active_f()); | ||
318 | gk20a_writel(g, gr_memfmt_hww_esr_r(), | ||
319 | gr_memfmt_hww_esr_en_enable_f() | | ||
320 | gr_memfmt_hww_esr_reset_active_f()); | ||
321 | } | ||
322 | |||
323 | void gr_gv11b_enable_exceptions(struct gk20a *g) | ||
324 | { | ||
325 | struct gr_gk20a *gr = &g->gr; | ||
326 | u32 reg_val; | ||
327 | |||
328 | /* | ||
329 | * clear exceptions : | ||
330 | * other than SM : hww_esr are reset in *enable_hww_excetpions* | ||
331 | * SM : cleared in *set_hww_esr_report_mask* | ||
332 | */ | ||
333 | |||
334 | /* enable exceptions */ | ||
335 | gk20a_writel(g, gr_exception2_en_r(), 0x0); /* BE not enabled */ | ||
336 | gk20a_writel(g, gr_exception1_en_r(), (1 << gr->gpc_count) - 1); | ||
337 | |||
338 | reg_val = gr_exception_en_fe_enabled_f() | | ||
339 | gr_exception_en_memfmt_enabled_f() | | ||
340 | gr_exception_en_ds_enabled_f() | | ||
341 | gr_exception_en_gpc_enabled_f(); | ||
342 | gk20a_writel(g, gr_exception_en_r(), reg_val); | ||
343 | |||
344 | } | ||
345 | |||
346 | static int gr_gv11b_handle_cbu_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
347 | bool *post_event, struct channel_gk20a *fault_ch, | ||
348 | u32 *hww_global_esr) | ||
349 | { | ||
350 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
351 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); | ||
352 | u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; | ||
353 | u32 cbu_ecc_status, cbu_ecc_corrected_err_status = 0; | ||
354 | u32 cbu_ecc_uncorrected_err_status = 0; | ||
355 | u32 cbu_corrected_err_count_delta = 0; | ||
356 | u32 cbu_uncorrected_err_count_delta = 0; | ||
357 | bool is_cbu_ecc_corrected_total_err_overflow = 0; | ||
358 | bool is_cbu_ecc_uncorrected_total_err_overflow = 0; | ||
359 | |||
360 | /* Check for CBU ECC errors. */ | ||
361 | cbu_ecc_status = gk20a_readl(g, | ||
362 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_r() + offset); | ||
363 | cbu_ecc_corrected_err_status = cbu_ecc_status & | ||
364 | (gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_warp_sm0_m() | | ||
365 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_warp_sm1_m() | | ||
366 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_barrier_sm0_m() | | ||
367 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_barrier_sm1_m()); | ||
368 | cbu_ecc_uncorrected_err_status = cbu_ecc_status & | ||
369 | (gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_warp_sm0_m() | | ||
370 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_warp_sm1_m() | | ||
371 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_barrier_sm0_m() | | ||
372 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_barrier_sm1_m()); | ||
373 | |||
374 | if ((cbu_ecc_corrected_err_status == 0) && (cbu_ecc_uncorrected_err_status == 0)) | ||
375 | return 0; | ||
376 | |||
377 | cbu_corrected_err_count_delta = | ||
378 | gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_v( | ||
379 | gk20a_readl(g, | ||
380 | gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_r() + | ||
381 | offset)); | ||
382 | cbu_uncorrected_err_count_delta = | ||
383 | gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_v( | ||
384 | gk20a_readl(g, | ||
385 | gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_r() + | ||
386 | offset)); | ||
387 | is_cbu_ecc_corrected_total_err_overflow = | ||
388 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_corrected_err_total_counter_overflow_v(cbu_ecc_status); | ||
389 | is_cbu_ecc_uncorrected_total_err_overflow = | ||
390 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_uncorrected_err_total_counter_overflow_v(cbu_ecc_status); | ||
391 | |||
392 | if ((cbu_corrected_err_count_delta > 0) || is_cbu_ecc_corrected_total_err_overflow) { | ||
393 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
394 | "corrected error (SBE) detected in SM CBU! err_mask [%08x] is_overf [%d]", | ||
395 | cbu_ecc_corrected_err_status, is_cbu_ecc_corrected_total_err_overflow); | ||
396 | |||
397 | /* HW uses 16-bits counter */ | ||
398 | cbu_corrected_err_count_delta += | ||
399 | (is_cbu_ecc_corrected_total_err_overflow << | ||
400 | gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_total_s()); | ||
401 | g->ecc.gr.t19x.sm_cbu_corrected_err_count.counters[tpc] += | ||
402 | cbu_corrected_err_count_delta; | ||
403 | gk20a_writel(g, | ||
404 | gr_pri_gpc0_tpc0_sm_cbu_ecc_corrected_err_count_r() + offset, | ||
405 | 0); | ||
406 | } | ||
407 | if ((cbu_uncorrected_err_count_delta > 0) || is_cbu_ecc_uncorrected_total_err_overflow) { | ||
408 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
409 | "Uncorrected error (DBE) detected in SM CBU! err_mask [%08x] is_overf [%d]", | ||
410 | cbu_ecc_uncorrected_err_status, is_cbu_ecc_uncorrected_total_err_overflow); | ||
411 | |||
412 | /* HW uses 16-bits counter */ | ||
413 | cbu_uncorrected_err_count_delta += | ||
414 | (is_cbu_ecc_uncorrected_total_err_overflow << | ||
415 | gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_total_s()); | ||
416 | g->ecc.gr.t19x.sm_cbu_uncorrected_err_count.counters[tpc] += | ||
417 | cbu_uncorrected_err_count_delta; | ||
418 | gk20a_writel(g, | ||
419 | gr_pri_gpc0_tpc0_sm_cbu_ecc_uncorrected_err_count_r() + offset, | ||
420 | 0); | ||
421 | } | ||
422 | |||
423 | gk20a_writel(g, gr_pri_gpc0_tpc0_sm_cbu_ecc_status_r() + offset, | ||
424 | gr_pri_gpc0_tpc0_sm_cbu_ecc_status_reset_task_f()); | ||
425 | |||
426 | return 0; | ||
427 | |||
428 | } | ||
429 | |||
430 | static int gr_gv11b_handle_l1_data_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
431 | bool *post_event, struct channel_gk20a *fault_ch, | ||
432 | u32 *hww_global_esr) | ||
433 | { | ||
434 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
435 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); | ||
436 | u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; | ||
437 | u32 l1_data_ecc_status, l1_data_ecc_corrected_err_status = 0; | ||
438 | u32 l1_data_ecc_uncorrected_err_status = 0; | ||
439 | u32 l1_data_corrected_err_count_delta = 0; | ||
440 | u32 l1_data_uncorrected_err_count_delta = 0; | ||
441 | bool is_l1_data_ecc_corrected_total_err_overflow = 0; | ||
442 | bool is_l1_data_ecc_uncorrected_total_err_overflow = 0; | ||
443 | |||
444 | /* Check for L1 data ECC errors. */ | ||
445 | l1_data_ecc_status = gk20a_readl(g, | ||
446 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_r() + offset); | ||
447 | l1_data_ecc_corrected_err_status = l1_data_ecc_status & | ||
448 | (gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_el1_0_m() | | ||
449 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_el1_1_m()); | ||
450 | l1_data_ecc_uncorrected_err_status = l1_data_ecc_status & | ||
451 | (gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_el1_0_m() | | ||
452 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_el1_1_m()); | ||
453 | |||
454 | if ((l1_data_ecc_corrected_err_status == 0) && (l1_data_ecc_uncorrected_err_status == 0)) | ||
455 | return 0; | ||
456 | |||
457 | l1_data_corrected_err_count_delta = | ||
458 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_v( | ||
459 | gk20a_readl(g, | ||
460 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_r() + | ||
461 | offset)); | ||
462 | l1_data_uncorrected_err_count_delta = | ||
463 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_v( | ||
464 | gk20a_readl(g, | ||
465 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_r() + | ||
466 | offset)); | ||
467 | is_l1_data_ecc_corrected_total_err_overflow = | ||
468 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_corrected_err_total_counter_overflow_v(l1_data_ecc_status); | ||
469 | is_l1_data_ecc_uncorrected_total_err_overflow = | ||
470 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_uncorrected_err_total_counter_overflow_v(l1_data_ecc_status); | ||
471 | |||
472 | if ((l1_data_corrected_err_count_delta > 0) || is_l1_data_ecc_corrected_total_err_overflow) { | ||
473 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
474 | "corrected error (SBE) detected in SM L1 data! err_mask [%08x] is_overf [%d]", | ||
475 | l1_data_ecc_corrected_err_status, is_l1_data_ecc_corrected_total_err_overflow); | ||
476 | |||
477 | /* HW uses 16-bits counter */ | ||
478 | l1_data_corrected_err_count_delta += | ||
479 | (is_l1_data_ecc_corrected_total_err_overflow << | ||
480 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_total_s()); | ||
481 | g->ecc.gr.t19x.sm_l1_data_corrected_err_count.counters[tpc] += | ||
482 | l1_data_corrected_err_count_delta; | ||
483 | gk20a_writel(g, | ||
484 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_corrected_err_count_r() + offset, | ||
485 | 0); | ||
486 | } | ||
487 | if ((l1_data_uncorrected_err_count_delta > 0) || is_l1_data_ecc_uncorrected_total_err_overflow) { | ||
488 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
489 | "Uncorrected error (DBE) detected in SM L1 data! err_mask [%08x] is_overf [%d]", | ||
490 | l1_data_ecc_uncorrected_err_status, is_l1_data_ecc_uncorrected_total_err_overflow); | ||
491 | |||
492 | /* HW uses 16-bits counter */ | ||
493 | l1_data_uncorrected_err_count_delta += | ||
494 | (is_l1_data_ecc_uncorrected_total_err_overflow << | ||
495 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_total_s()); | ||
496 | g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count.counters[tpc] += | ||
497 | l1_data_uncorrected_err_count_delta; | ||
498 | gk20a_writel(g, | ||
499 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_uncorrected_err_count_r() + offset, | ||
500 | 0); | ||
501 | } | ||
502 | |||
503 | gk20a_writel(g, gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_r() + offset, | ||
504 | gr_pri_gpc0_tpc0_sm_l1_data_ecc_status_reset_task_f()); | ||
505 | |||
506 | return 0; | ||
507 | |||
508 | } | ||
509 | |||
510 | static int gr_gv11b_handle_icache_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
511 | bool *post_event, struct channel_gk20a *fault_ch, | ||
512 | u32 *hww_global_esr) | ||
513 | { | ||
514 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
515 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_TPC_IN_GPC_STRIDE); | ||
516 | u32 offset = gpc_stride * gpc + tpc_in_gpc_stride * tpc; | ||
517 | u32 icache_ecc_status, icache_ecc_corrected_err_status = 0; | ||
518 | u32 icache_ecc_uncorrected_err_status = 0; | ||
519 | u32 icache_corrected_err_count_delta = 0; | ||
520 | u32 icache_uncorrected_err_count_delta = 0; | ||
521 | bool is_icache_ecc_corrected_total_err_overflow = 0; | ||
522 | bool is_icache_ecc_uncorrected_total_err_overflow = 0; | ||
523 | |||
524 | /* Check for L0 && L1 icache ECC errors. */ | ||
525 | icache_ecc_status = gk20a_readl(g, | ||
526 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_r() + offset); | ||
527 | icache_ecc_corrected_err_status = icache_ecc_status & | ||
528 | (gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l0_data_m() | | ||
529 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l0_predecode_m() | | ||
530 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l1_data_m() | | ||
531 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_l1_predecode_m()); | ||
532 | icache_ecc_uncorrected_err_status = icache_ecc_status & | ||
533 | (gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l0_data_m() | | ||
534 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l0_predecode_m() | | ||
535 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l1_data_m() | | ||
536 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_l1_predecode_m()); | ||
537 | |||
538 | if ((icache_ecc_corrected_err_status == 0) && (icache_ecc_uncorrected_err_status == 0)) | ||
539 | return 0; | ||
540 | |||
541 | icache_corrected_err_count_delta = | ||
542 | gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_v( | ||
543 | gk20a_readl(g, | ||
544 | gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_r() + | ||
545 | offset)); | ||
546 | icache_uncorrected_err_count_delta = | ||
547 | gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_v( | ||
548 | gk20a_readl(g, | ||
549 | gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_r() + | ||
550 | offset)); | ||
551 | is_icache_ecc_corrected_total_err_overflow = | ||
552 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_corrected_err_total_counter_overflow_v(icache_ecc_status); | ||
553 | is_icache_ecc_uncorrected_total_err_overflow = | ||
554 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_uncorrected_err_total_counter_overflow_v(icache_ecc_status); | ||
555 | |||
556 | if ((icache_corrected_err_count_delta > 0) || is_icache_ecc_corrected_total_err_overflow) { | ||
557 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
558 | "corrected error (SBE) detected in SM L0 && L1 icache! err_mask [%08x] is_overf [%d]", | ||
559 | icache_ecc_corrected_err_status, is_icache_ecc_corrected_total_err_overflow); | ||
560 | |||
561 | /* HW uses 16-bits counter */ | ||
562 | icache_corrected_err_count_delta += | ||
563 | (is_icache_ecc_corrected_total_err_overflow << | ||
564 | gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_total_s()); | ||
565 | g->ecc.gr.t19x.sm_icache_corrected_err_count.counters[tpc] += | ||
566 | icache_corrected_err_count_delta; | ||
567 | gk20a_writel(g, | ||
568 | gr_pri_gpc0_tpc0_sm_icache_ecc_corrected_err_count_r() + offset, | ||
569 | 0); | ||
570 | } | ||
571 | if ((icache_uncorrected_err_count_delta > 0) || is_icache_ecc_uncorrected_total_err_overflow) { | ||
572 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_intr, | ||
573 | "Uncorrected error (DBE) detected in SM L0 && L1 icache! err_mask [%08x] is_overf [%d]", | ||
574 | icache_ecc_uncorrected_err_status, is_icache_ecc_uncorrected_total_err_overflow); | ||
575 | |||
576 | /* HW uses 16-bits counter */ | ||
577 | icache_uncorrected_err_count_delta += | ||
578 | (is_icache_ecc_uncorrected_total_err_overflow << | ||
579 | gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_total_s()); | ||
580 | g->ecc.gr.t19x.sm_icache_uncorrected_err_count.counters[tpc] += | ||
581 | icache_uncorrected_err_count_delta; | ||
582 | gk20a_writel(g, | ||
583 | gr_pri_gpc0_tpc0_sm_icache_ecc_uncorrected_err_count_r() + offset, | ||
584 | 0); | ||
585 | } | ||
586 | |||
587 | gk20a_writel(g, gr_pri_gpc0_tpc0_sm_icache_ecc_status_r() + offset, | ||
588 | gr_pri_gpc0_tpc0_sm_icache_ecc_status_reset_task_f()); | ||
589 | |||
590 | return 0; | ||
591 | |||
592 | } | ||
593 | |||
594 | int gr_gv11b_handle_tpc_sm_ecc_exception(struct gk20a *g, | ||
595 | u32 gpc, u32 tpc, | ||
596 | bool *post_event, struct channel_gk20a *fault_ch, | ||
597 | u32 *hww_global_esr) | ||
598 | { | ||
599 | int ret = 0; | ||
600 | |||
601 | /* Check for L1 tag ECC errors. */ | ||
602 | gr_gv11b_handle_l1_tag_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); | ||
603 | |||
604 | /* Check for LRF ECC errors. */ | ||
605 | gr_gv11b_handle_lrf_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); | ||
606 | |||
607 | /* Check for CBU ECC errors. */ | ||
608 | gr_gv11b_handle_cbu_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); | ||
609 | |||
610 | /* Check for L1 data ECC errors. */ | ||
611 | gr_gv11b_handle_l1_data_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); | ||
612 | |||
613 | /* Check for L0 && L1 icache ECC errors. */ | ||
614 | gr_gv11b_handle_icache_exception(g, gpc, tpc, post_event, fault_ch, hww_global_esr); | ||
615 | |||
616 | return ret; | ||
617 | } | ||
618 | |||
619 | int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
620 | bool *post_event, struct channel_gk20a *fault_ch, | ||
621 | u32 *hww_global_esr) | ||
622 | { | ||
623 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
624 | u32 offset = gpc_stride * gpc; | ||
625 | u32 gcc_l15_ecc_status, gcc_l15_ecc_corrected_err_status = 0; | ||
626 | u32 gcc_l15_ecc_uncorrected_err_status = 0; | ||
627 | u32 gcc_l15_corrected_err_count_delta = 0; | ||
628 | u32 gcc_l15_uncorrected_err_count_delta = 0; | ||
629 | bool is_gcc_l15_ecc_corrected_total_err_overflow = 0; | ||
630 | bool is_gcc_l15_ecc_uncorrected_total_err_overflow = 0; | ||
631 | |||
632 | /* Check for gcc l15 ECC errors. */ | ||
633 | gcc_l15_ecc_status = gk20a_readl(g, | ||
634 | gr_pri_gpc0_gcc_l15_ecc_status_r() + offset); | ||
635 | gcc_l15_ecc_corrected_err_status = gcc_l15_ecc_status & | ||
636 | (gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_bank0_m() | | ||
637 | gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_bank1_m()); | ||
638 | gcc_l15_ecc_uncorrected_err_status = gcc_l15_ecc_status & | ||
639 | (gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_bank0_m() | | ||
640 | gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_bank1_m()); | ||
641 | |||
642 | if ((gcc_l15_ecc_corrected_err_status == 0) && (gcc_l15_ecc_uncorrected_err_status == 0)) | ||
643 | return 0; | ||
644 | |||
645 | gcc_l15_corrected_err_count_delta = | ||
646 | gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_v( | ||
647 | gk20a_readl(g, | ||
648 | gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_r() + | ||
649 | offset)); | ||
650 | gcc_l15_uncorrected_err_count_delta = | ||
651 | gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_v( | ||
652 | gk20a_readl(g, | ||
653 | gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_r() + | ||
654 | offset)); | ||
655 | is_gcc_l15_ecc_corrected_total_err_overflow = | ||
656 | gr_pri_gpc0_gcc_l15_ecc_status_corrected_err_total_counter_overflow_v(gcc_l15_ecc_status); | ||
657 | is_gcc_l15_ecc_uncorrected_total_err_overflow = | ||
658 | gr_pri_gpc0_gcc_l15_ecc_status_uncorrected_err_total_counter_overflow_v(gcc_l15_ecc_status); | ||
659 | |||
660 | if ((gcc_l15_corrected_err_count_delta > 0) || is_gcc_l15_ecc_corrected_total_err_overflow) { | ||
661 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr, | ||
662 | "corrected error (SBE) detected in GCC L1.5! err_mask [%08x] is_overf [%d]", | ||
663 | gcc_l15_ecc_corrected_err_status, is_gcc_l15_ecc_corrected_total_err_overflow); | ||
664 | |||
665 | /* HW uses 16-bits counter */ | ||
666 | gcc_l15_corrected_err_count_delta += | ||
667 | (is_gcc_l15_ecc_corrected_total_err_overflow << | ||
668 | gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_total_s()); | ||
669 | g->ecc.gr.t19x.gcc_l15_corrected_err_count.counters[gpc] += | ||
670 | gcc_l15_corrected_err_count_delta; | ||
671 | gk20a_writel(g, | ||
672 | gr_pri_gpc0_gcc_l15_ecc_corrected_err_count_r() + offset, | ||
673 | 0); | ||
674 | } | ||
675 | if ((gcc_l15_uncorrected_err_count_delta > 0) || is_gcc_l15_ecc_uncorrected_total_err_overflow) { | ||
676 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_intr, | ||
677 | "Uncorrected error (DBE) detected in GCC L1.5! err_mask [%08x] is_overf [%d]", | ||
678 | gcc_l15_ecc_uncorrected_err_status, is_gcc_l15_ecc_uncorrected_total_err_overflow); | ||
679 | |||
680 | /* HW uses 16-bits counter */ | ||
681 | gcc_l15_uncorrected_err_count_delta += | ||
682 | (is_gcc_l15_ecc_uncorrected_total_err_overflow << | ||
683 | gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_total_s()); | ||
684 | g->ecc.gr.t19x.gcc_l15_uncorrected_err_count.counters[gpc] += | ||
685 | gcc_l15_uncorrected_err_count_delta; | ||
686 | gk20a_writel(g, | ||
687 | gr_pri_gpc0_gcc_l15_ecc_uncorrected_err_count_r() + offset, | ||
688 | 0); | ||
689 | } | ||
690 | |||
691 | gk20a_writel(g, gr_pri_gpc0_gcc_l15_ecc_status_r() + offset, | ||
692 | gr_pri_gpc0_gcc_l15_ecc_status_reset_task_f()); | ||
693 | |||
694 | return 0; | ||
695 | } | ||
696 | |||
697 | static int gr_gv11b_handle_gpcmmu_ecc_exception(struct gk20a *g, u32 gpc, | ||
698 | u32 exception) | ||
699 | { | ||
700 | int ret = 0; | ||
701 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
702 | u32 offset = gpc_stride * gpc; | ||
703 | u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt; | ||
704 | u32 corrected_delta, uncorrected_delta; | ||
705 | u32 corrected_overflow, uncorrected_overflow; | ||
706 | int hww_esr; | ||
707 | |||
708 | hww_esr = gk20a_readl(g, gr_gpc0_mmu_gpcmmu_global_esr_r() + offset); | ||
709 | |||
710 | if (!(hww_esr & (gr_gpc0_mmu_gpcmmu_global_esr_ecc_corrected_m() | | ||
711 | gr_gpc0_mmu_gpcmmu_global_esr_ecc_uncorrected_m()))) | ||
712 | return ret; | ||
713 | |||
714 | ecc_status = gk20a_readl(g, | ||
715 | gr_gpc0_mmu_l1tlb_ecc_status_r() + offset); | ||
716 | ecc_addr = gk20a_readl(g, | ||
717 | gr_gpc0_mmu_l1tlb_ecc_address_r() + offset); | ||
718 | corrected_cnt = gk20a_readl(g, | ||
719 | gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_r() + offset); | ||
720 | uncorrected_cnt = gk20a_readl(g, | ||
721 | gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_r() + offset); | ||
722 | |||
723 | corrected_delta = gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_total_v( | ||
724 | corrected_cnt); | ||
725 | uncorrected_delta = gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_total_v( | ||
726 | uncorrected_cnt); | ||
727 | corrected_overflow = ecc_status & | ||
728 | gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
729 | |||
730 | uncorrected_overflow = ecc_status & | ||
731 | gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
732 | |||
733 | |||
734 | /* clear the interrupt */ | ||
735 | if ((corrected_delta > 0) || corrected_overflow) | ||
736 | gk20a_writel(g, | ||
737 | gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_r() + | ||
738 | offset, 0); | ||
739 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
740 | gk20a_writel(g, | ||
741 | gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_r() + | ||
742 | offset, 0); | ||
743 | |||
744 | gk20a_writel(g, gr_gpc0_mmu_l1tlb_ecc_status_r() + offset, | ||
745 | gr_gpc0_mmu_l1tlb_ecc_status_reset_task_f()); | ||
746 | |||
747 | /* Handle overflow */ | ||
748 | if (corrected_overflow) | ||
749 | corrected_delta += (0x1UL << gr_gpc0_mmu_l1tlb_ecc_corrected_err_count_total_s()); | ||
750 | if (uncorrected_overflow) | ||
751 | uncorrected_delta += (0x1UL << gr_gpc0_mmu_l1tlb_ecc_uncorrected_err_count_total_s()); | ||
752 | |||
753 | |||
754 | g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count.counters[gpc] += | ||
755 | corrected_delta; | ||
756 | g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count.counters[gpc] += | ||
757 | uncorrected_delta; | ||
758 | nvgpu_log(g, gpu_dbg_intr, | ||
759 | "mmu l1tlb gpc:%d ecc interrupt intr: 0x%x", gpc, hww_esr); | ||
760 | |||
761 | if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_l1tlb_sa_data_m()) | ||
762 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
763 | if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_l1tlb_sa_data_m()) | ||
764 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
765 | if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_corrected_err_l1tlb_fa_data_m()) | ||
766 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc fa data error"); | ||
767 | if (ecc_status & gr_gpc0_mmu_l1tlb_ecc_status_uncorrected_err_l1tlb_fa_data_m()) | ||
768 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc fa data error"); | ||
769 | if (corrected_overflow || uncorrected_overflow) | ||
770 | nvgpu_info(g, "mmu l1tlb ecc counter overflow!"); | ||
771 | |||
772 | nvgpu_log(g, gpu_dbg_intr, | ||
773 | "ecc error address: 0x%x", ecc_addr); | ||
774 | nvgpu_log(g, gpu_dbg_intr, | ||
775 | "ecc error count corrected: %d, uncorrected %d", | ||
776 | g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count.counters[gpc], | ||
777 | g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count.counters[gpc]); | ||
778 | |||
779 | return ret; | ||
780 | } | ||
781 | |||
782 | static int gr_gv11b_handle_gpccs_ecc_exception(struct gk20a *g, u32 gpc, | ||
783 | u32 exception) | ||
784 | { | ||
785 | int ret = 0; | ||
786 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
787 | u32 offset = gpc_stride * gpc; | ||
788 | u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt; | ||
789 | u32 corrected_delta, uncorrected_delta; | ||
790 | u32 corrected_overflow, uncorrected_overflow; | ||
791 | int hww_esr; | ||
792 | |||
793 | hww_esr = gk20a_readl(g, gr_gpc0_gpccs_hww_esr_r() + offset); | ||
794 | |||
795 | if (!(hww_esr & (gr_gpc0_gpccs_hww_esr_ecc_uncorrected_m() | | ||
796 | gr_gpc0_gpccs_hww_esr_ecc_corrected_m()))) | ||
797 | return ret; | ||
798 | |||
799 | ecc_status = gk20a_readl(g, | ||
800 | gr_gpc0_gpccs_falcon_ecc_status_r() + offset); | ||
801 | ecc_addr = gk20a_readl(g, | ||
802 | gr_gpc0_gpccs_falcon_ecc_address_r() + offset); | ||
803 | corrected_cnt = gk20a_readl(g, | ||
804 | gr_gpc0_gpccs_falcon_ecc_corrected_err_count_r() + offset); | ||
805 | uncorrected_cnt = gk20a_readl(g, | ||
806 | gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_r() + offset); | ||
807 | |||
808 | corrected_delta = gr_gpc0_gpccs_falcon_ecc_corrected_err_count_total_v( | ||
809 | corrected_cnt); | ||
810 | uncorrected_delta = gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_total_v( | ||
811 | uncorrected_cnt); | ||
812 | corrected_overflow = ecc_status & | ||
813 | gr_gpc0_gpccs_falcon_ecc_status_corrected_err_total_counter_overflow_m(); | ||
814 | |||
815 | uncorrected_overflow = ecc_status & | ||
816 | gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
817 | |||
818 | |||
819 | /* clear the interrupt */ | ||
820 | if ((corrected_delta > 0) || corrected_overflow) | ||
821 | gk20a_writel(g, | ||
822 | gr_gpc0_gpccs_falcon_ecc_corrected_err_count_r() + | ||
823 | offset, 0); | ||
824 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
825 | gk20a_writel(g, | ||
826 | gr_gpc0_gpccs_falcon_ecc_uncorrected_err_count_r() + | ||
827 | offset, 0); | ||
828 | |||
829 | gk20a_writel(g, gr_gpc0_gpccs_falcon_ecc_status_r() + offset, | ||
830 | gr_gpc0_gpccs_falcon_ecc_status_reset_task_f()); | ||
831 | |||
832 | g->ecc.gr.t19x.gpccs_corrected_err_count.counters[gpc] += | ||
833 | corrected_delta; | ||
834 | g->ecc.gr.t19x.gpccs_uncorrected_err_count.counters[gpc] += | ||
835 | uncorrected_delta; | ||
836 | nvgpu_log(g, gpu_dbg_intr, | ||
837 | "gppcs gpc:%d ecc interrupt intr: 0x%x", gpc, hww_esr); | ||
838 | |||
839 | if (ecc_status & gr_gpc0_gpccs_falcon_ecc_status_corrected_err_imem_m()) | ||
840 | nvgpu_log(g, gpu_dbg_intr, "imem ecc error corrected"); | ||
841 | if (ecc_status & | ||
842 | gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_imem_m()) | ||
843 | nvgpu_log(g, gpu_dbg_intr, "imem ecc error uncorrected"); | ||
844 | if (ecc_status & | ||
845 | gr_gpc0_gpccs_falcon_ecc_status_corrected_err_dmem_m()) | ||
846 | nvgpu_log(g, gpu_dbg_intr, "dmem ecc error corrected"); | ||
847 | if (ecc_status & | ||
848 | gr_gpc0_gpccs_falcon_ecc_status_uncorrected_err_dmem_m()) | ||
849 | nvgpu_log(g, gpu_dbg_intr, "dmem ecc error uncorrected"); | ||
850 | if (corrected_overflow || uncorrected_overflow) | ||
851 | nvgpu_info(g, "gpccs ecc counter overflow!"); | ||
852 | |||
853 | nvgpu_log(g, gpu_dbg_intr, | ||
854 | "ecc error row address: 0x%x", | ||
855 | gr_gpc0_gpccs_falcon_ecc_address_row_address_v(ecc_addr)); | ||
856 | |||
857 | nvgpu_log(g, gpu_dbg_intr, | ||
858 | "ecc error count corrected: %d, uncorrected %d", | ||
859 | g->ecc.gr.t19x.gpccs_corrected_err_count.counters[gpc], | ||
860 | g->ecc.gr.t19x.gpccs_uncorrected_err_count.counters[gpc]); | ||
861 | |||
862 | return ret; | ||
863 | } | ||
864 | |||
865 | int gr_gv11b_handle_gpc_gpcmmu_exception(struct gk20a *g, u32 gpc, | ||
866 | u32 gpc_exception) | ||
867 | { | ||
868 | if (gpc_exception & gr_gpc0_gpccs_gpc_exception_gpcmmu_m()) | ||
869 | return gr_gv11b_handle_gpcmmu_ecc_exception(g, gpc, | ||
870 | gpc_exception); | ||
871 | return 0; | ||
872 | } | ||
873 | |||
874 | int gr_gv11b_handle_gpc_gpccs_exception(struct gk20a *g, u32 gpc, | ||
875 | u32 gpc_exception) | ||
876 | { | ||
877 | if (gpc_exception & gr_gpc0_gpccs_gpc_exception_gpccs_m()) | ||
878 | return gr_gv11b_handle_gpccs_ecc_exception(g, gpc, | ||
879 | gpc_exception); | ||
880 | |||
881 | return 0; | ||
882 | } | ||
883 | |||
884 | void gr_gv11b_enable_gpc_exceptions(struct gk20a *g) | ||
885 | { | ||
886 | struct gr_gk20a *gr = &g->gr; | ||
887 | u32 tpc_mask; | ||
888 | |||
889 | gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), | ||
890 | gr_gpcs_tpcs_tpccs_tpc_exception_en_sm_enabled_f() | | ||
891 | gr_gpcs_tpcs_tpccs_tpc_exception_en_mpc_enabled_f()); | ||
892 | |||
893 | tpc_mask = | ||
894 | gr_gpcs_gpccs_gpc_exception_en_tpc_f((1 << gr->tpc_count) - 1); | ||
895 | |||
896 | gk20a_writel(g, gr_gpcs_gpccs_gpc_exception_en_r(), | ||
897 | (tpc_mask | gr_gpcs_gpccs_gpc_exception_en_gcc_f(1) | | ||
898 | gr_gpcs_gpccs_gpc_exception_en_gpccs_f(1) | | ||
899 | gr_gpcs_gpccs_gpc_exception_en_gpcmmu_f(1))); | ||
900 | } | ||
901 | |||
902 | int gr_gv11b_handle_tex_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
903 | bool *post_event) | ||
904 | { | ||
905 | return 0; | ||
906 | } | ||
907 | |||
908 | int gr_gv11b_zbc_s_query_table(struct gk20a *g, struct gr_gk20a *gr, | ||
909 | struct zbc_query_params *query_params) | ||
910 | { | ||
911 | u32 index = query_params->index_size; | ||
912 | |||
913 | if (index >= GK20A_ZBC_TABLE_SIZE) { | ||
914 | nvgpu_err(g, "invalid zbc stencil table index"); | ||
915 | return -EINVAL; | ||
916 | } | ||
917 | query_params->depth = gr->zbc_s_tbl[index].stencil; | ||
918 | query_params->format = gr->zbc_s_tbl[index].format; | ||
919 | query_params->ref_cnt = gr->zbc_s_tbl[index].ref_cnt; | ||
920 | |||
921 | return 0; | ||
922 | } | ||
923 | |||
924 | bool gr_gv11b_add_zbc_type_s(struct gk20a *g, struct gr_gk20a *gr, | ||
925 | struct zbc_entry *zbc_val, int *ret_val) | ||
926 | { | ||
927 | struct zbc_s_table *s_tbl; | ||
928 | u32 i; | ||
929 | bool added = false; | ||
930 | |||
931 | *ret_val = -ENOMEM; | ||
932 | |||
933 | /* search existing tables */ | ||
934 | for (i = 0; i < gr->max_used_s_index; i++) { | ||
935 | |||
936 | s_tbl = &gr->zbc_s_tbl[i]; | ||
937 | |||
938 | if (s_tbl->ref_cnt && | ||
939 | s_tbl->stencil == zbc_val->depth && | ||
940 | s_tbl->format == zbc_val->format) { | ||
941 | added = true; | ||
942 | s_tbl->ref_cnt++; | ||
943 | *ret_val = 0; | ||
944 | break; | ||
945 | } | ||
946 | } | ||
947 | /* add new table */ | ||
948 | if (!added && | ||
949 | gr->max_used_s_index < GK20A_ZBC_TABLE_SIZE) { | ||
950 | |||
951 | s_tbl = &gr->zbc_s_tbl[gr->max_used_s_index]; | ||
952 | WARN_ON(s_tbl->ref_cnt != 0); | ||
953 | |||
954 | *ret_val = g->ops.gr.add_zbc_s(g, gr, | ||
955 | zbc_val, gr->max_used_s_index); | ||
956 | |||
957 | if (!(*ret_val)) | ||
958 | gr->max_used_s_index++; | ||
959 | } | ||
960 | return added; | ||
961 | } | ||
962 | |||
963 | int gr_gv11b_add_zbc_stencil(struct gk20a *g, struct gr_gk20a *gr, | ||
964 | struct zbc_entry *stencil_val, u32 index) | ||
965 | { | ||
966 | u32 zbc_s; | ||
967 | |||
968 | /* update l2 table */ | ||
969 | g->ops.ltc.set_zbc_s_entry(g, stencil_val, index); | ||
970 | |||
971 | /* update local copy */ | ||
972 | gr->zbc_s_tbl[index].stencil = stencil_val->depth; | ||
973 | gr->zbc_s_tbl[index].format = stencil_val->format; | ||
974 | gr->zbc_s_tbl[index].ref_cnt++; | ||
975 | |||
976 | gk20a_writel(g, gr_gpcs_swdx_dss_zbc_s_r(index), stencil_val->depth); | ||
977 | zbc_s = gk20a_readl(g, gr_gpcs_swdx_dss_zbc_s_01_to_04_format_r() + | ||
978 | (index & ~3)); | ||
979 | zbc_s &= ~(0x7f << (index % 4) * 7); | ||
980 | zbc_s |= stencil_val->format << (index % 4) * 7; | ||
981 | gk20a_writel(g, gr_gpcs_swdx_dss_zbc_s_01_to_04_format_r() + | ||
982 | (index & ~3), zbc_s); | ||
983 | |||
984 | return 0; | ||
985 | } | ||
986 | |||
987 | int gr_gv11b_load_stencil_default_tbl(struct gk20a *g, | ||
988 | struct gr_gk20a *gr) | ||
989 | { | ||
990 | struct zbc_entry zbc_val; | ||
991 | u32 err; | ||
992 | |||
993 | /* load default stencil table */ | ||
994 | zbc_val.type = GV11B_ZBC_TYPE_STENCIL; | ||
995 | |||
996 | zbc_val.depth = 0x0; | ||
997 | zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8; | ||
998 | err = gr_gk20a_add_zbc(g, gr, &zbc_val); | ||
999 | |||
1000 | zbc_val.depth = 0x1; | ||
1001 | zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8; | ||
1002 | err |= gr_gk20a_add_zbc(g, gr, &zbc_val); | ||
1003 | |||
1004 | zbc_val.depth = 0xff; | ||
1005 | zbc_val.format = ZBC_STENCIL_CLEAR_FMT_U8; | ||
1006 | err |= gr_gk20a_add_zbc(g, gr, &zbc_val); | ||
1007 | |||
1008 | if (!err) { | ||
1009 | gr->max_default_s_index = 3; | ||
1010 | } else { | ||
1011 | nvgpu_err(g, "fail to load default zbc stencil table"); | ||
1012 | return err; | ||
1013 | } | ||
1014 | |||
1015 | return 0; | ||
1016 | } | ||
1017 | |||
1018 | int gr_gv11b_load_stencil_tbl(struct gk20a *g, struct gr_gk20a *gr) | ||
1019 | { | ||
1020 | int ret; | ||
1021 | u32 i; | ||
1022 | |||
1023 | for (i = 0; i < gr->max_used_s_index; i++) { | ||
1024 | struct zbc_s_table *s_tbl = &gr->zbc_s_tbl[i]; | ||
1025 | struct zbc_entry zbc_val; | ||
1026 | |||
1027 | zbc_val.type = GV11B_ZBC_TYPE_STENCIL; | ||
1028 | zbc_val.depth = s_tbl->stencil; | ||
1029 | zbc_val.format = s_tbl->format; | ||
1030 | |||
1031 | ret = g->ops.gr.add_zbc_s(g, gr, &zbc_val, i); | ||
1032 | if (ret) | ||
1033 | return ret; | ||
1034 | } | ||
1035 | return 0; | ||
1036 | } | ||
1037 | |||
1038 | u32 gr_gv11b_pagepool_default_size(struct gk20a *g) | ||
1039 | { | ||
1040 | return gr_scc_pagepool_total_pages_hwmax_value_v(); | ||
1041 | } | ||
1042 | |||
1043 | int gr_gv11b_calc_global_ctx_buffer_size(struct gk20a *g) | ||
1044 | { | ||
1045 | struct gr_gk20a *gr = &g->gr; | ||
1046 | int size; | ||
1047 | |||
1048 | gr->attrib_cb_size = gr->attrib_cb_default_size; | ||
1049 | gr->alpha_cb_size = gr->alpha_cb_default_size; | ||
1050 | |||
1051 | gr->attrib_cb_size = min(gr->attrib_cb_size, | ||
1052 | gr_gpc0_ppc0_cbm_beta_cb_size_v_f(~0) / g->gr.tpc_count); | ||
1053 | gr->alpha_cb_size = min(gr->alpha_cb_size, | ||
1054 | gr_gpc0_ppc0_cbm_alpha_cb_size_v_f(~0) / g->gr.tpc_count); | ||
1055 | |||
1056 | size = gr->attrib_cb_size * | ||
1057 | gr_gpc0_ppc0_cbm_beta_cb_size_v_granularity_v() * | ||
1058 | gr->max_tpc_count; | ||
1059 | |||
1060 | size += gr->alpha_cb_size * | ||
1061 | gr_gpc0_ppc0_cbm_alpha_cb_size_v_granularity_v() * | ||
1062 | gr->max_tpc_count; | ||
1063 | |||
1064 | size = ALIGN(size, 128); | ||
1065 | |||
1066 | return size; | ||
1067 | } | ||
1068 | |||
1069 | static void gr_gv11b_set_go_idle_timeout(struct gk20a *g, u32 data) | ||
1070 | { | ||
1071 | gk20a_writel(g, gr_fe_go_idle_timeout_r(), data); | ||
1072 | } | ||
1073 | |||
1074 | static void gr_gv11b_set_coalesce_buffer_size(struct gk20a *g, u32 data) | ||
1075 | { | ||
1076 | u32 val; | ||
1077 | |||
1078 | gk20a_dbg_fn(""); | ||
1079 | |||
1080 | val = gk20a_readl(g, gr_gpcs_tc_debug0_r()); | ||
1081 | val = set_field(val, gr_gpcs_tc_debug0_limit_coalesce_buffer_size_m(), | ||
1082 | gr_gpcs_tc_debug0_limit_coalesce_buffer_size_f(data)); | ||
1083 | gk20a_writel(g, gr_gpcs_tc_debug0_r(), val); | ||
1084 | |||
1085 | gk20a_dbg_fn("done"); | ||
1086 | } | ||
1087 | |||
1088 | static void gr_gv11b_set_tex_in_dbg(struct gk20a *g, u32 data) | ||
1089 | { | ||
1090 | u32 val; | ||
1091 | bool flag; | ||
1092 | |||
1093 | gk20a_dbg_fn(""); | ||
1094 | |||
1095 | val = gk20a_readl(g, gr_gpcs_tpcs_tex_in_dbg_r()); | ||
1096 | flag = (data & NVC397_SET_TEX_IN_DBG_TSL1_RVCH_INVALIDATE) ? 1 : 0; | ||
1097 | val = set_field(val, gr_gpcs_tpcs_tex_in_dbg_tsl1_rvch_invalidate_m(), | ||
1098 | gr_gpcs_tpcs_tex_in_dbg_tsl1_rvch_invalidate_f(flag)); | ||
1099 | gk20a_writel(g, gr_gpcs_tpcs_tex_in_dbg_r(), val); | ||
1100 | |||
1101 | val = gk20a_readl(g, gr_gpcs_tpcs_sm_l1tag_ctrl_r()); | ||
1102 | flag = (data & | ||
1103 | NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_LD) ? 1 : 0; | ||
1104 | val = set_field(val, gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_ld_m(), | ||
1105 | gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_ld_f(flag)); | ||
1106 | flag = (data & | ||
1107 | NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_ST) ? 1 : 0; | ||
1108 | val = set_field(val, gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_st_m(), | ||
1109 | gr_gpcs_tpcs_sm_l1tag_ctrl_cache_surface_st_f(flag)); | ||
1110 | gk20a_writel(g, gr_gpcs_tpcs_sm_l1tag_ctrl_r(), val); | ||
1111 | } | ||
1112 | |||
1113 | static void gr_gv11b_set_skedcheck(struct gk20a *g, u32 data) | ||
1114 | { | ||
1115 | u32 reg_val; | ||
1116 | |||
1117 | reg_val = gk20a_readl(g, gr_sked_hww_esr_en_r()); | ||
1118 | |||
1119 | if ((data & NVC397_SET_SKEDCHECK_18_MASK) == | ||
1120 | NVC397_SET_SKEDCHECK_18_DISABLE) { | ||
1121 | reg_val = set_field(reg_val, | ||
1122 | gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_m(), | ||
1123 | gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_disabled_f() | ||
1124 | ); | ||
1125 | } else if ((data & NVC397_SET_SKEDCHECK_18_MASK) == | ||
1126 | NVC397_SET_SKEDCHECK_18_ENABLE) { | ||
1127 | reg_val = set_field(reg_val, | ||
1128 | gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_m(), | ||
1129 | gr_sked_hww_esr_en_skedcheck18_l1_config_too_small_enabled_f() | ||
1130 | ); | ||
1131 | } | ||
1132 | nvgpu_log_info(g, "sked_hww_esr_en = 0x%x", reg_val); | ||
1133 | gk20a_writel(g, gr_sked_hww_esr_en_r(), reg_val); | ||
1134 | |||
1135 | } | ||
1136 | |||
1137 | static void gv11b_gr_set_shader_exceptions(struct gk20a *g, u32 data) | ||
1138 | { | ||
1139 | gk20a_dbg_fn(""); | ||
1140 | |||
1141 | if (data == NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE) { | ||
1142 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r(), | ||
1143 | 0); | ||
1144 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r(), | ||
1145 | 0); | ||
1146 | } else { | ||
1147 | g->ops.gr.set_hww_esr_report_mask(g); | ||
1148 | } | ||
1149 | } | ||
1150 | |||
1151 | int gr_gv11b_handle_sw_method(struct gk20a *g, u32 addr, | ||
1152 | u32 class_num, u32 offset, u32 data) | ||
1153 | { | ||
1154 | gk20a_dbg_fn(""); | ||
1155 | |||
1156 | if (class_num == VOLTA_COMPUTE_A) { | ||
1157 | switch (offset << 2) { | ||
1158 | case NVC0C0_SET_SHADER_EXCEPTIONS: | ||
1159 | gv11b_gr_set_shader_exceptions(g, data); | ||
1160 | break; | ||
1161 | case NVC3C0_SET_SKEDCHECK: | ||
1162 | gr_gv11b_set_skedcheck(g, data); | ||
1163 | break; | ||
1164 | default: | ||
1165 | goto fail; | ||
1166 | } | ||
1167 | } | ||
1168 | |||
1169 | if (class_num == VOLTA_A) { | ||
1170 | switch (offset << 2) { | ||
1171 | case NVC397_SET_SHADER_EXCEPTIONS: | ||
1172 | gv11b_gr_set_shader_exceptions(g, data); | ||
1173 | break; | ||
1174 | case NVC397_SET_CIRCULAR_BUFFER_SIZE: | ||
1175 | g->ops.gr.set_circular_buffer_size(g, data); | ||
1176 | break; | ||
1177 | case NVC397_SET_ALPHA_CIRCULAR_BUFFER_SIZE: | ||
1178 | g->ops.gr.set_alpha_circular_buffer_size(g, data); | ||
1179 | break; | ||
1180 | case NVC397_SET_GO_IDLE_TIMEOUT: | ||
1181 | gr_gv11b_set_go_idle_timeout(g, data); | ||
1182 | break; | ||
1183 | case NVC097_SET_COALESCE_BUFFER_SIZE: | ||
1184 | gr_gv11b_set_coalesce_buffer_size(g, data); | ||
1185 | break; | ||
1186 | case NVC397_SET_TEX_IN_DBG: | ||
1187 | gr_gv11b_set_tex_in_dbg(g, data); | ||
1188 | break; | ||
1189 | case NVC397_SET_SKEDCHECK: | ||
1190 | gr_gv11b_set_skedcheck(g, data); | ||
1191 | break; | ||
1192 | case NVC397_SET_BES_CROP_DEBUG3: | ||
1193 | g->ops.gr.set_bes_crop_debug3(g, data); | ||
1194 | break; | ||
1195 | default: | ||
1196 | goto fail; | ||
1197 | } | ||
1198 | } | ||
1199 | return 0; | ||
1200 | |||
1201 | fail: | ||
1202 | return -EINVAL; | ||
1203 | } | ||
1204 | |||
1205 | void gr_gv11b_bundle_cb_defaults(struct gk20a *g) | ||
1206 | { | ||
1207 | struct gr_gk20a *gr = &g->gr; | ||
1208 | |||
1209 | gr->bundle_cb_default_size = | ||
1210 | gr_scc_bundle_cb_size_div_256b__prod_v(); | ||
1211 | gr->min_gpm_fifo_depth = | ||
1212 | gr_pd_ab_dist_cfg2_state_limit_min_gpm_fifo_depths_v(); | ||
1213 | gr->bundle_cb_token_limit = | ||
1214 | gr_pd_ab_dist_cfg2_token_limit_init_v(); | ||
1215 | } | ||
1216 | |||
1217 | void gr_gv11b_cb_size_default(struct gk20a *g) | ||
1218 | { | ||
1219 | struct gr_gk20a *gr = &g->gr; | ||
1220 | |||
1221 | if (!gr->attrib_cb_default_size) | ||
1222 | gr->attrib_cb_default_size = | ||
1223 | gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v(); | ||
1224 | gr->alpha_cb_default_size = | ||
1225 | gr_gpc0_ppc0_cbm_alpha_cb_size_v_default_v(); | ||
1226 | } | ||
1227 | |||
1228 | void gr_gv11b_set_alpha_circular_buffer_size(struct gk20a *g, u32 data) | ||
1229 | { | ||
1230 | struct gr_gk20a *gr = &g->gr; | ||
1231 | u32 gpc_index, ppc_index, stride, val; | ||
1232 | u32 pd_ab_max_output; | ||
1233 | u32 alpha_cb_size = data * 4; | ||
1234 | |||
1235 | gk20a_dbg_fn(""); | ||
1236 | |||
1237 | if (alpha_cb_size > gr->alpha_cb_size) | ||
1238 | alpha_cb_size = gr->alpha_cb_size; | ||
1239 | |||
1240 | gk20a_writel(g, gr_ds_tga_constraintlogic_alpha_r(), | ||
1241 | (gk20a_readl(g, gr_ds_tga_constraintlogic_alpha_r()) & | ||
1242 | ~gr_ds_tga_constraintlogic_alpha_cbsize_f(~0)) | | ||
1243 | gr_ds_tga_constraintlogic_alpha_cbsize_f(alpha_cb_size)); | ||
1244 | |||
1245 | pd_ab_max_output = alpha_cb_size * | ||
1246 | gr_gpc0_ppc0_cbm_alpha_cb_size_v_granularity_v() / | ||
1247 | gr_pd_ab_dist_cfg1_max_output_granularity_v(); | ||
1248 | |||
1249 | gk20a_writel(g, gr_pd_ab_dist_cfg1_r(), | ||
1250 | gr_pd_ab_dist_cfg1_max_output_f(pd_ab_max_output) | | ||
1251 | gr_pd_ab_dist_cfg1_max_batches_init_f()); | ||
1252 | |||
1253 | for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { | ||
1254 | stride = proj_gpc_stride_v() * gpc_index; | ||
1255 | |||
1256 | for (ppc_index = 0; ppc_index < gr->gpc_ppc_count[gpc_index]; | ||
1257 | ppc_index++) { | ||
1258 | |||
1259 | val = gk20a_readl(g, gr_gpc0_ppc0_cbm_alpha_cb_size_r() + | ||
1260 | stride + | ||
1261 | proj_ppc_in_gpc_stride_v() * ppc_index); | ||
1262 | |||
1263 | val = set_field(val, gr_gpc0_ppc0_cbm_alpha_cb_size_v_m(), | ||
1264 | gr_gpc0_ppc0_cbm_alpha_cb_size_v_f(alpha_cb_size * | ||
1265 | gr->pes_tpc_count[ppc_index][gpc_index])); | ||
1266 | |||
1267 | gk20a_writel(g, gr_gpc0_ppc0_cbm_alpha_cb_size_r() + | ||
1268 | stride + | ||
1269 | proj_ppc_in_gpc_stride_v() * ppc_index, val); | ||
1270 | } | ||
1271 | } | ||
1272 | } | ||
1273 | |||
1274 | void gr_gv11b_set_circular_buffer_size(struct gk20a *g, u32 data) | ||
1275 | { | ||
1276 | struct gr_gk20a *gr = &g->gr; | ||
1277 | u32 gpc_index, ppc_index, stride, val; | ||
1278 | u32 cb_size_steady = data * 4, cb_size; | ||
1279 | |||
1280 | gk20a_dbg_fn(""); | ||
1281 | |||
1282 | if (cb_size_steady > gr->attrib_cb_size) | ||
1283 | cb_size_steady = gr->attrib_cb_size; | ||
1284 | if (gk20a_readl(g, gr_gpc0_ppc0_cbm_beta_cb_size_r()) != | ||
1285 | gk20a_readl(g, | ||
1286 | gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_r())) { | ||
1287 | cb_size = cb_size_steady + | ||
1288 | (gr_gpc0_ppc0_cbm_beta_cb_size_v_gfxp_v() - | ||
1289 | gr_gpc0_ppc0_cbm_beta_cb_size_v_default_v()); | ||
1290 | } else { | ||
1291 | cb_size = cb_size_steady; | ||
1292 | } | ||
1293 | |||
1294 | gk20a_writel(g, gr_ds_tga_constraintlogic_beta_r(), | ||
1295 | (gk20a_readl(g, gr_ds_tga_constraintlogic_beta_r()) & | ||
1296 | ~gr_ds_tga_constraintlogic_beta_cbsize_f(~0)) | | ||
1297 | gr_ds_tga_constraintlogic_beta_cbsize_f(cb_size_steady)); | ||
1298 | |||
1299 | for (gpc_index = 0; gpc_index < gr->gpc_count; gpc_index++) { | ||
1300 | stride = proj_gpc_stride_v() * gpc_index; | ||
1301 | |||
1302 | for (ppc_index = 0; ppc_index < gr->gpc_ppc_count[gpc_index]; | ||
1303 | ppc_index++) { | ||
1304 | |||
1305 | val = gk20a_readl(g, gr_gpc0_ppc0_cbm_beta_cb_size_r() + | ||
1306 | stride + | ||
1307 | proj_ppc_in_gpc_stride_v() * ppc_index); | ||
1308 | |||
1309 | val = set_field(val, | ||
1310 | gr_gpc0_ppc0_cbm_beta_cb_size_v_m(), | ||
1311 | gr_gpc0_ppc0_cbm_beta_cb_size_v_f(cb_size * | ||
1312 | gr->pes_tpc_count[ppc_index][gpc_index])); | ||
1313 | |||
1314 | gk20a_writel(g, gr_gpc0_ppc0_cbm_beta_cb_size_r() + | ||
1315 | stride + | ||
1316 | proj_ppc_in_gpc_stride_v() * ppc_index, val); | ||
1317 | |||
1318 | gk20a_writel(g, proj_ppc_in_gpc_stride_v() * ppc_index + | ||
1319 | gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_r() + | ||
1320 | stride, | ||
1321 | gr_gpc0_ppc0_cbm_beta_steady_state_cb_size_v_f( | ||
1322 | cb_size_steady)); | ||
1323 | |||
1324 | val = gk20a_readl(g, gr_gpcs_swdx_tc_beta_cb_size_r( | ||
1325 | ppc_index + gpc_index)); | ||
1326 | |||
1327 | val = set_field(val, | ||
1328 | gr_gpcs_swdx_tc_beta_cb_size_v_m(), | ||
1329 | gr_gpcs_swdx_tc_beta_cb_size_v_f( | ||
1330 | cb_size_steady * | ||
1331 | gr->gpc_ppc_count[gpc_index])); | ||
1332 | |||
1333 | gk20a_writel(g, gr_gpcs_swdx_tc_beta_cb_size_r( | ||
1334 | ppc_index + gpc_index), val); | ||
1335 | } | ||
1336 | } | ||
1337 | } | ||
1338 | |||
1339 | int gr_gv11b_alloc_buffer(struct vm_gk20a *vm, size_t size, | ||
1340 | struct nvgpu_mem *mem) | ||
1341 | { | ||
1342 | int err; | ||
1343 | |||
1344 | gk20a_dbg_fn(""); | ||
1345 | |||
1346 | err = nvgpu_dma_alloc_sys(vm->mm->g, size, mem); | ||
1347 | if (err) | ||
1348 | return err; | ||
1349 | |||
1350 | mem->gpu_va = nvgpu_gmmu_map(vm, | ||
1351 | mem, | ||
1352 | size, | ||
1353 | NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE, | ||
1354 | gk20a_mem_flag_none, | ||
1355 | false, | ||
1356 | mem->aperture); | ||
1357 | |||
1358 | if (!mem->gpu_va) { | ||
1359 | err = -ENOMEM; | ||
1360 | goto fail_free; | ||
1361 | } | ||
1362 | |||
1363 | return 0; | ||
1364 | |||
1365 | fail_free: | ||
1366 | nvgpu_dma_free(vm->mm->g, mem); | ||
1367 | return err; | ||
1368 | } | ||
1369 | |||
1370 | static void gr_gv11b_dump_gr_per_sm_regs(struct gk20a *g, | ||
1371 | struct gk20a_debug_output *o, | ||
1372 | u32 gpc, u32 tpc, u32 sm, u32 offset) | ||
1373 | { | ||
1374 | |||
1375 | gk20a_debug_output(o, | ||
1376 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_WARP_ESR: 0x%x\n", | ||
1377 | gpc, tpc, sm, gk20a_readl(g, | ||
1378 | gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset)); | ||
1379 | |||
1380 | gk20a_debug_output(o, | ||
1381 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_WARP_ESR_REPORT_MASK: 0x%x\n", | ||
1382 | gpc, tpc, sm, gk20a_readl(g, | ||
1383 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset)); | ||
1384 | |||
1385 | gk20a_debug_output(o, | ||
1386 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_GLOBAL_ESR: 0x%x\n", | ||
1387 | gpc, tpc, sm, gk20a_readl(g, | ||
1388 | gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset)); | ||
1389 | |||
1390 | gk20a_debug_output(o, | ||
1391 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_HWW_GLOBAL_ESR_REPORT_MASK: 0x%x\n", | ||
1392 | gpc, tpc, sm, gk20a_readl(g, | ||
1393 | gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset)); | ||
1394 | |||
1395 | gk20a_debug_output(o, | ||
1396 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_DBGR_CONTROL0: 0x%x\n", | ||
1397 | gpc, tpc, sm, gk20a_readl(g, | ||
1398 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset)); | ||
1399 | |||
1400 | gk20a_debug_output(o, | ||
1401 | "NV_PGRAPH_PRI_GPC%d_TPC%d_SM%d_DBGR_STATUS0: 0x%x\n", | ||
1402 | gpc, tpc, sm, gk20a_readl(g, | ||
1403 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset)); | ||
1404 | } | ||
1405 | |||
1406 | static int gr_gv11b_dump_gr_sm_regs(struct gk20a *g, | ||
1407 | struct gk20a_debug_output *o) | ||
1408 | { | ||
1409 | u32 gpc, tpc, sm, sm_per_tpc; | ||
1410 | u32 gpc_offset, tpc_offset, offset; | ||
1411 | |||
1412 | gk20a_debug_output(o, | ||
1413 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR_REPORT_MASK: 0x%x\n", | ||
1414 | gk20a_readl(g, | ||
1415 | gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r())); | ||
1416 | gk20a_debug_output(o, | ||
1417 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_WARP_ESR_REPORT_MASK: 0x%x\n", | ||
1418 | gk20a_readl(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r())); | ||
1419 | gk20a_debug_output(o, | ||
1420 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_HWW_GLOBAL_ESR: 0x%x\n", | ||
1421 | gk20a_readl(g, gr_gpcs_tpcs_sms_hww_global_esr_r())); | ||
1422 | gk20a_debug_output(o, | ||
1423 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_CONTROL0: 0x%x\n", | ||
1424 | gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_control0_r())); | ||
1425 | gk20a_debug_output(o, | ||
1426 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_STATUS0: 0x%x\n", | ||
1427 | gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_status0_r())); | ||
1428 | gk20a_debug_output(o, | ||
1429 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_0: 0x%x\n", | ||
1430 | gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_0_r())); | ||
1431 | gk20a_debug_output(o, | ||
1432 | "NV_PGRAPH_PRI_GPCS_TPCS_SMS_DBGR_BPT_PAUSE_MASK_1: 0x%x\n", | ||
1433 | gk20a_readl(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_1_r())); | ||
1434 | |||
1435 | sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); | ||
1436 | for (gpc = 0; gpc < g->gr.gpc_count; gpc++) { | ||
1437 | gpc_offset = gk20a_gr_gpc_offset(g, gpc); | ||
1438 | |||
1439 | for (tpc = 0; tpc < g->gr.tpc_count; tpc++) { | ||
1440 | tpc_offset = gk20a_gr_tpc_offset(g, tpc); | ||
1441 | |||
1442 | for (sm = 0; sm < sm_per_tpc; sm++) { | ||
1443 | offset = gpc_offset + tpc_offset + | ||
1444 | gv11b_gr_sm_offset(g, sm); | ||
1445 | |||
1446 | gr_gv11b_dump_gr_per_sm_regs(g, o, | ||
1447 | gpc, tpc, sm, offset); | ||
1448 | } | ||
1449 | } | ||
1450 | } | ||
1451 | |||
1452 | return 0; | ||
1453 | } | ||
1454 | |||
1455 | int gr_gv11b_dump_gr_status_regs(struct gk20a *g, | ||
1456 | struct gk20a_debug_output *o) | ||
1457 | { | ||
1458 | struct gr_gk20a *gr = &g->gr; | ||
1459 | u32 gr_engine_id; | ||
1460 | |||
1461 | gr_engine_id = gk20a_fifo_get_gr_engine_id(g); | ||
1462 | |||
1463 | gk20a_debug_output(o, "NV_PGRAPH_STATUS: 0x%x\n", | ||
1464 | gk20a_readl(g, gr_status_r())); | ||
1465 | gk20a_debug_output(o, "NV_PGRAPH_STATUS1: 0x%x\n", | ||
1466 | gk20a_readl(g, gr_status_1_r())); | ||
1467 | gk20a_debug_output(o, "NV_PGRAPH_STATUS2: 0x%x\n", | ||
1468 | gk20a_readl(g, gr_status_2_r())); | ||
1469 | gk20a_debug_output(o, "NV_PGRAPH_ENGINE_STATUS: 0x%x\n", | ||
1470 | gk20a_readl(g, gr_engine_status_r())); | ||
1471 | gk20a_debug_output(o, "NV_PGRAPH_GRFIFO_STATUS : 0x%x\n", | ||
1472 | gk20a_readl(g, gr_gpfifo_status_r())); | ||
1473 | gk20a_debug_output(o, "NV_PGRAPH_GRFIFO_CONTROL : 0x%x\n", | ||
1474 | gk20a_readl(g, gr_gpfifo_ctl_r())); | ||
1475 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x%x\n", | ||
1476 | gk20a_readl(g, gr_fecs_host_int_status_r())); | ||
1477 | gk20a_debug_output(o, "NV_PGRAPH_EXCEPTION : 0x%x\n", | ||
1478 | gk20a_readl(g, gr_exception_r())); | ||
1479 | gk20a_debug_output(o, "NV_PGRAPH_FECS_INTR : 0x%x\n", | ||
1480 | gk20a_readl(g, gr_fecs_intr_r())); | ||
1481 | gk20a_debug_output(o, "NV_PFIFO_ENGINE_STATUS(GR) : 0x%x\n", | ||
1482 | gk20a_readl(g, fifo_engine_status_r(gr_engine_id))); | ||
1483 | gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY0: 0x%x\n", | ||
1484 | gk20a_readl(g, gr_activity_0_r())); | ||
1485 | gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY1: 0x%x\n", | ||
1486 | gk20a_readl(g, gr_activity_1_r())); | ||
1487 | gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY2: 0x%x\n", | ||
1488 | gk20a_readl(g, gr_activity_2_r())); | ||
1489 | gk20a_debug_output(o, "NV_PGRAPH_ACTIVITY4: 0x%x\n", | ||
1490 | gk20a_readl(g, gr_activity_4_r())); | ||
1491 | gk20a_debug_output(o, "NV_PGRAPH_PRI_SKED_ACTIVITY: 0x%x\n", | ||
1492 | gk20a_readl(g, gr_pri_sked_activity_r())); | ||
1493 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY0: 0x%x\n", | ||
1494 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity0_r())); | ||
1495 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY1: 0x%x\n", | ||
1496 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity1_r())); | ||
1497 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY2: 0x%x\n", | ||
1498 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity2_r())); | ||
1499 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_ACTIVITY3: 0x%x\n", | ||
1500 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_activity3_r())); | ||
1501 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1502 | gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_activity_0_r())); | ||
1503 | if (gr->gpc_tpc_count[0] == 2) | ||
1504 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC1_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1505 | gk20a_readl(g, gr_pri_gpc0_tpc1_tpccs_tpc_activity_0_r())); | ||
1506 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPCS_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1507 | gk20a_readl(g, gr_pri_gpc0_tpcs_tpccs_tpc_activity_0_r())); | ||
1508 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY0: 0x%x\n", | ||
1509 | gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_0_r())); | ||
1510 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY1: 0x%x\n", | ||
1511 | gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_1_r())); | ||
1512 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY2: 0x%x\n", | ||
1513 | gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_2_r())); | ||
1514 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_GPCCS_GPC_ACTIVITY3: 0x%x\n", | ||
1515 | gk20a_readl(g, gr_pri_gpcs_gpccs_gpc_activity_3_r())); | ||
1516 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPC0_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1517 | gk20a_readl(g, gr_pri_gpcs_tpc0_tpccs_tpc_activity_0_r())); | ||
1518 | if (gr->gpc_tpc_count[0] == 2) | ||
1519 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPC1_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1520 | gk20a_readl(g, gr_pri_gpcs_tpc1_tpccs_tpc_activity_0_r())); | ||
1521 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPCS_TPCS_TPCCS_TPC_ACTIVITY0: 0x%x\n", | ||
1522 | gk20a_readl(g, gr_pri_gpcs_tpcs_tpccs_tpc_activity_0_r())); | ||
1523 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_ACTIVITY0: 0x%x\n", | ||
1524 | gk20a_readl(g, gr_pri_be0_becs_be_activity0_r())); | ||
1525 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE1_BECS_BE_ACTIVITY0: 0x%x\n", | ||
1526 | gk20a_readl(g, gr_pri_be1_becs_be_activity0_r())); | ||
1527 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_BECS_BE_ACTIVITY0: 0x%x\n", | ||
1528 | gk20a_readl(g, gr_pri_bes_becs_be_activity0_r())); | ||
1529 | gk20a_debug_output(o, "NV_PGRAPH_PRI_DS_MPIPE_STATUS: 0x%x\n", | ||
1530 | gk20a_readl(g, gr_pri_ds_mpipe_status_r())); | ||
1531 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_GO_IDLE_TIMEOUT : 0x%x\n", | ||
1532 | gk20a_readl(g, gr_fe_go_idle_timeout_r())); | ||
1533 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_GO_IDLE_INFO : 0x%x\n", | ||
1534 | gk20a_readl(g, gr_pri_fe_go_idle_info_r())); | ||
1535 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TEX_M_TEX_SUBUNITS_STATUS: 0x%x\n", | ||
1536 | gk20a_readl(g, gr_pri_gpc0_tpc0_tex_m_tex_subunits_status_r())); | ||
1537 | gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_FS: 0x%x\n", | ||
1538 | gk20a_readl(g, gr_cwd_fs_r())); | ||
1539 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FE_TPC_FS(0): 0x%x\n", | ||
1540 | gk20a_readl(g, gr_fe_tpc_fs_r(0))); | ||
1541 | gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_GPC_TPC_ID: 0x%x\n", | ||
1542 | gk20a_readl(g, gr_cwd_gpc_tpc_id_r(0))); | ||
1543 | gk20a_debug_output(o, "NV_PGRAPH_PRI_CWD_SM_ID(0): 0x%x\n", | ||
1544 | gk20a_readl(g, gr_cwd_sm_id_r(0))); | ||
1545 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_STATUS_FE_0: 0x%x\n", | ||
1546 | gk20a_readl(g, gr_fecs_ctxsw_status_fe_0_r())); | ||
1547 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_STATUS_1: 0x%x\n", | ||
1548 | gk20a_readl(g, gr_fecs_ctxsw_status_1_r())); | ||
1549 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_GPC_0: 0x%x\n", | ||
1550 | gk20a_readl(g, gr_gpc0_gpccs_ctxsw_status_gpc_0_r())); | ||
1551 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_STATUS_1: 0x%x\n", | ||
1552 | gk20a_readl(g, gr_gpc0_gpccs_ctxsw_status_1_r())); | ||
1553 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CTXSW_IDLESTATE : 0x%x\n", | ||
1554 | gk20a_readl(g, gr_fecs_ctxsw_idlestate_r())); | ||
1555 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_CTXSW_IDLESTATE : 0x%x\n", | ||
1556 | gk20a_readl(g, gr_gpc0_gpccs_ctxsw_idlestate_r())); | ||
1557 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_CURRENT_CTX : 0x%x\n", | ||
1558 | gk20a_readl(g, gr_fecs_current_ctx_r())); | ||
1559 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_NEW_CTX : 0x%x\n", | ||
1560 | gk20a_readl(g, gr_fecs_new_ctx_r())); | ||
1561 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_ENABLE : 0x%x\n", | ||
1562 | gk20a_readl(g, gr_fecs_host_int_enable_r())); | ||
1563 | gk20a_debug_output(o, "NV_PGRAPH_PRI_FECS_HOST_INT_STATUS : 0x%x\n", | ||
1564 | gk20a_readl(g, gr_fecs_host_int_status_r())); | ||
1565 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_CROP_STATUS1 : 0x%x\n", | ||
1566 | gk20a_readl(g, gr_pri_be0_crop_status1_r())); | ||
1567 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_CROP_STATUS1 : 0x%x\n", | ||
1568 | gk20a_readl(g, gr_pri_bes_crop_status1_r())); | ||
1569 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_ZROP_STATUS : 0x%x\n", | ||
1570 | gk20a_readl(g, gr_pri_be0_zrop_status_r())); | ||
1571 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_ZROP_STATUS2 : 0x%x\n", | ||
1572 | gk20a_readl(g, gr_pri_be0_zrop_status2_r())); | ||
1573 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_ZROP_STATUS : 0x%x\n", | ||
1574 | gk20a_readl(g, gr_pri_bes_zrop_status_r())); | ||
1575 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BES_ZROP_STATUS2 : 0x%x\n", | ||
1576 | gk20a_readl(g, gr_pri_bes_zrop_status2_r())); | ||
1577 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_EXCEPTION: 0x%x\n", | ||
1578 | gk20a_readl(g, gr_pri_be0_becs_be_exception_r())); | ||
1579 | gk20a_debug_output(o, "NV_PGRAPH_PRI_BE0_BECS_BE_EXCEPTION_EN: 0x%x\n", | ||
1580 | gk20a_readl(g, gr_pri_be0_becs_be_exception_en_r())); | ||
1581 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION: 0x%x\n", | ||
1582 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_exception_r())); | ||
1583 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_GPCCS_GPC_EXCEPTION_EN: 0x%x\n", | ||
1584 | gk20a_readl(g, gr_pri_gpc0_gpccs_gpc_exception_en_r())); | ||
1585 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION: 0x%x\n", | ||
1586 | gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_exception_r())); | ||
1587 | gk20a_debug_output(o, "NV_PGRAPH_PRI_GPC0_TPC0_TPCCS_TPC_EXCEPTION_EN: 0x%x\n", | ||
1588 | gk20a_readl(g, gr_pri_gpc0_tpc0_tpccs_tpc_exception_en_r())); | ||
1589 | |||
1590 | gr_gv11b_dump_gr_sm_regs(g, o); | ||
1591 | |||
1592 | return 0; | ||
1593 | } | ||
1594 | |||
1595 | static bool gr_activity_empty_or_preempted(u32 val) | ||
1596 | { | ||
1597 | while (val) { | ||
1598 | u32 v = val & 7; | ||
1599 | if (v != gr_activity_4_gpc0_empty_v() && | ||
1600 | v != gr_activity_4_gpc0_preempted_v()) | ||
1601 | return false; | ||
1602 | val >>= 3; | ||
1603 | } | ||
1604 | |||
1605 | return true; | ||
1606 | } | ||
1607 | |||
1608 | int gr_gv11b_wait_empty(struct gk20a *g, unsigned long duration_ms, | ||
1609 | u32 expect_delay) | ||
1610 | { | ||
1611 | u32 delay = expect_delay; | ||
1612 | bool gr_enabled; | ||
1613 | bool ctxsw_active; | ||
1614 | bool gr_busy; | ||
1615 | u32 gr_status; | ||
1616 | u32 activity0, activity1, activity2, activity4; | ||
1617 | struct nvgpu_timeout timeout; | ||
1618 | |||
1619 | gk20a_dbg_fn(""); | ||
1620 | |||
1621 | nvgpu_timeout_init(g, &timeout, duration_ms, NVGPU_TIMER_CPU_TIMER); | ||
1622 | |||
1623 | do { | ||
1624 | /* fmodel: host gets fifo_engine_status(gr) from gr | ||
1625 | only when gr_status is read */ | ||
1626 | gr_status = gk20a_readl(g, gr_status_r()); | ||
1627 | |||
1628 | gr_enabled = gk20a_readl(g, mc_enable_r()) & | ||
1629 | mc_enable_pgraph_enabled_f(); | ||
1630 | |||
1631 | ctxsw_active = gr_status & 1<<7; | ||
1632 | |||
1633 | activity0 = gk20a_readl(g, gr_activity_0_r()); | ||
1634 | activity1 = gk20a_readl(g, gr_activity_1_r()); | ||
1635 | activity2 = gk20a_readl(g, gr_activity_2_r()); | ||
1636 | activity4 = gk20a_readl(g, gr_activity_4_r()); | ||
1637 | |||
1638 | gr_busy = !(gr_activity_empty_or_preempted(activity0) && | ||
1639 | gr_activity_empty_or_preempted(activity1) && | ||
1640 | activity2 == 0 && | ||
1641 | gr_activity_empty_or_preempted(activity4)); | ||
1642 | |||
1643 | if (!gr_enabled || (!gr_busy && !ctxsw_active)) { | ||
1644 | gk20a_dbg_fn("done"); | ||
1645 | return 0; | ||
1646 | } | ||
1647 | |||
1648 | usleep_range(delay, delay * 2); | ||
1649 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
1650 | |||
1651 | } while (!nvgpu_timeout_expired(&timeout)); | ||
1652 | |||
1653 | nvgpu_err(g, | ||
1654 | "timeout, ctxsw busy : %d, gr busy : %d, %08x, %08x, %08x, %08x", | ||
1655 | ctxsw_active, gr_busy, activity0, activity1, activity2, activity4); | ||
1656 | |||
1657 | return -EAGAIN; | ||
1658 | } | ||
1659 | |||
1660 | void gr_gv11b_commit_global_attrib_cb(struct gk20a *g, | ||
1661 | struct channel_ctx_gk20a *ch_ctx, | ||
1662 | u64 addr, bool patch) | ||
1663 | { | ||
1664 | struct gr_ctx_desc *gr_ctx = ch_ctx->gr_ctx; | ||
1665 | int attrBufferSize; | ||
1666 | |||
1667 | if (gr_ctx->t18x.preempt_ctxsw_buffer.gpu_va) | ||
1668 | attrBufferSize = gr_ctx->t18x.betacb_ctxsw_buffer.size; | ||
1669 | else | ||
1670 | attrBufferSize = g->ops.gr.calc_global_ctx_buffer_size(g); | ||
1671 | |||
1672 | attrBufferSize /= gr_gpcs_tpcs_tex_rm_cb_1_size_div_128b_granularity_f(); | ||
1673 | |||
1674 | gr_gm20b_commit_global_attrib_cb(g, ch_ctx, addr, patch); | ||
1675 | |||
1676 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_r(), | ||
1677 | gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_v_f(addr) | | ||
1678 | gr_gpcs_tpcs_mpc_vtg_cb_global_base_addr_valid_true_f(), patch); | ||
1679 | |||
1680 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_tex_rm_cb_0_r(), | ||
1681 | gr_gpcs_tpcs_tex_rm_cb_0_base_addr_43_12_f(addr), patch); | ||
1682 | |||
1683 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_tex_rm_cb_1_r(), | ||
1684 | gr_gpcs_tpcs_tex_rm_cb_1_size_div_128b_f(attrBufferSize) | | ||
1685 | gr_gpcs_tpcs_tex_rm_cb_1_valid_true_f(), patch); | ||
1686 | } | ||
1687 | |||
1688 | void gr_gv11b_set_gpc_tpc_mask(struct gk20a *g, u32 gpc_index) | ||
1689 | { | ||
1690 | #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) | ||
1691 | tegra_fuse_writel(0x1, FUSE_FUSEBYPASS_0); | ||
1692 | tegra_fuse_writel(0x0, FUSE_WRITE_ACCESS_SW_0); | ||
1693 | #else | ||
1694 | tegra_fuse_control_write(0x1, FUSE_FUSEBYPASS_0); | ||
1695 | tegra_fuse_control_write(0x0, FUSE_WRITE_ACCESS_SW_0); | ||
1696 | #endif | ||
1697 | |||
1698 | if (g->gr.gpc_tpc_mask[gpc_index] == 0x1) | ||
1699 | tegra_fuse_writel(0x2, FUSE_OPT_GPU_TPC0_DISABLE_0); | ||
1700 | else if (g->gr.gpc_tpc_mask[gpc_index] == 0x2) | ||
1701 | tegra_fuse_writel(0x1, FUSE_OPT_GPU_TPC0_DISABLE_0); | ||
1702 | else | ||
1703 | tegra_fuse_writel(0x0, FUSE_OPT_GPU_TPC0_DISABLE_0); | ||
1704 | } | ||
1705 | |||
1706 | void gr_gv11b_get_access_map(struct gk20a *g, | ||
1707 | u32 **whitelist, int *num_entries) | ||
1708 | { | ||
1709 | static u32 wl_addr_gv11b[] = { | ||
1710 | /* this list must be sorted (low to high) */ | ||
1711 | 0x404468, /* gr_pri_mme_max_instructions */ | ||
1712 | 0x418300, /* gr_pri_gpcs_rasterarb_line_class */ | ||
1713 | 0x418800, /* gr_pri_gpcs_setup_debug */ | ||
1714 | 0x418e00, /* gr_pri_gpcs_swdx_config */ | ||
1715 | 0x418e40, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */ | ||
1716 | 0x418e44, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */ | ||
1717 | 0x418e48, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */ | ||
1718 | 0x418e4c, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */ | ||
1719 | 0x418e50, /* gr_pri_gpcs_swdx_tc_bundle_ctrl */ | ||
1720 | 0x418e58, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1721 | 0x418e5c, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1722 | 0x418e60, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1723 | 0x418e64, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1724 | 0x418e68, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1725 | 0x418e6c, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1726 | 0x418e70, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1727 | 0x418e74, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1728 | 0x418e78, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1729 | 0x418e7c, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1730 | 0x418e80, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1731 | 0x418e84, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1732 | 0x418e88, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1733 | 0x418e8c, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1734 | 0x418e90, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1735 | 0x418e94, /* gr_pri_gpcs_swdx_tc_bundle_addr */ | ||
1736 | 0x419864, /* gr_pri_gpcs_tpcs_pe_l2_evict_policy */ | ||
1737 | 0x419a04, /* gr_pri_gpcs_tpcs_tex_lod_dbg */ | ||
1738 | 0x419a08, /* gr_pri_gpcs_tpcs_tex_samp_dbg */ | ||
1739 | 0x419e84, /* gr_pri_gpcs_tpcs_sms_dbgr_control0 */ | ||
1740 | 0x419ba4, /* gr_pri_gpcs_tpcs_sm_disp_ctrl */ | ||
1741 | }; | ||
1742 | |||
1743 | *whitelist = wl_addr_gv11b; | ||
1744 | *num_entries = ARRAY_SIZE(wl_addr_gv11b); | ||
1745 | } | ||
1746 | |||
1747 | /* @brief pre-process work on the SM exceptions to determine if we clear them or not. | ||
1748 | * | ||
1749 | * On Pascal, if we are in CILP preemtion mode, preempt the channel and handle errors with special processing | ||
1750 | */ | ||
1751 | int gr_gv11b_pre_process_sm_exception(struct gk20a *g, | ||
1752 | u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr, | ||
1753 | bool sm_debugger_attached, struct channel_gk20a *fault_ch, | ||
1754 | bool *early_exit, bool *ignore_debugger) | ||
1755 | { | ||
1756 | int ret; | ||
1757 | bool cilp_enabled = false; | ||
1758 | u32 global_mask = 0, dbgr_control0, global_esr_copy; | ||
1759 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + | ||
1760 | gk20a_gr_tpc_offset(g, tpc) + | ||
1761 | gv11b_gr_sm_offset(g, sm); | ||
1762 | |||
1763 | *early_exit = false; | ||
1764 | *ignore_debugger = false; | ||
1765 | |||
1766 | if (fault_ch) | ||
1767 | cilp_enabled = (fault_ch->ch_ctx.gr_ctx->compute_preempt_mode == | ||
1768 | NVGPU_PREEMPTION_MODE_COMPUTE_CILP); | ||
1769 | |||
1770 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1771 | "SM Exception received on gpc %d tpc %d sm %d = 0x%08x", | ||
1772 | gpc, tpc, sm, global_esr); | ||
1773 | |||
1774 | if (cilp_enabled && sm_debugger_attached) { | ||
1775 | if (global_esr & gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f()) | ||
1776 | gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset, | ||
1777 | gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f()); | ||
1778 | |||
1779 | if (global_esr & gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f()) | ||
1780 | gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset, | ||
1781 | gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f()); | ||
1782 | |||
1783 | global_mask = gr_gpc0_tpc0_sm0_hww_global_esr_multiple_warp_errors_pending_f() | | ||
1784 | gr_gpc0_tpc0_sm0_hww_global_esr_bpt_pause_pending_f(); | ||
1785 | |||
1786 | if (warp_esr != 0 || (global_esr & global_mask) != 0) { | ||
1787 | *ignore_debugger = true; | ||
1788 | |||
1789 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1790 | "CILP: starting wait for LOCKED_DOWN on " | ||
1791 | "gpc %d tpc %d sm %d", | ||
1792 | gpc, tpc, sm); | ||
1793 | |||
1794 | if (gk20a_dbg_gpu_broadcast_stop_trigger(fault_ch)) { | ||
1795 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1796 | "CILP: Broadcasting STOP_TRIGGER from " | ||
1797 | "gpc %d tpc %d sm %d", | ||
1798 | gpc, tpc, sm); | ||
1799 | g->ops.gr.suspend_all_sms(g, | ||
1800 | global_mask, false); | ||
1801 | |||
1802 | gk20a_dbg_gpu_clear_broadcast_stop_trigger(fault_ch); | ||
1803 | } else { | ||
1804 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1805 | "CILP: STOP_TRIGGER from " | ||
1806 | "gpc %d tpc %d sm %d", | ||
1807 | gpc, tpc, sm); | ||
1808 | g->ops.gr.suspend_single_sm(g, | ||
1809 | gpc, tpc, sm, global_mask, true); | ||
1810 | } | ||
1811 | |||
1812 | /* reset the HWW errors after locking down */ | ||
1813 | global_esr_copy = g->ops.gr.get_sm_hww_global_esr(g, | ||
1814 | gpc, tpc, sm); | ||
1815 | g->ops.gr.clear_sm_hww(g, | ||
1816 | gpc, tpc, sm, global_esr_copy); | ||
1817 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1818 | "CILP: HWWs cleared for " | ||
1819 | "gpc %d tpc %d sm %d", | ||
1820 | gpc, tpc, sm); | ||
1821 | |||
1822 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, "CILP: Setting CILP preempt pending\n"); | ||
1823 | ret = gr_gp10b_set_cilp_preempt_pending(g, fault_ch); | ||
1824 | if (ret) { | ||
1825 | nvgpu_err(g, "CILP: error while setting CILP preempt pending!"); | ||
1826 | return ret; | ||
1827 | } | ||
1828 | |||
1829 | dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
1830 | if (dbgr_control0 & gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_enable_f()) { | ||
1831 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1832 | "CILP: clearing SINGLE_STEP_MODE " | ||
1833 | "before resume for gpc %d tpc %d sm %d", | ||
1834 | gpc, tpc, sm); | ||
1835 | dbgr_control0 = set_field(dbgr_control0, | ||
1836 | gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_m(), | ||
1837 | gr_gpc0_tpc0_sm0_dbgr_control0_single_step_mode_disable_f()); | ||
1838 | gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset, dbgr_control0); | ||
1839 | } | ||
1840 | |||
1841 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1842 | "CILP: resume for gpc %d tpc %d sm %d", | ||
1843 | gpc, tpc, sm); | ||
1844 | g->ops.gr.resume_single_sm(g, gpc, tpc, sm); | ||
1845 | |||
1846 | *ignore_debugger = true; | ||
1847 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
1848 | "CILP: All done on gpc %d, tpc %d sm %d", | ||
1849 | gpc, tpc, sm); | ||
1850 | } | ||
1851 | |||
1852 | *early_exit = true; | ||
1853 | } | ||
1854 | return 0; | ||
1855 | } | ||
1856 | |||
1857 | static void gr_gv11b_handle_fecs_ecc_error(struct gk20a *g, u32 intr) | ||
1858 | { | ||
1859 | u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt; | ||
1860 | u32 corrected_delta, uncorrected_delta; | ||
1861 | u32 corrected_overflow, uncorrected_overflow; | ||
1862 | |||
1863 | if (intr & (gr_fecs_host_int_status_ecc_uncorrected_m() | | ||
1864 | gr_fecs_host_int_status_ecc_corrected_m())) { | ||
1865 | ecc_status = gk20a_readl(g, gr_fecs_falcon_ecc_status_r()); | ||
1866 | ecc_addr = gk20a_readl(g, | ||
1867 | gr_fecs_falcon_ecc_address_r()); | ||
1868 | corrected_cnt = gk20a_readl(g, | ||
1869 | gr_fecs_falcon_ecc_corrected_err_count_r()); | ||
1870 | uncorrected_cnt = gk20a_readl(g, | ||
1871 | gr_fecs_falcon_ecc_uncorrected_err_count_r()); | ||
1872 | |||
1873 | corrected_delta = | ||
1874 | gr_fecs_falcon_ecc_corrected_err_count_total_v( | ||
1875 | corrected_cnt); | ||
1876 | uncorrected_delta = | ||
1877 | gr_fecs_falcon_ecc_uncorrected_err_count_total_v( | ||
1878 | uncorrected_cnt); | ||
1879 | |||
1880 | corrected_overflow = ecc_status & | ||
1881 | gr_fecs_falcon_ecc_status_corrected_err_total_counter_overflow_m(); | ||
1882 | uncorrected_overflow = ecc_status & | ||
1883 | gr_fecs_falcon_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
1884 | |||
1885 | /* clear the interrupt */ | ||
1886 | if ((corrected_delta > 0) || corrected_overflow) | ||
1887 | gk20a_writel(g, | ||
1888 | gr_fecs_falcon_ecc_corrected_err_count_r(), 0); | ||
1889 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
1890 | gk20a_writel(g, | ||
1891 | gr_fecs_falcon_ecc_uncorrected_err_count_r(), | ||
1892 | 0); | ||
1893 | |||
1894 | |||
1895 | /* clear the interrupt */ | ||
1896 | gk20a_writel(g, gr_fecs_falcon_ecc_uncorrected_err_count_r(), | ||
1897 | 0); | ||
1898 | gk20a_writel(g, gr_fecs_falcon_ecc_corrected_err_count_r(), 0); | ||
1899 | |||
1900 | /* clear the interrupt */ | ||
1901 | gk20a_writel(g, gr_fecs_falcon_ecc_status_r(), | ||
1902 | gr_fecs_falcon_ecc_status_reset_task_f()); | ||
1903 | |||
1904 | g->ecc.gr.t19x.fecs_corrected_err_count.counters[0] += | ||
1905 | corrected_delta; | ||
1906 | g->ecc.gr.t19x.fecs_uncorrected_err_count.counters[0] += | ||
1907 | uncorrected_delta; | ||
1908 | |||
1909 | nvgpu_log(g, gpu_dbg_intr, | ||
1910 | "fecs ecc interrupt intr: 0x%x", intr); | ||
1911 | |||
1912 | if (ecc_status & | ||
1913 | gr_fecs_falcon_ecc_status_corrected_err_imem_m()) | ||
1914 | nvgpu_log(g, gpu_dbg_intr, "imem ecc error corrected"); | ||
1915 | if (ecc_status & | ||
1916 | gr_fecs_falcon_ecc_status_uncorrected_err_imem_m()) | ||
1917 | nvgpu_log(g, gpu_dbg_intr, | ||
1918 | "imem ecc error uncorrected"); | ||
1919 | if (ecc_status & | ||
1920 | gr_fecs_falcon_ecc_status_corrected_err_dmem_m()) | ||
1921 | nvgpu_log(g, gpu_dbg_intr, "dmem ecc error corrected"); | ||
1922 | if (ecc_status & | ||
1923 | gr_fecs_falcon_ecc_status_uncorrected_err_dmem_m()) | ||
1924 | nvgpu_log(g, gpu_dbg_intr, | ||
1925 | "dmem ecc error uncorrected"); | ||
1926 | if (corrected_overflow || uncorrected_overflow) | ||
1927 | nvgpu_info(g, "fecs ecc counter overflow!"); | ||
1928 | |||
1929 | nvgpu_log(g, gpu_dbg_intr, | ||
1930 | "ecc error row address: 0x%x", | ||
1931 | gr_fecs_falcon_ecc_address_row_address_v(ecc_addr)); | ||
1932 | |||
1933 | nvgpu_log(g, gpu_dbg_intr, | ||
1934 | "ecc error count corrected: %d, uncorrected %d", | ||
1935 | g->ecc.gr.t19x.fecs_corrected_err_count.counters[0], | ||
1936 | g->ecc.gr.t19x.fecs_uncorrected_err_count.counters[0]); | ||
1937 | } | ||
1938 | } | ||
1939 | |||
1940 | int gr_gv11b_handle_fecs_error(struct gk20a *g, | ||
1941 | struct channel_gk20a *__ch, | ||
1942 | struct gr_gk20a_isr_data *isr_data) | ||
1943 | { | ||
1944 | u32 gr_fecs_intr = gk20a_readl(g, gr_fecs_host_int_status_r()); | ||
1945 | int ret; | ||
1946 | |||
1947 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, ""); | ||
1948 | |||
1949 | ret = gr_gp10b_handle_fecs_error(g, __ch, isr_data); | ||
1950 | |||
1951 | /* Handle ECC errors */ | ||
1952 | gr_gv11b_handle_fecs_ecc_error(g, gr_fecs_intr); | ||
1953 | |||
1954 | return ret; | ||
1955 | } | ||
1956 | |||
1957 | int gr_gv11b_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr) | ||
1958 | { | ||
1959 | u32 map; | ||
1960 | u32 i, j, mapregs; | ||
1961 | u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); | ||
1962 | u32 num_tpc_per_gpc = nvgpu_get_litter_value(g, | ||
1963 | GPU_LIT_NUM_TPC_PER_GPC); | ||
1964 | |||
1965 | gk20a_dbg_fn(""); | ||
1966 | |||
1967 | if (!gr->map_tiles) | ||
1968 | return -1; | ||
1969 | |||
1970 | gk20a_writel(g, gr_crstr_map_table_cfg_r(), | ||
1971 | gr_crstr_map_table_cfg_row_offset_f(gr->map_row_offset) | | ||
1972 | gr_crstr_map_table_cfg_num_entries_f(gr->tpc_count)); | ||
1973 | |||
1974 | /* 6 tpc can be stored in one map register */ | ||
1975 | mapregs = (num_gpcs * num_tpc_per_gpc + 5) / 6; | ||
1976 | |||
1977 | for (i = 0, j = 0; i < mapregs; i++, j = j + 6) { | ||
1978 | map = gr_crstr_gpc_map_tile0_f(gr->map_tiles[j]) | | ||
1979 | gr_crstr_gpc_map_tile1_f(gr->map_tiles[j + 1]) | | ||
1980 | gr_crstr_gpc_map_tile2_f(gr->map_tiles[j + 2]) | | ||
1981 | gr_crstr_gpc_map_tile3_f(gr->map_tiles[j + 3]) | | ||
1982 | gr_crstr_gpc_map_tile4_f(gr->map_tiles[j + 4]) | | ||
1983 | gr_crstr_gpc_map_tile5_f(gr->map_tiles[j + 5]); | ||
1984 | |||
1985 | gk20a_writel(g, gr_crstr_gpc_map_r(i), map); | ||
1986 | gk20a_writel(g, gr_ppcs_wwdx_map_gpc_map_r(i), map); | ||
1987 | gk20a_writel(g, gr_rstr2d_gpc_map_r(i), map); | ||
1988 | } | ||
1989 | |||
1990 | gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_r(), | ||
1991 | gr_ppcs_wwdx_map_table_cfg_row_offset_f(gr->map_row_offset) | | ||
1992 | gr_ppcs_wwdx_map_table_cfg_num_entries_f(gr->tpc_count)); | ||
1993 | |||
1994 | for (i = 0, j = 1; i < gr_ppcs_wwdx_map_table_cfg_coeff__size_1_v(); | ||
1995 | i++, j = j + 4) { | ||
1996 | gk20a_writel(g, gr_ppcs_wwdx_map_table_cfg_coeff_r(i), | ||
1997 | gr_ppcs_wwdx_map_table_cfg_coeff_0_mod_value_f( | ||
1998 | ((1 << j) % gr->tpc_count)) | | ||
1999 | gr_ppcs_wwdx_map_table_cfg_coeff_1_mod_value_f( | ||
2000 | ((1 << (j + 1)) % gr->tpc_count)) | | ||
2001 | gr_ppcs_wwdx_map_table_cfg_coeff_2_mod_value_f( | ||
2002 | ((1 << (j + 2)) % gr->tpc_count)) | | ||
2003 | gr_ppcs_wwdx_map_table_cfg_coeff_3_mod_value_f( | ||
2004 | ((1 << (j + 3)) % gr->tpc_count))); | ||
2005 | } | ||
2006 | |||
2007 | gk20a_writel(g, gr_rstr2d_map_table_cfg_r(), | ||
2008 | gr_rstr2d_map_table_cfg_row_offset_f(gr->map_row_offset) | | ||
2009 | gr_rstr2d_map_table_cfg_num_entries_f(gr->tpc_count)); | ||
2010 | |||
2011 | return 0; | ||
2012 | } | ||
2013 | |||
2014 | static int gv11b_write_bundle_veid_state(struct gk20a *g, u32 index) | ||
2015 | { | ||
2016 | struct av_list_gk20a *sw_veid_bundle_init = | ||
2017 | &g->gr.ctx_vars.sw_veid_bundle_init; | ||
2018 | u32 j; | ||
2019 | u32 num_subctx, err = 0; | ||
2020 | |||
2021 | num_subctx = g->fifo.t19x.max_subctx_count; | ||
2022 | |||
2023 | for (j = 0; j < num_subctx; j++) { | ||
2024 | nvgpu_log_fn(g, "write bundle_address_r for subctx: %d", j); | ||
2025 | gk20a_writel(g, gr_pipe_bundle_address_r(), | ||
2026 | sw_veid_bundle_init->l[index].addr | | ||
2027 | gr_pipe_bundle_address_veid_f(j)); | ||
2028 | |||
2029 | err = gr_gk20a_wait_fe_idle(g, gk20a_get_gr_idle_timeout(g), | ||
2030 | GR_IDLE_CHECK_DEFAULT); | ||
2031 | } | ||
2032 | return err; | ||
2033 | } | ||
2034 | |||
2035 | int gr_gv11b_init_sw_veid_bundle(struct gk20a *g) | ||
2036 | { | ||
2037 | struct av_list_gk20a *sw_veid_bundle_init = | ||
2038 | &g->gr.ctx_vars.sw_veid_bundle_init; | ||
2039 | u32 i; | ||
2040 | u32 last_bundle_data = 0; | ||
2041 | u32 err = 0; | ||
2042 | |||
2043 | for (i = 0; i < sw_veid_bundle_init->count; i++) { | ||
2044 | nvgpu_log_fn(g, "veid bundle count: %d", i); | ||
2045 | |||
2046 | if (i == 0 || last_bundle_data != | ||
2047 | sw_veid_bundle_init->l[i].value) { | ||
2048 | gk20a_writel(g, gr_pipe_bundle_data_r(), | ||
2049 | sw_veid_bundle_init->l[i].value); | ||
2050 | last_bundle_data = sw_veid_bundle_init->l[i].value; | ||
2051 | nvgpu_log_fn(g, "last_bundle_data : 0x%08x", | ||
2052 | last_bundle_data); | ||
2053 | } | ||
2054 | |||
2055 | if (gr_pipe_bundle_address_value_v( | ||
2056 | sw_veid_bundle_init->l[i].addr) == GR_GO_IDLE_BUNDLE) { | ||
2057 | nvgpu_log_fn(g, "go idle bundle"); | ||
2058 | gk20a_writel(g, gr_pipe_bundle_address_r(), | ||
2059 | sw_veid_bundle_init->l[i].addr); | ||
2060 | err |= gr_gk20a_wait_idle(g, | ||
2061 | gk20a_get_gr_idle_timeout(g), | ||
2062 | GR_IDLE_CHECK_DEFAULT); | ||
2063 | } else | ||
2064 | err = gv11b_write_bundle_veid_state(g, i); | ||
2065 | |||
2066 | if (err) { | ||
2067 | nvgpu_err(g, "failed to init sw veid bundle"); | ||
2068 | break; | ||
2069 | } | ||
2070 | } | ||
2071 | return err; | ||
2072 | } | ||
2073 | |||
2074 | void gr_gv11b_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries, | ||
2075 | u32 *zcull_map_tiles) | ||
2076 | { | ||
2077 | u32 val, i, j; | ||
2078 | |||
2079 | gk20a_dbg_fn(""); | ||
2080 | |||
2081 | for (i = 0, j = 0; i < (zcull_num_entries / 8); i++, j += 8) { | ||
2082 | val = | ||
2083 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_0_f( | ||
2084 | zcull_map_tiles[j+0]) | | ||
2085 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_1_f( | ||
2086 | zcull_map_tiles[j+1]) | | ||
2087 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_2_f( | ||
2088 | zcull_map_tiles[j+2]) | | ||
2089 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_3_f( | ||
2090 | zcull_map_tiles[j+3]) | | ||
2091 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_4_f( | ||
2092 | zcull_map_tiles[j+4]) | | ||
2093 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_5_f( | ||
2094 | zcull_map_tiles[j+5]) | | ||
2095 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_6_f( | ||
2096 | zcull_map_tiles[j+6]) | | ||
2097 | gr_gpcs_zcull_sm_in_gpc_number_map_tile_7_f( | ||
2098 | zcull_map_tiles[j+7]); | ||
2099 | |||
2100 | gk20a_writel(g, gr_gpcs_zcull_sm_in_gpc_number_map_r(i), val); | ||
2101 | } | ||
2102 | } | ||
2103 | |||
2104 | void gr_gv11b_detect_sm_arch(struct gk20a *g) | ||
2105 | { | ||
2106 | u32 v = gk20a_readl(g, gr_gpc0_tpc0_sm_arch_r()); | ||
2107 | |||
2108 | g->params.sm_arch_spa_version = | ||
2109 | gr_gpc0_tpc0_sm_arch_spa_version_v(v); | ||
2110 | g->params.sm_arch_sm_version = | ||
2111 | gr_gpc0_tpc0_sm_arch_sm_version_v(v); | ||
2112 | g->params.sm_arch_warp_count = | ||
2113 | gr_gpc0_tpc0_sm_arch_warp_count_v(v); | ||
2114 | } | ||
2115 | |||
2116 | void gr_gv11b_program_sm_id_numbering(struct gk20a *g, | ||
2117 | u32 gpc, u32 tpc, u32 smid) | ||
2118 | { | ||
2119 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
2120 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, | ||
2121 | GPU_LIT_TPC_IN_GPC_STRIDE); | ||
2122 | u32 gpc_offset = gpc_stride * gpc; | ||
2123 | u32 tpc_offset = tpc_in_gpc_stride * tpc; | ||
2124 | u32 global_tpc_index = g->gr.sm_to_cluster[smid].global_tpc_index; | ||
2125 | |||
2126 | gk20a_writel(g, gr_gpc0_tpc0_sm_cfg_r() + gpc_offset + tpc_offset, | ||
2127 | gr_gpc0_tpc0_sm_cfg_tpc_id_f(global_tpc_index)); | ||
2128 | gk20a_writel(g, gr_gpc0_gpm_pd_sm_id_r(tpc) + gpc_offset, | ||
2129 | gr_gpc0_gpm_pd_sm_id_id_f(global_tpc_index)); | ||
2130 | gk20a_writel(g, gr_gpc0_tpc0_pe_cfg_smid_r() + gpc_offset + tpc_offset, | ||
2131 | gr_gpc0_tpc0_pe_cfg_smid_value_f(global_tpc_index)); | ||
2132 | } | ||
2133 | |||
2134 | int gr_gv11b_load_smid_config(struct gk20a *g) | ||
2135 | { | ||
2136 | u32 *tpc_sm_id; | ||
2137 | u32 i, j; | ||
2138 | u32 tpc_index, gpc_index, tpc_id; | ||
2139 | u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); | ||
2140 | int num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); | ||
2141 | |||
2142 | tpc_sm_id = nvgpu_kcalloc(g, gr_cwd_sm_id__size_1_v(), sizeof(u32)); | ||
2143 | if (!tpc_sm_id) | ||
2144 | return -ENOMEM; | ||
2145 | |||
2146 | /* Each NV_PGRAPH_PRI_CWD_GPC_TPC_ID can store 4 TPCs.*/ | ||
2147 | for (i = 0; i <= ((g->gr.tpc_count-1) / 4); i++) { | ||
2148 | u32 reg = 0; | ||
2149 | u32 bit_stride = gr_cwd_gpc_tpc_id_gpc0_s() + | ||
2150 | gr_cwd_gpc_tpc_id_tpc0_s(); | ||
2151 | |||
2152 | for (j = 0; j < 4; j++) { | ||
2153 | u32 sm_id; | ||
2154 | u32 bits; | ||
2155 | |||
2156 | tpc_id = (i << 2) + j; | ||
2157 | sm_id = tpc_id * sm_per_tpc; | ||
2158 | |||
2159 | if (sm_id >= g->gr.no_of_sm) | ||
2160 | break; | ||
2161 | |||
2162 | gpc_index = g->gr.sm_to_cluster[sm_id].gpc_index; | ||
2163 | tpc_index = g->gr.sm_to_cluster[sm_id].tpc_index; | ||
2164 | |||
2165 | bits = gr_cwd_gpc_tpc_id_gpc0_f(gpc_index) | | ||
2166 | gr_cwd_gpc_tpc_id_tpc0_f(tpc_index); | ||
2167 | reg |= bits << (j * bit_stride); | ||
2168 | |||
2169 | tpc_sm_id[gpc_index + (num_gpcs * ((tpc_index & 4) | ||
2170 | >> 2))] |= tpc_id << tpc_index * bit_stride; | ||
2171 | } | ||
2172 | gk20a_writel(g, gr_cwd_gpc_tpc_id_r(i), reg); | ||
2173 | } | ||
2174 | |||
2175 | for (i = 0; i < gr_cwd_sm_id__size_1_v(); i++) | ||
2176 | gk20a_writel(g, gr_cwd_sm_id_r(i), tpc_sm_id[i]); | ||
2177 | nvgpu_kfree(g, tpc_sm_id); | ||
2178 | |||
2179 | return 0; | ||
2180 | } | ||
2181 | |||
2182 | int gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va) | ||
2183 | { | ||
2184 | u32 addr_lo; | ||
2185 | u32 addr_hi; | ||
2186 | struct ctx_header_desc *ctx; | ||
2187 | int err; | ||
2188 | |||
2189 | gk20a_dbg_fn(""); | ||
2190 | |||
2191 | err = gv11b_alloc_subctx_header(c); | ||
2192 | if (err) | ||
2193 | return err; | ||
2194 | |||
2195 | err = gv11b_update_subctx_header(c, gpu_va); | ||
2196 | if (err) | ||
2197 | return err; | ||
2198 | |||
2199 | ctx = &c->ch_ctx.ctx_header; | ||
2200 | addr_lo = u64_lo32(ctx->mem.gpu_va) >> ram_in_base_shift_v(); | ||
2201 | addr_hi = u64_hi32(ctx->mem.gpu_va); | ||
2202 | |||
2203 | /* point this address to engine_wfi_ptr */ | ||
2204 | nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_engine_wfi_target_w(), | ||
2205 | ram_in_engine_cs_wfi_v() | | ||
2206 | ram_in_engine_wfi_mode_f(ram_in_engine_wfi_mode_virtual_v()) | | ||
2207 | ram_in_engine_wfi_ptr_lo_f(addr_lo)); | ||
2208 | |||
2209 | nvgpu_mem_wr32(c->g, &c->inst_block, ram_in_engine_wfi_ptr_hi_w(), | ||
2210 | ram_in_engine_wfi_ptr_hi_f(addr_hi)); | ||
2211 | |||
2212 | return 0; | ||
2213 | } | ||
2214 | |||
2215 | |||
2216 | |||
2217 | int gr_gv11b_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c) | ||
2218 | { | ||
2219 | struct channel_ctx_gk20a *ch_ctx = NULL; | ||
2220 | u32 pd_ab_dist_cfg0; | ||
2221 | u32 ds_debug; | ||
2222 | u32 mpc_vtg_debug; | ||
2223 | u32 pe_vaf; | ||
2224 | u32 pe_vsc_vpc; | ||
2225 | |||
2226 | gk20a_dbg_fn(""); | ||
2227 | |||
2228 | pd_ab_dist_cfg0 = gk20a_readl(g, gr_pd_ab_dist_cfg0_r()); | ||
2229 | ds_debug = gk20a_readl(g, gr_ds_debug_r()); | ||
2230 | mpc_vtg_debug = gk20a_readl(g, gr_gpcs_tpcs_mpc_vtg_debug_r()); | ||
2231 | |||
2232 | pe_vaf = gk20a_readl(g, gr_gpcs_tpcs_pe_vaf_r()); | ||
2233 | pe_vsc_vpc = gk20a_readl(g, gr_gpcs_tpcs_pes_vsc_vpc_r()); | ||
2234 | |||
2235 | pe_vaf = gr_gpcs_tpcs_pe_vaf_fast_mode_switch_true_f() | pe_vaf; | ||
2236 | pe_vsc_vpc = gr_gpcs_tpcs_pes_vsc_vpc_fast_mode_switch_true_f() | | ||
2237 | pe_vsc_vpc; | ||
2238 | pd_ab_dist_cfg0 = gr_pd_ab_dist_cfg0_timeslice_enable_en_f() | | ||
2239 | pd_ab_dist_cfg0; | ||
2240 | ds_debug = gr_ds_debug_timeslice_mode_enable_f() | ds_debug; | ||
2241 | mpc_vtg_debug = gr_gpcs_tpcs_mpc_vtg_debug_timeslice_mode_enabled_f() | | ||
2242 | mpc_vtg_debug; | ||
2243 | |||
2244 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_pe_vaf_r(), pe_vaf, | ||
2245 | false); | ||
2246 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_pes_vsc_vpc_r(), | ||
2247 | pe_vsc_vpc, false); | ||
2248 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_pd_ab_dist_cfg0_r(), | ||
2249 | pd_ab_dist_cfg0, false); | ||
2250 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_ds_debug_r(), ds_debug, false); | ||
2251 | gr_gk20a_ctx_patch_write(g, ch_ctx, gr_gpcs_tpcs_mpc_vtg_debug_r(), | ||
2252 | mpc_vtg_debug, false); | ||
2253 | |||
2254 | return 0; | ||
2255 | } | ||
2256 | |||
2257 | void gr_gv11b_write_zcull_ptr(struct gk20a *g, | ||
2258 | struct nvgpu_mem *mem, u64 gpu_va) | ||
2259 | { | ||
2260 | u32 va_lo, va_hi; | ||
2261 | |||
2262 | gpu_va = gpu_va >> 8; | ||
2263 | va_lo = u64_lo32(gpu_va); | ||
2264 | va_hi = u64_hi32(gpu_va); | ||
2265 | nvgpu_mem_wr(g, mem, | ||
2266 | ctxsw_prog_main_image_zcull_ptr_o(), va_lo); | ||
2267 | nvgpu_mem_wr(g, mem, | ||
2268 | ctxsw_prog_main_image_zcull_ptr_hi_o(), va_hi); | ||
2269 | } | ||
2270 | |||
2271 | |||
2272 | void gr_gv11b_write_pm_ptr(struct gk20a *g, | ||
2273 | struct nvgpu_mem *mem, u64 gpu_va) | ||
2274 | { | ||
2275 | u32 va_lo, va_hi; | ||
2276 | |||
2277 | gpu_va = gpu_va >> 8; | ||
2278 | va_lo = u64_lo32(gpu_va); | ||
2279 | va_hi = u64_hi32(gpu_va); | ||
2280 | nvgpu_mem_wr(g, mem, | ||
2281 | ctxsw_prog_main_image_pm_ptr_o(), va_lo); | ||
2282 | nvgpu_mem_wr(g, mem, | ||
2283 | ctxsw_prog_main_image_pm_ptr_hi_o(), va_hi); | ||
2284 | } | ||
2285 | |||
2286 | void gr_gv11b_init_elcg_mode(struct gk20a *g, u32 mode, u32 engine) | ||
2287 | { | ||
2288 | u32 gate_ctrl; | ||
2289 | |||
2290 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_ELCG)) | ||
2291 | return; | ||
2292 | |||
2293 | gate_ctrl = gk20a_readl(g, therm_gate_ctrl_r(engine)); | ||
2294 | |||
2295 | switch (mode) { | ||
2296 | case ELCG_RUN: | ||
2297 | gate_ctrl = set_field(gate_ctrl, | ||
2298 | therm_gate_ctrl_eng_clk_m(), | ||
2299 | therm_gate_ctrl_eng_clk_run_f()); | ||
2300 | gate_ctrl = set_field(gate_ctrl, | ||
2301 | therm_gate_ctrl_idle_holdoff_m(), | ||
2302 | therm_gate_ctrl_idle_holdoff_on_f()); | ||
2303 | break; | ||
2304 | case ELCG_STOP: | ||
2305 | gate_ctrl = set_field(gate_ctrl, | ||
2306 | therm_gate_ctrl_eng_clk_m(), | ||
2307 | therm_gate_ctrl_eng_clk_stop_f()); | ||
2308 | break; | ||
2309 | case ELCG_AUTO: | ||
2310 | gate_ctrl = set_field(gate_ctrl, | ||
2311 | therm_gate_ctrl_eng_clk_m(), | ||
2312 | therm_gate_ctrl_eng_clk_auto_f()); | ||
2313 | break; | ||
2314 | default: | ||
2315 | nvgpu_err(g, "invalid elcg mode %d", mode); | ||
2316 | } | ||
2317 | |||
2318 | gk20a_writel(g, therm_gate_ctrl_r(engine), gate_ctrl); | ||
2319 | } | ||
2320 | |||
2321 | void gr_gv11b_load_tpc_mask(struct gk20a *g) | ||
2322 | { | ||
2323 | u32 pes_tpc_mask = 0, fuse_tpc_mask; | ||
2324 | u32 gpc, pes, val; | ||
2325 | u32 num_tpc_per_gpc = nvgpu_get_litter_value(g, | ||
2326 | GPU_LIT_NUM_TPC_PER_GPC); | ||
2327 | |||
2328 | /* gv11b has 1 GPC and 4 TPC/GPC, so mask will not overflow u32 */ | ||
2329 | for (gpc = 0; gpc < g->gr.gpc_count; gpc++) { | ||
2330 | for (pes = 0; pes < g->gr.pe_count_per_gpc; pes++) { | ||
2331 | pes_tpc_mask |= g->gr.pes_tpc_mask[pes][gpc] << | ||
2332 | num_tpc_per_gpc * gpc; | ||
2333 | } | ||
2334 | } | ||
2335 | |||
2336 | gk20a_dbg_info("pes_tpc_mask %u\n", pes_tpc_mask); | ||
2337 | fuse_tpc_mask = g->ops.gr.get_gpc_tpc_mask(g, gpc); | ||
2338 | if (g->tpc_fs_mask_user && | ||
2339 | g->tpc_fs_mask_user != fuse_tpc_mask && | ||
2340 | fuse_tpc_mask == (0x1U << g->gr.max_tpc_count) - 1U) { | ||
2341 | val = g->tpc_fs_mask_user; | ||
2342 | val &= (0x1U << g->gr.max_tpc_count) - 1U; | ||
2343 | val = (0x1U << hweight32(val)) - 1U; | ||
2344 | gk20a_writel(g, gr_fe_tpc_fs_r(0), val); | ||
2345 | } else { | ||
2346 | gk20a_writel(g, gr_fe_tpc_fs_r(0), pes_tpc_mask); | ||
2347 | } | ||
2348 | |||
2349 | } | ||
2350 | |||
2351 | void gr_gv11b_set_preemption_buffer_va(struct gk20a *g, | ||
2352 | struct nvgpu_mem *mem, u64 gpu_va) | ||
2353 | { | ||
2354 | u32 addr_lo, addr_hi; | ||
2355 | |||
2356 | addr_lo = u64_lo32(gpu_va); | ||
2357 | addr_hi = u64_hi32(gpu_va); | ||
2358 | |||
2359 | nvgpu_mem_wr(g, mem, | ||
2360 | ctxsw_prog_main_image_full_preemption_ptr_o(), addr_lo); | ||
2361 | nvgpu_mem_wr(g, mem, | ||
2362 | ctxsw_prog_main_image_full_preemption_ptr_hi_o(), addr_hi); | ||
2363 | |||
2364 | nvgpu_mem_wr(g, mem, | ||
2365 | ctxsw_prog_main_image_full_preemption_ptr_veid0_o(), addr_lo); | ||
2366 | nvgpu_mem_wr(g, mem, | ||
2367 | ctxsw_prog_main_image_full_preemption_ptr_veid0_hi_o(), | ||
2368 | addr_hi); | ||
2369 | |||
2370 | } | ||
2371 | |||
2372 | int gr_gv11b_init_fs_state(struct gk20a *g) | ||
2373 | { | ||
2374 | u32 data; | ||
2375 | |||
2376 | gk20a_dbg_fn(""); | ||
2377 | |||
2378 | data = gk20a_readl(g, gr_gpcs_tpcs_sm_texio_control_r()); | ||
2379 | data = set_field(data, gr_gpcs_tpcs_sm_texio_control_oor_addr_check_mode_m(), | ||
2380 | gr_gpcs_tpcs_sm_texio_control_oor_addr_check_mode_arm_63_48_match_f()); | ||
2381 | gk20a_writel(g, gr_gpcs_tpcs_sm_texio_control_r(), data); | ||
2382 | |||
2383 | data = gk20a_readl(g, gr_gpcs_tpcs_sm_disp_ctrl_r()); | ||
2384 | data = set_field(data, gr_gpcs_tpcs_sm_disp_ctrl_re_suppress_m(), | ||
2385 | gr_gpcs_tpcs_sm_disp_ctrl_re_suppress_disable_f()); | ||
2386 | gk20a_writel(g, gr_gpcs_tpcs_sm_disp_ctrl_r(), data); | ||
2387 | |||
2388 | if (g->gr.t18x.fecs_feature_override_ecc_val != 0) { | ||
2389 | gk20a_writel(g, | ||
2390 | gr_fecs_feature_override_ecc_r(), | ||
2391 | g->gr.t18x.fecs_feature_override_ecc_val); | ||
2392 | } | ||
2393 | |||
2394 | return gr_gm20b_init_fs_state(g); | ||
2395 | } | ||
2396 | |||
2397 | void gv11b_gr_get_esr_sm_sel(struct gk20a *g, u32 gpc, u32 tpc, | ||
2398 | u32 *esr_sm_sel) | ||
2399 | { | ||
2400 | u32 reg_val; | ||
2401 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); | ||
2402 | |||
2403 | reg_val = gk20a_readl(g, gr_gpc0_tpc0_sm_tpc_esr_sm_sel_r() + offset); | ||
2404 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2405 | "sm tpc esr sm sel reg val: 0x%x", reg_val); | ||
2406 | *esr_sm_sel = 0; | ||
2407 | if (gr_gpc0_tpc0_sm_tpc_esr_sm_sel_sm0_error_v(reg_val)) | ||
2408 | *esr_sm_sel = 1; | ||
2409 | if (gr_gpc0_tpc0_sm_tpc_esr_sm_sel_sm1_error_v(reg_val)) | ||
2410 | *esr_sm_sel |= 1 << 1; | ||
2411 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2412 | "esr_sm_sel bitmask: 0x%x", *esr_sm_sel); | ||
2413 | } | ||
2414 | |||
2415 | int gv11b_gr_sm_trigger_suspend(struct gk20a *g) | ||
2416 | { | ||
2417 | u32 dbgr_control0; | ||
2418 | |||
2419 | /* assert stop trigger. uniformity assumption: all SMs will have | ||
2420 | * the same state in dbg_control0. | ||
2421 | */ | ||
2422 | dbgr_control0 = | ||
2423 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2424 | dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f(); | ||
2425 | |||
2426 | /* broadcast write */ | ||
2427 | gk20a_writel(g, | ||
2428 | gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0); | ||
2429 | |||
2430 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
2431 | "stop trigger enable: broadcast dbgr_control0: 0x%x ", | ||
2432 | dbgr_control0); | ||
2433 | |||
2434 | return 0; | ||
2435 | } | ||
2436 | |||
2437 | void gv11b_gr_bpt_reg_info(struct gk20a *g, struct nvgpu_warpstate *w_state) | ||
2438 | { | ||
2439 | /* Check if we have at least one valid warp | ||
2440 | * get paused state on maxwell | ||
2441 | */ | ||
2442 | struct gr_gk20a *gr = &g->gr; | ||
2443 | u32 gpc, tpc, sm, sm_id; | ||
2444 | u32 offset; | ||
2445 | u64 warps_valid = 0, warps_paused = 0, warps_trapped = 0; | ||
2446 | |||
2447 | for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) { | ||
2448 | gpc = g->gr.sm_to_cluster[sm_id].gpc_index; | ||
2449 | tpc = g->gr.sm_to_cluster[sm_id].tpc_index; | ||
2450 | sm = g->gr.sm_to_cluster[sm_id].sm_index; | ||
2451 | |||
2452 | offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2453 | gk20a_gr_tpc_offset(g, tpc) + | ||
2454 | gv11b_gr_sm_offset(g, sm); | ||
2455 | |||
2456 | /* 64 bit read */ | ||
2457 | warps_valid = (u64)gk20a_readl(g, | ||
2458 | gr_gpc0_tpc0_sm0_warp_valid_mask_1_r() + | ||
2459 | offset) << 32; | ||
2460 | warps_valid |= gk20a_readl(g, | ||
2461 | gr_gpc0_tpc0_sm0_warp_valid_mask_0_r() + | ||
2462 | offset); | ||
2463 | |||
2464 | /* 64 bit read */ | ||
2465 | warps_paused = (u64)gk20a_readl(g, | ||
2466 | gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_1_r() + | ||
2467 | offset) << 32; | ||
2468 | warps_paused |= gk20a_readl(g, | ||
2469 | gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_0_r() + | ||
2470 | offset); | ||
2471 | |||
2472 | /* 64 bit read */ | ||
2473 | warps_trapped = (u64)gk20a_readl(g, | ||
2474 | gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_1_r() + | ||
2475 | offset) << 32; | ||
2476 | warps_trapped |= gk20a_readl(g, | ||
2477 | gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_0_r() + | ||
2478 | offset); | ||
2479 | |||
2480 | w_state[sm_id].valid_warps[0] = warps_valid; | ||
2481 | w_state[sm_id].trapped_warps[0] = warps_trapped; | ||
2482 | w_state[sm_id].paused_warps[0] = warps_paused; | ||
2483 | } | ||
2484 | |||
2485 | |||
2486 | /* Only for debug purpose */ | ||
2487 | for (sm_id = 0; sm_id < gr->no_of_sm; sm_id++) { | ||
2488 | gk20a_dbg_fn("w_state[%d].valid_warps[0]: %llx\n", | ||
2489 | sm_id, w_state[sm_id].valid_warps[0]); | ||
2490 | gk20a_dbg_fn("w_state[%d].valid_warps[1]: %llx\n", | ||
2491 | sm_id, w_state[sm_id].valid_warps[1]); | ||
2492 | |||
2493 | gk20a_dbg_fn("w_state[%d].trapped_warps[0]: %llx\n", | ||
2494 | sm_id, w_state[sm_id].trapped_warps[0]); | ||
2495 | gk20a_dbg_fn("w_state[%d].trapped_warps[1]: %llx\n", | ||
2496 | sm_id, w_state[sm_id].trapped_warps[1]); | ||
2497 | |||
2498 | gk20a_dbg_fn("w_state[%d].paused_warps[0]: %llx\n", | ||
2499 | sm_id, w_state[sm_id].paused_warps[0]); | ||
2500 | gk20a_dbg_fn("w_state[%d].paused_warps[1]: %llx\n", | ||
2501 | sm_id, w_state[sm_id].paused_warps[1]); | ||
2502 | } | ||
2503 | } | ||
2504 | |||
2505 | int gv11b_gr_update_sm_error_state(struct gk20a *g, | ||
2506 | struct channel_gk20a *ch, u32 sm_id, | ||
2507 | struct nvgpu_gr_sm_error_state *sm_error_state) | ||
2508 | { | ||
2509 | u32 gpc, tpc, sm, offset; | ||
2510 | struct gr_gk20a *gr = &g->gr; | ||
2511 | struct channel_ctx_gk20a *ch_ctx = &ch->ch_ctx; | ||
2512 | int err = 0; | ||
2513 | |||
2514 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | ||
2515 | |||
2516 | gr->sm_error_states[sm_id].hww_global_esr = | ||
2517 | sm_error_state->hww_global_esr; | ||
2518 | gr->sm_error_states[sm_id].hww_warp_esr = | ||
2519 | sm_error_state->hww_warp_esr; | ||
2520 | gr->sm_error_states[sm_id].hww_warp_esr_pc = | ||
2521 | sm_error_state->hww_warp_esr_pc; | ||
2522 | gr->sm_error_states[sm_id].hww_global_esr_report_mask = | ||
2523 | sm_error_state->hww_global_esr_report_mask; | ||
2524 | gr->sm_error_states[sm_id].hww_warp_esr_report_mask = | ||
2525 | sm_error_state->hww_warp_esr_report_mask; | ||
2526 | |||
2527 | err = gr_gk20a_disable_ctxsw(g); | ||
2528 | if (err) { | ||
2529 | nvgpu_err(g, "unable to stop gr ctxsw"); | ||
2530 | goto fail; | ||
2531 | } | ||
2532 | |||
2533 | gpc = g->gr.sm_to_cluster[sm_id].gpc_index; | ||
2534 | tpc = g->gr.sm_to_cluster[sm_id].tpc_index; | ||
2535 | sm = g->gr.sm_to_cluster[sm_id].sm_index; | ||
2536 | |||
2537 | offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2538 | gk20a_gr_tpc_offset(g, tpc) + | ||
2539 | gv11b_gr_sm_offset(g, sm); | ||
2540 | |||
2541 | if (gk20a_is_channel_ctx_resident(ch)) { | ||
2542 | gk20a_writel(g, | ||
2543 | gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset, | ||
2544 | gr->sm_error_states[sm_id].hww_global_esr); | ||
2545 | gk20a_writel(g, | ||
2546 | gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset, | ||
2547 | gr->sm_error_states[sm_id].hww_warp_esr); | ||
2548 | gk20a_writel(g, | ||
2549 | gr_gpc0_tpc0_sm0_hww_warp_esr_pc_r() + offset, | ||
2550 | gr->sm_error_states[sm_id].hww_warp_esr_pc); | ||
2551 | gk20a_writel(g, | ||
2552 | gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset, | ||
2553 | gr->sm_error_states[sm_id].hww_global_esr_report_mask); | ||
2554 | gk20a_writel(g, | ||
2555 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset, | ||
2556 | gr->sm_error_states[sm_id].hww_warp_esr_report_mask); | ||
2557 | } else { | ||
2558 | err = gr_gk20a_ctx_patch_write_begin(g, ch_ctx, false); | ||
2559 | if (err) | ||
2560 | goto enable_ctxsw; | ||
2561 | |||
2562 | gr_gk20a_ctx_patch_write(g, ch_ctx, | ||
2563 | gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r() + | ||
2564 | offset, | ||
2565 | gr->sm_error_states[sm_id].hww_global_esr_report_mask, | ||
2566 | true); | ||
2567 | gr_gk20a_ctx_patch_write(g, ch_ctx, | ||
2568 | gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r() + | ||
2569 | offset, | ||
2570 | gr->sm_error_states[sm_id].hww_warp_esr_report_mask, | ||
2571 | true); | ||
2572 | |||
2573 | gr_gk20a_ctx_patch_write_end(g, ch_ctx, false); | ||
2574 | } | ||
2575 | |||
2576 | enable_ctxsw: | ||
2577 | err = gr_gk20a_enable_ctxsw(g); | ||
2578 | |||
2579 | fail: | ||
2580 | nvgpu_mutex_release(&g->dbg_sessions_lock); | ||
2581 | return err; | ||
2582 | } | ||
2583 | |||
2584 | int gv11b_gr_set_sm_debug_mode(struct gk20a *g, | ||
2585 | struct channel_gk20a *ch, u64 sms, bool enable) | ||
2586 | { | ||
2587 | struct nvgpu_dbg_gpu_reg_op *ops; | ||
2588 | unsigned int i = 0, sm_id; | ||
2589 | int err; | ||
2590 | |||
2591 | ops = nvgpu_kcalloc(g, g->gr.no_of_sm, sizeof(*ops)); | ||
2592 | if (!ops) | ||
2593 | return -ENOMEM; | ||
2594 | for (sm_id = 0; sm_id < g->gr.no_of_sm; sm_id++) { | ||
2595 | u32 gpc, tpc, sm; | ||
2596 | u32 reg_offset, reg_mask, reg_val; | ||
2597 | |||
2598 | if (!(sms & (1 << sm_id))) | ||
2599 | continue; | ||
2600 | |||
2601 | gpc = g->gr.sm_to_cluster[sm_id].gpc_index; | ||
2602 | tpc = g->gr.sm_to_cluster[sm_id].tpc_index; | ||
2603 | sm = g->gr.sm_to_cluster[sm_id].sm_index; | ||
2604 | |||
2605 | reg_offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2606 | gk20a_gr_tpc_offset(g, tpc) + | ||
2607 | gv11b_gr_sm_offset(g, sm); | ||
2608 | |||
2609 | ops[i].op = REGOP(WRITE_32); | ||
2610 | ops[i].type = REGOP(TYPE_GR_CTX); | ||
2611 | ops[i].offset = gr_gpc0_tpc0_sm0_dbgr_control0_r() + reg_offset; | ||
2612 | |||
2613 | reg_mask = 0; | ||
2614 | reg_val = 0; | ||
2615 | if (enable) { | ||
2616 | nvgpu_log(g, gpu_dbg_gpu_dbg, | ||
2617 | "SM:%d debuggger mode ON", sm); | ||
2618 | reg_mask |= | ||
2619 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_m(); | ||
2620 | reg_val |= | ||
2621 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_on_f(); | ||
2622 | } else { | ||
2623 | nvgpu_log(g, gpu_dbg_gpu_dbg, | ||
2624 | "SM:%d debuggger mode Off", sm); | ||
2625 | reg_mask |= | ||
2626 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_m(); | ||
2627 | reg_val |= | ||
2628 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_off_f(); | ||
2629 | } | ||
2630 | |||
2631 | ops[i].and_n_mask_lo = reg_mask; | ||
2632 | ops[i].value_lo = reg_val; | ||
2633 | i++; | ||
2634 | } | ||
2635 | |||
2636 | err = gr_gk20a_exec_ctx_ops(ch, ops, i, i, 0); | ||
2637 | if (err) | ||
2638 | nvgpu_err(g, "Failed to access register\n"); | ||
2639 | nvgpu_kfree(g, ops); | ||
2640 | return err; | ||
2641 | } | ||
2642 | |||
2643 | int gv11b_gr_record_sm_error_state(struct gk20a *g, u32 gpc, u32 tpc) | ||
2644 | { | ||
2645 | int sm_id; | ||
2646 | struct gr_gk20a *gr = &g->gr; | ||
2647 | u32 offset, sm, sm_per_tpc; | ||
2648 | u32 gpc_tpc_offset; | ||
2649 | |||
2650 | nvgpu_mutex_acquire(&g->dbg_sessions_lock); | ||
2651 | |||
2652 | sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); | ||
2653 | gpc_tpc_offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2654 | gk20a_gr_tpc_offset(g, tpc); | ||
2655 | |||
2656 | sm_id = gr_gpc0_tpc0_sm_cfg_tpc_id_v(gk20a_readl(g, | ||
2657 | gr_gpc0_tpc0_sm_cfg_r() + gpc_tpc_offset)); | ||
2658 | |||
2659 | sm = sm_id % sm_per_tpc; | ||
2660 | |||
2661 | offset = gpc_tpc_offset + gv11b_gr_sm_offset(g, sm); | ||
2662 | |||
2663 | gr->sm_error_states[sm_id].hww_global_esr = gk20a_readl(g, | ||
2664 | gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset); | ||
2665 | |||
2666 | gr->sm_error_states[sm_id].hww_warp_esr = gk20a_readl(g, | ||
2667 | gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset); | ||
2668 | |||
2669 | gr->sm_error_states[sm_id].hww_warp_esr_pc = gk20a_readl(g, | ||
2670 | gr_gpc0_tpc0_sm0_hww_warp_esr_pc_r() + offset); | ||
2671 | |||
2672 | gr->sm_error_states[sm_id].hww_global_esr_report_mask = gk20a_readl(g, | ||
2673 | gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_r() + offset); | ||
2674 | |||
2675 | gr->sm_error_states[sm_id].hww_warp_esr_report_mask = gk20a_readl(g, | ||
2676 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_r() + offset); | ||
2677 | |||
2678 | nvgpu_mutex_release(&g->dbg_sessions_lock); | ||
2679 | |||
2680 | return 0; | ||
2681 | } | ||
2682 | |||
2683 | void gv11b_gr_set_hww_esr_report_mask(struct gk20a *g) | ||
2684 | { | ||
2685 | |||
2686 | /* clear hww */ | ||
2687 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_r(), 0xffffffff); | ||
2688 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_r(), 0xffffffff); | ||
2689 | |||
2690 | /* setup sm warp esr report masks */ | ||
2691 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_warp_esr_report_mask_r(), | ||
2692 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_stack_error_report_f() | | ||
2693 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_api_stack_error_report_f() | | ||
2694 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_pc_wrap_report_f() | | ||
2695 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_pc_report_f() | | ||
2696 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_pc_overflow_report_f() | | ||
2697 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_reg_report_f() | | ||
2698 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_illegal_instr_encoding_report_f() | | ||
2699 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_illegal_instr_param_report_f() | | ||
2700 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_oor_reg_report_f() | | ||
2701 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_oor_addr_report_f() | | ||
2702 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_misaligned_addr_report_f() | | ||
2703 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_invalid_addr_space_report_f() | | ||
2704 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_invalid_const_addr_ldc_report_f() | | ||
2705 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_stack_overflow_report_f() | | ||
2706 | gr_gpc0_tpc0_sm0_hww_warp_esr_report_mask_mmu_fault_report_f()); | ||
2707 | |||
2708 | /* setup sm global esr report mask. vat_alarm_report is not enabled */ | ||
2709 | gk20a_writel(g, gr_gpcs_tpcs_sms_hww_global_esr_report_mask_r(), | ||
2710 | gr_gpc0_tpc0_sm0_hww_global_esr_report_mask_multiple_warp_errors_report_f()); | ||
2711 | } | ||
2712 | |||
2713 | bool gv11b_gr_sm_debugger_attached(struct gk20a *g) | ||
2714 | { | ||
2715 | u32 debugger_mode; | ||
2716 | u32 dbgr_control0 = gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2717 | |||
2718 | /* check if sm debugger is attached. | ||
2719 | * assumption: all SMs will have debug mode enabled/disabled | ||
2720 | * uniformly. | ||
2721 | */ | ||
2722 | debugger_mode = | ||
2723 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_v(dbgr_control0); | ||
2724 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
2725 | "SM Debugger Mode: %d", debugger_mode); | ||
2726 | if (debugger_mode == | ||
2727 | gr_gpc0_tpc0_sm0_dbgr_control0_debugger_mode_on_v()) | ||
2728 | return true; | ||
2729 | |||
2730 | return false; | ||
2731 | } | ||
2732 | |||
2733 | void gv11b_gr_suspend_single_sm(struct gk20a *g, | ||
2734 | u32 gpc, u32 tpc, u32 sm, | ||
2735 | u32 global_esr_mask, bool check_errors) | ||
2736 | { | ||
2737 | int err; | ||
2738 | u32 dbgr_control0; | ||
2739 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2740 | gk20a_gr_tpc_offset(g, tpc) + | ||
2741 | gv11b_gr_sm_offset(g, sm); | ||
2742 | |||
2743 | /* if an SM debugger isn't attached, skip suspend */ | ||
2744 | if (!g->ops.gr.sm_debugger_attached(g)) { | ||
2745 | nvgpu_err(g, | ||
2746 | "SM debugger not attached, skipping suspend!"); | ||
2747 | return; | ||
2748 | } | ||
2749 | |||
2750 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2751 | "suspending gpc:%d, tpc:%d, sm%d", gpc, tpc, sm); | ||
2752 | |||
2753 | /* assert stop trigger. */ | ||
2754 | dbgr_control0 = gk20a_readl(g, | ||
2755 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
2756 | dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f(); | ||
2757 | gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset, | ||
2758 | dbgr_control0); | ||
2759 | |||
2760 | err = g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, | ||
2761 | global_esr_mask, check_errors); | ||
2762 | if (err) { | ||
2763 | nvgpu_err(g, | ||
2764 | "SuspendSm failed"); | ||
2765 | return; | ||
2766 | } | ||
2767 | } | ||
2768 | |||
2769 | void gv11b_gr_suspend_all_sms(struct gk20a *g, | ||
2770 | u32 global_esr_mask, bool check_errors) | ||
2771 | { | ||
2772 | struct gr_gk20a *gr = &g->gr; | ||
2773 | u32 gpc, tpc, sm; | ||
2774 | int err; | ||
2775 | u32 dbgr_control0; | ||
2776 | u32 sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); | ||
2777 | |||
2778 | /* if an SM debugger isn't attached, skip suspend */ | ||
2779 | if (!g->ops.gr.sm_debugger_attached(g)) { | ||
2780 | nvgpu_err(g, | ||
2781 | "SM debugger not attached, skipping suspend!"); | ||
2782 | return; | ||
2783 | } | ||
2784 | |||
2785 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "suspending all sms"); | ||
2786 | |||
2787 | /* assert stop trigger. uniformity assumption: all SMs will have | ||
2788 | * the same state in dbg_control0. | ||
2789 | */ | ||
2790 | dbgr_control0 = | ||
2791 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2792 | dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f(); | ||
2793 | |||
2794 | /* broadcast write */ | ||
2795 | gk20a_writel(g, | ||
2796 | gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0); | ||
2797 | |||
2798 | for (gpc = 0; gpc < gr->gpc_count; gpc++) { | ||
2799 | for (tpc = 0; tpc < gr_gk20a_get_tpc_count(gr, gpc); tpc++) { | ||
2800 | for (sm = 0; sm < sm_per_tpc; sm++) { | ||
2801 | err = g->ops.gr.wait_for_sm_lock_down(g, | ||
2802 | gpc, tpc, sm, | ||
2803 | global_esr_mask, check_errors); | ||
2804 | if (err) { | ||
2805 | nvgpu_err(g, | ||
2806 | "SuspendAllSms failed"); | ||
2807 | return; | ||
2808 | } | ||
2809 | } | ||
2810 | } | ||
2811 | } | ||
2812 | } | ||
2813 | |||
2814 | void gv11b_gr_resume_single_sm(struct gk20a *g, | ||
2815 | u32 gpc, u32 tpc, u32 sm) | ||
2816 | { | ||
2817 | u32 dbgr_control0, dbgr_status0; | ||
2818 | u32 offset; | ||
2819 | /* | ||
2820 | * The following requires some clarification. Despite the fact that both | ||
2821 | * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their | ||
2822 | * names, only one is actually a trigger, and that is the STOP_TRIGGER. | ||
2823 | * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to | ||
2824 | * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0 | ||
2825 | * (_DISABLE) as well. | ||
2826 | |||
2827 | * Advice from the arch group: Disable the stop trigger first, as a | ||
2828 | * separate operation, in order to ensure that the trigger has taken | ||
2829 | * effect, before enabling the run trigger. | ||
2830 | */ | ||
2831 | |||
2832 | offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) + | ||
2833 | gv11b_gr_sm_offset(g, sm); | ||
2834 | |||
2835 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2836 | "resuming gpc:%d, tpc:%d, sm%d", gpc, tpc, sm); | ||
2837 | dbgr_control0 = gk20a_readl(g, | ||
2838 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
2839 | dbgr_status0 = gk20a_readl(g, | ||
2840 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset); | ||
2841 | |||
2842 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2843 | "before stop trigger disable: " | ||
2844 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2845 | dbgr_control0, dbgr_status0); | ||
2846 | |||
2847 | /*De-assert stop trigger */ | ||
2848 | dbgr_control0 = set_field(dbgr_control0, | ||
2849 | gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_m(), | ||
2850 | gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_disable_f()); | ||
2851 | gk20a_writel(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + | ||
2852 | offset, dbgr_control0); | ||
2853 | |||
2854 | dbgr_control0 = gk20a_readl(g, | ||
2855 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
2856 | dbgr_status0 = gk20a_readl(g, | ||
2857 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset); | ||
2858 | |||
2859 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2860 | "before run trigger: " | ||
2861 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2862 | dbgr_control0, dbgr_status0); | ||
2863 | /* Run trigger */ | ||
2864 | dbgr_control0 |= | ||
2865 | gr_gpc0_tpc0_sm0_dbgr_control0_run_trigger_task_f(); | ||
2866 | gk20a_writel(g, | ||
2867 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + | ||
2868 | offset, dbgr_control0); | ||
2869 | |||
2870 | dbgr_control0 = gk20a_readl(g, | ||
2871 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
2872 | dbgr_status0 = gk20a_readl(g, | ||
2873 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset); | ||
2874 | /* run trigger is not sticky bit. SM clears it immediately */ | ||
2875 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2876 | "after run trigger: " | ||
2877 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2878 | dbgr_control0, dbgr_status0); | ||
2879 | |||
2880 | } | ||
2881 | |||
2882 | void gv11b_gr_resume_all_sms(struct gk20a *g) | ||
2883 | { | ||
2884 | u32 dbgr_control0, dbgr_status0; | ||
2885 | /* | ||
2886 | * The following requires some clarification. Despite the fact that both | ||
2887 | * RUN_TRIGGER and STOP_TRIGGER have the word "TRIGGER" in their | ||
2888 | * names, only one is actually a trigger, and that is the STOP_TRIGGER. | ||
2889 | * Merely writing a 1(_TASK) to the RUN_TRIGGER is not sufficient to | ||
2890 | * resume the gpu - the _STOP_TRIGGER must explicitly be set to 0 | ||
2891 | * (_DISABLE) as well. | ||
2892 | |||
2893 | * Advice from the arch group: Disable the stop trigger first, as a | ||
2894 | * separate operation, in order to ensure that the trigger has taken | ||
2895 | * effect, before enabling the run trigger. | ||
2896 | */ | ||
2897 | |||
2898 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "resuming all sms"); | ||
2899 | |||
2900 | /* Read from unicast registers */ | ||
2901 | dbgr_control0 = | ||
2902 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2903 | dbgr_status0 = | ||
2904 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r()); | ||
2905 | |||
2906 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2907 | "before stop trigger disable: " | ||
2908 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2909 | dbgr_control0, dbgr_status0); | ||
2910 | |||
2911 | dbgr_control0 = set_field(dbgr_control0, | ||
2912 | gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_m(), | ||
2913 | gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_disable_f()); | ||
2914 | /* Write to broadcast registers */ | ||
2915 | gk20a_writel(g, | ||
2916 | gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0); | ||
2917 | |||
2918 | /* Read from unicast registers */ | ||
2919 | dbgr_control0 = | ||
2920 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2921 | dbgr_status0 = | ||
2922 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r()); | ||
2923 | |||
2924 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2925 | "before run trigger: " | ||
2926 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2927 | dbgr_control0, dbgr_status0); | ||
2928 | /* Run trigger */ | ||
2929 | dbgr_control0 |= | ||
2930 | gr_gpc0_tpc0_sm0_dbgr_control0_run_trigger_task_f(); | ||
2931 | /* Write to broadcast registers */ | ||
2932 | gk20a_writel(g, | ||
2933 | gr_gpcs_tpcs_sms_dbgr_control0_r(), dbgr_control0); | ||
2934 | |||
2935 | /* Read from unicast registers */ | ||
2936 | dbgr_control0 = | ||
2937 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r()); | ||
2938 | dbgr_status0 = | ||
2939 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_status0_r()); | ||
2940 | /* run trigger is not sticky bit. SM clears it immediately */ | ||
2941 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
2942 | "after run trigger: " | ||
2943 | "dbgr_control0 = 0x%x dbgr_status0: 0x%x", | ||
2944 | dbgr_control0, dbgr_status0); | ||
2945 | } | ||
2946 | |||
2947 | int gv11b_gr_resume_from_pause(struct gk20a *g) | ||
2948 | { | ||
2949 | int err = 0; | ||
2950 | u32 reg_val; | ||
2951 | |||
2952 | /* Clear the pause mask to tell the GPU we want to resume everyone */ | ||
2953 | gk20a_writel(g, gr_gpcs_tpcs_sms_dbgr_bpt_pause_mask_0_r(), 0); | ||
2954 | |||
2955 | /* explicitly re-enable forwarding of SM interrupts upon any resume */ | ||
2956 | reg_val = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_en_r()); | ||
2957 | reg_val |= gr_gpc0_tpc0_tpccs_tpc_exception_en_sm_enabled_f(); | ||
2958 | |||
2959 | gk20a_writel(g, gr_gpcs_tpcs_tpccs_tpc_exception_en_r(), reg_val); | ||
2960 | |||
2961 | g->ops.gr.resume_all_sms(g); | ||
2962 | |||
2963 | return err; | ||
2964 | } | ||
2965 | |||
2966 | u32 gv11b_gr_get_sm_hww_warp_esr(struct gk20a *g, | ||
2967 | u32 gpc, u32 tpc, u32 sm) | ||
2968 | { | ||
2969 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2970 | gk20a_gr_tpc_offset(g, tpc) + | ||
2971 | gv11b_gr_sm_offset(g, sm); | ||
2972 | |||
2973 | u32 hww_warp_esr = gk20a_readl(g, | ||
2974 | gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset); | ||
2975 | return hww_warp_esr; | ||
2976 | } | ||
2977 | |||
2978 | u32 gv11b_gr_get_sm_hww_global_esr(struct gk20a *g, | ||
2979 | u32 gpc, u32 tpc, u32 sm) | ||
2980 | { | ||
2981 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + | ||
2982 | gk20a_gr_tpc_offset(g, tpc) + | ||
2983 | gv11b_gr_sm_offset(g, sm); | ||
2984 | |||
2985 | u32 hww_global_esr = gk20a_readl(g, | ||
2986 | gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset); | ||
2987 | |||
2988 | return hww_global_esr; | ||
2989 | } | ||
2990 | |||
2991 | u32 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g) | ||
2992 | { | ||
2993 | /* | ||
2994 | * These three interrupts don't require locking down the SM. They can | ||
2995 | * be handled by usermode clients as they aren't fatal. Additionally, | ||
2996 | * usermode clients may wish to allow some warps to execute while others | ||
2997 | * are at breakpoints, as opposed to fatal errors where all warps should | ||
2998 | * halt. | ||
2999 | */ | ||
3000 | u32 global_esr_mask = | ||
3001 | gr_gpc0_tpc0_sm0_hww_global_esr_bpt_int_pending_f() | | ||
3002 | gr_gpc0_tpc0_sm0_hww_global_esr_bpt_pause_pending_f() | | ||
3003 | gr_gpc0_tpc0_sm0_hww_global_esr_single_step_complete_pending_f(); | ||
3004 | |||
3005 | return global_esr_mask; | ||
3006 | } | ||
3007 | |||
3008 | static void gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(struct gk20a *g, | ||
3009 | u32 offset, bool timeout) | ||
3010 | { | ||
3011 | u64 warps_valid = 0, warps_paused = 0, warps_trapped = 0; | ||
3012 | u32 dbgr_control0 = gk20a_readl(g, | ||
3013 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
3014 | u32 dbgr_status0 = gk20a_readl(g, | ||
3015 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset); | ||
3016 | /* 64 bit read */ | ||
3017 | warps_valid = | ||
3018 | (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_warp_valid_mask_1_r() + | ||
3019 | offset) << 32; | ||
3020 | warps_valid |= gk20a_readl(g, | ||
3021 | gr_gpc0_tpc0_sm0_warp_valid_mask_0_r() + offset); | ||
3022 | |||
3023 | /* 64 bit read */ | ||
3024 | warps_paused = | ||
3025 | (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_1_r() + | ||
3026 | offset) << 32; | ||
3027 | warps_paused |= gk20a_readl(g, | ||
3028 | gr_gpc0_tpc0_sm0_dbgr_bpt_pause_mask_0_r() + offset); | ||
3029 | |||
3030 | /* 64 bit read */ | ||
3031 | warps_trapped = | ||
3032 | (u64)gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_1_r() + | ||
3033 | offset) << 32; | ||
3034 | warps_trapped |= gk20a_readl(g, | ||
3035 | gr_gpc0_tpc0_sm0_dbgr_bpt_trap_mask_0_r() + offset); | ||
3036 | if (timeout) | ||
3037 | nvgpu_err(g, | ||
3038 | "STATUS0=0x%x CONTROL0=0x%x VALID_MASK=0x%llx " | ||
3039 | "PAUSE_MASK=0x%llx TRAP_MASK=0x%llx\n", | ||
3040 | dbgr_status0, dbgr_control0, warps_valid, | ||
3041 | warps_paused, warps_trapped); | ||
3042 | else | ||
3043 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3044 | "STATUS0=0x%x CONTROL0=0x%x VALID_MASK=0x%llx " | ||
3045 | "PAUSE_MASK=0x%llx TRAP_MASK=0x%llx\n", | ||
3046 | dbgr_status0, dbgr_control0, warps_valid, | ||
3047 | warps_paused, warps_trapped); | ||
3048 | } | ||
3049 | |||
3050 | int gv11b_gr_wait_for_sm_lock_down(struct gk20a *g, | ||
3051 | u32 gpc, u32 tpc, u32 sm, | ||
3052 | u32 global_esr_mask, bool check_errors) | ||
3053 | { | ||
3054 | bool locked_down; | ||
3055 | bool no_error_pending; | ||
3056 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
3057 | bool mmu_debug_mode_enabled = g->ops.fb.is_debug_mode_enabled(g); | ||
3058 | u32 dbgr_status0 = 0; | ||
3059 | u32 warp_esr, global_esr; | ||
3060 | struct nvgpu_timeout timeout; | ||
3061 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + | ||
3062 | gk20a_gr_tpc_offset(g, tpc) + | ||
3063 | gv11b_gr_sm_offset(g, sm); | ||
3064 | |||
3065 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3066 | "GPC%d TPC%d: locking down SM%d", gpc, tpc, sm); | ||
3067 | |||
3068 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
3069 | NVGPU_TIMER_CPU_TIMER); | ||
3070 | |||
3071 | /* wait for the sm to lock down */ | ||
3072 | do { | ||
3073 | global_esr = g->ops.gr.get_sm_hww_global_esr(g, gpc, tpc, sm); | ||
3074 | dbgr_status0 = gk20a_readl(g, | ||
3075 | gr_gpc0_tpc0_sm0_dbgr_status0_r() + offset); | ||
3076 | |||
3077 | warp_esr = g->ops.gr.get_sm_hww_warp_esr(g, gpc, tpc, sm); | ||
3078 | |||
3079 | locked_down = | ||
3080 | (gr_gpc0_tpc0_sm0_dbgr_status0_locked_down_v(dbgr_status0) == | ||
3081 | gr_gpc0_tpc0_sm0_dbgr_status0_locked_down_true_v()); | ||
3082 | no_error_pending = | ||
3083 | check_errors && | ||
3084 | (gr_gpc0_tpc0_sm0_hww_warp_esr_error_v(warp_esr) == | ||
3085 | gr_gpc0_tpc0_sm0_hww_warp_esr_error_none_v()) && | ||
3086 | ((global_esr & global_esr_mask) == 0); | ||
3087 | |||
3088 | if (locked_down) { | ||
3089 | /* | ||
3090 | * if SM reports locked down, it means that SM is idle and | ||
3091 | * trapped and also that one of the these conditions are true | ||
3092 | * 1) sm is nonempty and all valid warps are paused | ||
3093 | * 2) sm is empty and held in trapped state due to stop trigger | ||
3094 | * 3) sm is nonempty and some warps are not paused, but are | ||
3095 | * instead held at RTT due to an "active" stop trigger | ||
3096 | * Check for Paused warp mask != Valid | ||
3097 | * warp mask after SM reports it is locked down in order to | ||
3098 | * distinguish case 1 from case 3. When case 3 is detected, | ||
3099 | * it implies a misprogrammed trap handler code, as all warps | ||
3100 | * in the handler must promise to BPT.PAUSE instead of RTT | ||
3101 | * whenever SR64 read in trap mode indicates stop trigger | ||
3102 | * is asserted. | ||
3103 | */ | ||
3104 | gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(g, | ||
3105 | offset, false); | ||
3106 | } | ||
3107 | |||
3108 | if (locked_down || no_error_pending) { | ||
3109 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3110 | "GPC%d TPC%d: locked down SM%d", gpc, tpc, sm); | ||
3111 | return 0; | ||
3112 | } | ||
3113 | |||
3114 | /* if an mmu fault is pending and mmu debug mode is not | ||
3115 | * enabled, the sm will never lock down. | ||
3116 | */ | ||
3117 | if (!mmu_debug_mode_enabled && | ||
3118 | (g->ops.mm.mmu_fault_pending(g))) { | ||
3119 | nvgpu_err(g, | ||
3120 | "GPC%d TPC%d: mmu fault pending," | ||
3121 | " SM%d will never lock down!", gpc, tpc, sm); | ||
3122 | return -EFAULT; | ||
3123 | } | ||
3124 | |||
3125 | nvgpu_usleep_range(delay, delay * 2); | ||
3126 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
3127 | } while (!nvgpu_timeout_expired(&timeout)); | ||
3128 | |||
3129 | nvgpu_err(g, "GPC%d TPC%d: timed out while trying to " | ||
3130 | "lock down SM%d", gpc, tpc, sm); | ||
3131 | gv11b_gr_sm_dump_warp_bpt_pause_trap_mask_regs(g, offset, true); | ||
3132 | |||
3133 | return -ETIMEDOUT; | ||
3134 | } | ||
3135 | |||
3136 | int gv11b_gr_lock_down_sm(struct gk20a *g, | ||
3137 | u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask, | ||
3138 | bool check_errors) | ||
3139 | { | ||
3140 | u32 dbgr_control0; | ||
3141 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) + | ||
3142 | gv11b_gr_sm_offset(g, sm); | ||
3143 | |||
3144 | gk20a_dbg(gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3145 | "GPC%d TPC%d SM%d: assert stop trigger", gpc, tpc, sm); | ||
3146 | |||
3147 | /* assert stop trigger */ | ||
3148 | dbgr_control0 = | ||
3149 | gk20a_readl(g, gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset); | ||
3150 | dbgr_control0 |= gr_gpc0_tpc0_sm0_dbgr_control0_stop_trigger_enable_f(); | ||
3151 | gk20a_writel(g, | ||
3152 | gr_gpc0_tpc0_sm0_dbgr_control0_r() + offset, dbgr_control0); | ||
3153 | |||
3154 | return g->ops.gr.wait_for_sm_lock_down(g, gpc, tpc, sm, global_esr_mask, | ||
3155 | check_errors); | ||
3156 | } | ||
3157 | |||
3158 | void gv11b_gr_clear_sm_hww(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
3159 | u32 global_esr) | ||
3160 | { | ||
3161 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc) + | ||
3162 | gv11b_gr_sm_offset(g, sm); | ||
3163 | |||
3164 | gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + offset, | ||
3165 | global_esr); | ||
3166 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
3167 | "Cleared HWW global esr, current reg val: 0x%x", | ||
3168 | gk20a_readl(g, gr_gpc0_tpc0_sm0_hww_global_esr_r() + | ||
3169 | offset)); | ||
3170 | |||
3171 | gk20a_writel(g, gr_gpc0_tpc0_sm0_hww_warp_esr_r() + offset, 0); | ||
3172 | gk20a_dbg(gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
3173 | "Cleared HWW warp esr, current reg val: 0x%x", | ||
3174 | gk20a_readl(g, gr_gpc0_tpc0_sm0_hww_warp_esr_r() + | ||
3175 | offset)); | ||
3176 | } | ||
3177 | |||
3178 | int gr_gv11b_handle_tpc_mpc_exception(struct gk20a *g, | ||
3179 | u32 gpc, u32 tpc, bool *post_event) | ||
3180 | { | ||
3181 | u32 esr; | ||
3182 | u32 offset = gk20a_gr_gpc_offset(g, gpc) + gk20a_gr_tpc_offset(g, tpc); | ||
3183 | u32 tpc_exception = gk20a_readl(g, gr_gpc0_tpc0_tpccs_tpc_exception_r() | ||
3184 | + offset); | ||
3185 | |||
3186 | if (!(tpc_exception & gr_gpc0_tpc0_tpccs_tpc_exception_mpc_m())) | ||
3187 | return 0; | ||
3188 | |||
3189 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3190 | "GPC%d TPC%d MPC exception", gpc, tpc); | ||
3191 | |||
3192 | esr = gk20a_readl(g, gr_gpc0_tpc0_mpc_hww_esr_r() + offset); | ||
3193 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, "mpc hww esr 0x%08x", esr); | ||
3194 | |||
3195 | esr = gk20a_readl(g, gr_gpc0_tpc0_mpc_hww_esr_info_r() + offset); | ||
3196 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_gpu_dbg, | ||
3197 | "mpc hww esr info: veid 0x%08x", | ||
3198 | gr_gpc0_tpc0_mpc_hww_esr_info_veid_v(esr)); | ||
3199 | |||
3200 | gk20a_writel(g, gr_gpc0_tpc0_mpc_hww_esr_r() + offset, | ||
3201 | gr_gpc0_tpc0_mpc_hww_esr_reset_trigger_f()); | ||
3202 | |||
3203 | return 0; | ||
3204 | } | ||
3205 | |||
3206 | static const u32 _num_ovr_perf_regs = 20; | ||
3207 | static u32 _ovr_perf_regs[20] = { 0, }; | ||
3208 | |||
3209 | void gv11b_gr_init_ovr_sm_dsm_perf(void) | ||
3210 | { | ||
3211 | if (_ovr_perf_regs[0] != 0) | ||
3212 | return; | ||
3213 | |||
3214 | _ovr_perf_regs[0] = gr_egpc0_etpc0_sm_dsm_perf_counter_control_sel0_r(); | ||
3215 | _ovr_perf_regs[1] = gr_egpc0_etpc0_sm_dsm_perf_counter_control_sel1_r(); | ||
3216 | _ovr_perf_regs[2] = gr_egpc0_etpc0_sm_dsm_perf_counter_control0_r(); | ||
3217 | _ovr_perf_regs[3] = gr_egpc0_etpc0_sm_dsm_perf_counter_control1_r(); | ||
3218 | _ovr_perf_regs[4] = gr_egpc0_etpc0_sm_dsm_perf_counter_control2_r(); | ||
3219 | _ovr_perf_regs[5] = gr_egpc0_etpc0_sm_dsm_perf_counter_control3_r(); | ||
3220 | _ovr_perf_regs[6] = gr_egpc0_etpc0_sm_dsm_perf_counter_control4_r(); | ||
3221 | _ovr_perf_regs[7] = gr_egpc0_etpc0_sm_dsm_perf_counter_control5_r(); | ||
3222 | _ovr_perf_regs[8] = gr_egpc0_etpc0_sm_dsm_perf_counter0_control_r(); | ||
3223 | _ovr_perf_regs[9] = gr_egpc0_etpc0_sm_dsm_perf_counter1_control_r(); | ||
3224 | _ovr_perf_regs[10] = gr_egpc0_etpc0_sm_dsm_perf_counter2_control_r(); | ||
3225 | _ovr_perf_regs[11] = gr_egpc0_etpc0_sm_dsm_perf_counter3_control_r(); | ||
3226 | _ovr_perf_regs[12] = gr_egpc0_etpc0_sm_dsm_perf_counter4_control_r(); | ||
3227 | _ovr_perf_regs[13] = gr_egpc0_etpc0_sm_dsm_perf_counter5_control_r(); | ||
3228 | _ovr_perf_regs[14] = gr_egpc0_etpc0_sm_dsm_perf_counter6_control_r(); | ||
3229 | _ovr_perf_regs[15] = gr_egpc0_etpc0_sm_dsm_perf_counter7_control_r(); | ||
3230 | |||
3231 | _ovr_perf_regs[16] = gr_egpc0_etpc0_sm0_dsm_perf_counter4_r(); | ||
3232 | _ovr_perf_regs[17] = gr_egpc0_etpc0_sm0_dsm_perf_counter5_r(); | ||
3233 | _ovr_perf_regs[18] = gr_egpc0_etpc0_sm0_dsm_perf_counter6_r(); | ||
3234 | _ovr_perf_regs[19] = gr_egpc0_etpc0_sm0_dsm_perf_counter7_r(); | ||
3235 | } | ||
3236 | |||
3237 | /* Following are the blocks of registers that the ucode | ||
3238 | * stores in the extended region. | ||
3239 | */ | ||
3240 | /* == ctxsw_extended_sm_dsm_perf_counter_register_stride_v() ? */ | ||
3241 | static const u32 _num_sm_dsm_perf_regs; | ||
3242 | /* == ctxsw_extended_sm_dsm_perf_counter_control_register_stride_v() ?*/ | ||
3243 | static const u32 _num_sm_dsm_perf_ctrl_regs = 2; | ||
3244 | static u32 *_sm_dsm_perf_regs; | ||
3245 | static u32 _sm_dsm_perf_ctrl_regs[2]; | ||
3246 | |||
3247 | void gv11b_gr_init_sm_dsm_reg_info(void) | ||
3248 | { | ||
3249 | if (_sm_dsm_perf_ctrl_regs[0] != 0) | ||
3250 | return; | ||
3251 | |||
3252 | _sm_dsm_perf_ctrl_regs[0] = | ||
3253 | gr_egpc0_etpc0_sm_dsm_perf_counter_control0_r(); | ||
3254 | _sm_dsm_perf_ctrl_regs[1] = | ||
3255 | gr_egpc0_etpc0_sm_dsm_perf_counter_control5_r(); | ||
3256 | } | ||
3257 | |||
3258 | void gv11b_gr_get_sm_dsm_perf_regs(struct gk20a *g, | ||
3259 | u32 *num_sm_dsm_perf_regs, | ||
3260 | u32 **sm_dsm_perf_regs, | ||
3261 | u32 *perf_register_stride) | ||
3262 | { | ||
3263 | *num_sm_dsm_perf_regs = _num_sm_dsm_perf_regs; | ||
3264 | *sm_dsm_perf_regs = _sm_dsm_perf_regs; | ||
3265 | *perf_register_stride = | ||
3266 | ctxsw_prog_extended_sm_dsm_perf_counter_register_stride_v(); | ||
3267 | } | ||
3268 | |||
3269 | void gv11b_gr_get_sm_dsm_perf_ctrl_regs(struct gk20a *g, | ||
3270 | u32 *num_sm_dsm_perf_ctrl_regs, | ||
3271 | u32 **sm_dsm_perf_ctrl_regs, | ||
3272 | u32 *ctrl_register_stride) | ||
3273 | { | ||
3274 | *num_sm_dsm_perf_ctrl_regs = _num_sm_dsm_perf_ctrl_regs; | ||
3275 | *sm_dsm_perf_ctrl_regs = _sm_dsm_perf_ctrl_regs; | ||
3276 | *ctrl_register_stride = | ||
3277 | ctxsw_prog_extended_sm_dsm_perf_counter_control_register_stride_v(); | ||
3278 | } | ||
3279 | |||
3280 | void gv11b_gr_get_ovr_perf_regs(struct gk20a *g, u32 *num_ovr_perf_regs, | ||
3281 | u32 **ovr_perf_regs) | ||
3282 | { | ||
3283 | *num_ovr_perf_regs = _num_ovr_perf_regs; | ||
3284 | *ovr_perf_regs = _ovr_perf_regs; | ||
3285 | } | ||
3286 | |||
3287 | void gv11b_gr_access_smpc_reg(struct gk20a *g, u32 quad, u32 offset) | ||
3288 | { | ||
3289 | u32 reg_val; | ||
3290 | u32 quad_ctrl; | ||
3291 | u32 half_ctrl; | ||
3292 | u32 tpc, gpc; | ||
3293 | u32 gpc_tpc_addr; | ||
3294 | u32 gpc_tpc_stride; | ||
3295 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3296 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, | ||
3297 | GPU_LIT_TPC_IN_GPC_STRIDE); | ||
3298 | |||
3299 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, "offset=0x%x", offset); | ||
3300 | |||
3301 | gpc = pri_get_gpc_num(g, offset); | ||
3302 | gpc_tpc_addr = pri_gpccs_addr_mask(offset); | ||
3303 | tpc = g->ops.gr.get_tpc_num(g, gpc_tpc_addr); | ||
3304 | |||
3305 | quad_ctrl = quad & 0x1; /* first bit tells us quad */ | ||
3306 | half_ctrl = (quad >> 1) & 0x1; /* second bit tells us half */ | ||
3307 | |||
3308 | gpc_tpc_stride = gpc * gpc_stride + tpc * tpc_in_gpc_stride; | ||
3309 | gpc_tpc_addr = gr_gpc0_tpc0_sm_halfctl_ctrl_r() + gpc_tpc_stride; | ||
3310 | |||
3311 | /* read from unicast reg */ | ||
3312 | reg_val = gk20a_readl(g, gpc_tpc_addr); | ||
3313 | reg_val = set_field(reg_val, | ||
3314 | gr_gpcs_tpcs_sm_halfctl_ctrl_sctl_read_quad_ctl_m(), | ||
3315 | gr_gpcs_tpcs_sm_halfctl_ctrl_sctl_read_quad_ctl_f(quad_ctrl)); | ||
3316 | |||
3317 | /* write to broadcast reg */ | ||
3318 | gk20a_writel(g, gr_gpcs_tpcs_sm_halfctl_ctrl_r(), reg_val); | ||
3319 | |||
3320 | gpc_tpc_addr = gr_gpc0_tpc0_sm_debug_sfe_control_r() + gpc_tpc_stride; | ||
3321 | reg_val = gk20a_readl(g, gpc_tpc_addr); | ||
3322 | reg_val = set_field(reg_val, | ||
3323 | gr_gpcs_tpcs_sm_debug_sfe_control_read_half_ctl_m(), | ||
3324 | gr_gpcs_tpcs_sm_debug_sfe_control_read_half_ctl_f(half_ctrl)); | ||
3325 | |||
3326 | /* write to broadcast reg */ | ||
3327 | gk20a_writel(g, gr_gpcs_tpcs_sm_debug_sfe_control_r(), reg_val); | ||
3328 | } | ||
3329 | |||
3330 | static bool pri_is_egpc_addr_shared(struct gk20a *g, u32 addr) | ||
3331 | { | ||
3332 | u32 egpc_shared_base = EGPC_PRI_SHARED_BASE; | ||
3333 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3334 | |||
3335 | return (addr >= egpc_shared_base) && | ||
3336 | (addr < egpc_shared_base + gpc_stride); | ||
3337 | } | ||
3338 | |||
3339 | bool gv11b_gr_pri_is_egpc_addr(struct gk20a *g, u32 addr) | ||
3340 | { | ||
3341 | u32 egpc_base = g->ops.gr.get_egpc_base(g); | ||
3342 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3343 | u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); | ||
3344 | |||
3345 | return ((addr >= egpc_base) && | ||
3346 | (addr < egpc_base + num_gpcs * gpc_stride)) || | ||
3347 | pri_is_egpc_addr_shared(g, addr); | ||
3348 | } | ||
3349 | |||
3350 | static inline u32 pri_smpc_in_etpc_addr_mask(struct gk20a *g, u32 addr) | ||
3351 | { | ||
3352 | u32 smpc_stride = nvgpu_get_litter_value(g, | ||
3353 | GPU_LIT_SMPC_PRI_STRIDE); | ||
3354 | |||
3355 | return (addr & (smpc_stride - 1)); | ||
3356 | } | ||
3357 | |||
3358 | static u32 pri_smpc_ext_addr(struct gk20a *g, u32 sm_offset, u32 gpc_num, | ||
3359 | u32 tpc_num, u32 sm_num) | ||
3360 | { | ||
3361 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3362 | u32 tpc_in_gpc_base = nvgpu_get_litter_value(g, | ||
3363 | GPU_LIT_TPC_IN_GPC_BASE); | ||
3364 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, | ||
3365 | GPU_LIT_TPC_IN_GPC_STRIDE); | ||
3366 | u32 egpc_base = g->ops.gr.get_egpc_base(g); | ||
3367 | u32 smpc_unique_base = nvgpu_get_litter_value(g, | ||
3368 | GPU_LIT_SMPC_PRI_UNIQUE_BASE); | ||
3369 | u32 smpc_stride = nvgpu_get_litter_value(g, | ||
3370 | GPU_LIT_SMPC_PRI_STRIDE); | ||
3371 | |||
3372 | return (egpc_base + (gpc_num * gpc_stride) + tpc_in_gpc_base + | ||
3373 | (tpc_num * tpc_in_gpc_stride) + | ||
3374 | (sm_num * smpc_stride) + | ||
3375 | (smpc_unique_base + sm_offset)); | ||
3376 | } | ||
3377 | |||
3378 | static bool pri_is_smpc_addr_in_etpc_shared(struct gk20a *g, u32 addr) | ||
3379 | { | ||
3380 | u32 smpc_shared_base = nvgpu_get_litter_value(g, | ||
3381 | GPU_LIT_SMPC_PRI_SHARED_BASE); | ||
3382 | u32 smpc_stride = nvgpu_get_litter_value(g, | ||
3383 | GPU_LIT_SMPC_PRI_STRIDE); | ||
3384 | |||
3385 | return (addr >= smpc_shared_base) && | ||
3386 | (addr < smpc_shared_base + smpc_stride); | ||
3387 | } | ||
3388 | |||
3389 | bool gv11b_gr_pri_is_etpc_addr(struct gk20a *g, u32 addr) | ||
3390 | { | ||
3391 | u32 egpc_addr = 0; | ||
3392 | |||
3393 | if (g->ops.gr.is_egpc_addr(g, addr)) { | ||
3394 | egpc_addr = pri_gpccs_addr_mask(addr); | ||
3395 | if (g->ops.gr.is_tpc_addr(g, egpc_addr)) | ||
3396 | return true; | ||
3397 | } | ||
3398 | |||
3399 | return false; | ||
3400 | } | ||
3401 | |||
3402 | static u32 pri_get_egpc_num(struct gk20a *g, u32 addr) | ||
3403 | { | ||
3404 | u32 i, start; | ||
3405 | u32 egpc_base = g->ops.gr.get_egpc_base(g); | ||
3406 | u32 num_gpcs = nvgpu_get_litter_value(g, GPU_LIT_NUM_GPCS); | ||
3407 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3408 | |||
3409 | for (i = 0; i < num_gpcs; i++) { | ||
3410 | start = egpc_base + (i * gpc_stride); | ||
3411 | if ((addr >= start) && (addr < (start + gpc_stride))) | ||
3412 | return i; | ||
3413 | } | ||
3414 | return 0; | ||
3415 | } | ||
3416 | |||
3417 | static u32 pri_egpc_addr(struct gk20a *g, u32 addr, u32 gpc) | ||
3418 | { | ||
3419 | u32 egpc_base = g->ops.gr.get_egpc_base(g); | ||
3420 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3421 | |||
3422 | return egpc_base + (gpc * gpc_stride) + addr; | ||
3423 | } | ||
3424 | |||
3425 | static u32 pri_etpc_addr(struct gk20a *g, u32 addr, u32 gpc, u32 tpc) | ||
3426 | { | ||
3427 | u32 egpc_base = g->ops.gr.get_egpc_base(g); | ||
3428 | u32 gpc_stride = nvgpu_get_litter_value(g, GPU_LIT_GPC_STRIDE); | ||
3429 | u32 tpc_in_gpc_base = nvgpu_get_litter_value(g, | ||
3430 | GPU_LIT_TPC_IN_GPC_BASE); | ||
3431 | u32 tpc_in_gpc_stride = nvgpu_get_litter_value(g, | ||
3432 | GPU_LIT_TPC_IN_GPC_STRIDE); | ||
3433 | |||
3434 | return egpc_base + (gpc * gpc_stride) + | ||
3435 | tpc_in_gpc_base + (tpc * tpc_in_gpc_stride) + | ||
3436 | addr; | ||
3437 | } | ||
3438 | |||
3439 | void gv11b_gr_get_egpc_etpc_num(struct gk20a *g, u32 addr, | ||
3440 | u32 *egpc_num, u32 *etpc_num) | ||
3441 | { | ||
3442 | u32 egpc_addr = 0; | ||
3443 | |||
3444 | *egpc_num = pri_get_egpc_num(g, addr); | ||
3445 | egpc_addr = pri_gpccs_addr_mask(addr); | ||
3446 | *etpc_num = g->ops.gr.get_tpc_num(g, egpc_addr); | ||
3447 | |||
3448 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
3449 | "egpc_num = %d etpc_num = %d", *egpc_num, *etpc_num); | ||
3450 | } | ||
3451 | |||
3452 | int gv11b_gr_decode_egpc_addr(struct gk20a *g, u32 addr, int *addr_type, | ||
3453 | u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags) | ||
3454 | { | ||
3455 | u32 gpc_addr; | ||
3456 | u32 tpc_addr; | ||
3457 | |||
3458 | if (g->ops.gr.is_egpc_addr(g, addr)) { | ||
3459 | nvgpu_log_info(g, "addr=0x%x is egpc", addr); | ||
3460 | |||
3461 | *addr_type = CTXSW_ADDR_TYPE_EGPC; | ||
3462 | gpc_addr = pri_gpccs_addr_mask(addr); | ||
3463 | if (pri_is_egpc_addr_shared(g, addr)) { | ||
3464 | *broadcast_flags |= PRI_BROADCAST_FLAGS_EGPC; | ||
3465 | *gpc_num = 0; | ||
3466 | nvgpu_log_info(g, "shared egpc"); | ||
3467 | } else { | ||
3468 | *gpc_num = pri_get_egpc_num(g, addr); | ||
3469 | nvgpu_log_info(g, "gpc=0x%x", *gpc_num); | ||
3470 | } | ||
3471 | if (g->ops.gr.is_tpc_addr(g, gpc_addr)) { | ||
3472 | nvgpu_log_info(g, "addr=0x%x is etpc", addr); | ||
3473 | *addr_type = CTXSW_ADDR_TYPE_ETPC; | ||
3474 | if (pri_is_tpc_addr_shared(g, gpc_addr)) { | ||
3475 | *broadcast_flags |= PRI_BROADCAST_FLAGS_ETPC; | ||
3476 | *tpc_num = 0; | ||
3477 | nvgpu_log_info(g, "shared etpc"); | ||
3478 | } else { | ||
3479 | *tpc_num = g->ops.gr.get_tpc_num(g, gpc_addr); | ||
3480 | nvgpu_log_info(g, "tpc=0x%x", *tpc_num); | ||
3481 | } | ||
3482 | tpc_addr = pri_tpccs_addr_mask(addr); | ||
3483 | if (pri_is_smpc_addr_in_etpc_shared(g, tpc_addr)) | ||
3484 | *broadcast_flags |= PRI_BROADCAST_FLAGS_SMPC; | ||
3485 | } | ||
3486 | |||
3487 | nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, | ||
3488 | "addr_type = %d, broadcast_flags = %#08x", | ||
3489 | *addr_type, *broadcast_flags); | ||
3490 | return 0; | ||
3491 | } | ||
3492 | return -EINVAL; | ||
3493 | } | ||
3494 | |||
3495 | static void gv11b_gr_update_priv_addr_table_smpc(struct gk20a *g, u32 gpc_num, | ||
3496 | u32 tpc_num, u32 addr, | ||
3497 | u32 *priv_addr_table, u32 *t) | ||
3498 | { | ||
3499 | u32 sm_per_tpc, sm_num; | ||
3500 | |||
3501 | nvgpu_log_info(g, "broadcast flags smpc"); | ||
3502 | |||
3503 | sm_per_tpc = nvgpu_get_litter_value(g, GPU_LIT_NUM_SM_PER_TPC); | ||
3504 | for (sm_num = 0; sm_num < sm_per_tpc; sm_num++) { | ||
3505 | priv_addr_table[*t] = pri_smpc_ext_addr(g, | ||
3506 | pri_smpc_in_etpc_addr_mask(g, addr), | ||
3507 | gpc_num, tpc_num, sm_num); | ||
3508 | nvgpu_log_info(g, "priv_addr_table[%d]:%#08x", | ||
3509 | *t, priv_addr_table[*t]); | ||
3510 | (*t)++; | ||
3511 | } | ||
3512 | } | ||
3513 | |||
3514 | void gv11b_gr_egpc_etpc_priv_addr_table(struct gk20a *g, u32 addr, | ||
3515 | u32 gpc, u32 broadcast_flags, u32 *priv_addr_table, u32 *t) | ||
3516 | { | ||
3517 | u32 gpc_num, tpc_num; | ||
3518 | |||
3519 | nvgpu_log_info(g, "addr=0x%x", addr); | ||
3520 | |||
3521 | /* The GPC/TPC unicast registers are included in the compressed PRI | ||
3522 | * tables. Convert a GPC/TPC broadcast address to unicast addresses so | ||
3523 | * that we can look up the offsets. | ||
3524 | */ | ||
3525 | if (broadcast_flags & PRI_BROADCAST_FLAGS_EGPC) { | ||
3526 | nvgpu_log_info(g, "broadcast flags egpc"); | ||
3527 | for (gpc_num = 0; gpc_num < g->gr.gpc_count; gpc_num++) { | ||
3528 | |||
3529 | if (broadcast_flags & PRI_BROADCAST_FLAGS_ETPC) { | ||
3530 | nvgpu_log_info(g, "broadcast flags etpc"); | ||
3531 | for (tpc_num = 0; | ||
3532 | tpc_num < g->gr.gpc_tpc_count[gpc_num]; | ||
3533 | tpc_num++) { | ||
3534 | if (broadcast_flags & | ||
3535 | PRI_BROADCAST_FLAGS_SMPC) { | ||
3536 | gv11b_gr_update_priv_addr_table_smpc( | ||
3537 | g, gpc_num, tpc_num, addr, | ||
3538 | priv_addr_table, t); | ||
3539 | } else { | ||
3540 | priv_addr_table[*t] = | ||
3541 | pri_etpc_addr(g, | ||
3542 | pri_tpccs_addr_mask(addr), | ||
3543 | gpc_num, tpc_num); | ||
3544 | nvgpu_log_info(g, | ||
3545 | "priv_addr_table[%d]:%#08x", | ||
3546 | *t, priv_addr_table[*t]); | ||
3547 | (*t)++; | ||
3548 | } | ||
3549 | } | ||
3550 | } else if (broadcast_flags & PRI_BROADCAST_FLAGS_SMPC) { | ||
3551 | tpc_num = 0; | ||
3552 | gv11b_gr_update_priv_addr_table_smpc( | ||
3553 | g, gpc_num, tpc_num, addr, | ||
3554 | priv_addr_table, t); | ||
3555 | } else { | ||
3556 | priv_addr_table[*t] = | ||
3557 | pri_egpc_addr(g, | ||
3558 | pri_gpccs_addr_mask(addr), | ||
3559 | gpc_num); | ||
3560 | nvgpu_log_info(g, "priv_addr_table[%d]:%#08x", | ||
3561 | *t, priv_addr_table[*t]); | ||
3562 | (*t)++; | ||
3563 | } | ||
3564 | } | ||
3565 | } else if (!(broadcast_flags & PRI_BROADCAST_FLAGS_EGPC)) { | ||
3566 | if (broadcast_flags & PRI_BROADCAST_FLAGS_ETPC) { | ||
3567 | nvgpu_log_info(g, "broadcast flags etpc but not egpc"); | ||
3568 | gpc_num = 0; | ||
3569 | for (tpc_num = 0; | ||
3570 | tpc_num < g->gr.gpc_tpc_count[gpc]; | ||
3571 | tpc_num++) { | ||
3572 | if (broadcast_flags & | ||
3573 | PRI_BROADCAST_FLAGS_SMPC) | ||
3574 | gv11b_gr_update_priv_addr_table_smpc( | ||
3575 | g, gpc_num, tpc_num, addr, | ||
3576 | priv_addr_table, t); | ||
3577 | else { | ||
3578 | priv_addr_table[*t] = | ||
3579 | pri_etpc_addr(g, | ||
3580 | pri_tpccs_addr_mask(addr), | ||
3581 | gpc, tpc_num); | ||
3582 | nvgpu_log_info(g, | ||
3583 | "priv_addr_table[%d]:%#08x", | ||
3584 | *t, priv_addr_table[*t]); | ||
3585 | (*t)++; | ||
3586 | } | ||
3587 | } | ||
3588 | } else if (broadcast_flags & PRI_BROADCAST_FLAGS_SMPC) { | ||
3589 | tpc_num = 0; | ||
3590 | gpc_num = 0; | ||
3591 | gv11b_gr_update_priv_addr_table_smpc( | ||
3592 | g, gpc_num, tpc_num, addr, | ||
3593 | priv_addr_table, t); | ||
3594 | } else { | ||
3595 | priv_addr_table[*t] = addr; | ||
3596 | nvgpu_log_info(g, "priv_addr_table[%d]:%#08x", | ||
3597 | *t, priv_addr_table[*t]); | ||
3598 | (*t)++; | ||
3599 | } | ||
3600 | } | ||
3601 | } | ||
3602 | |||
3603 | u32 gv11b_gr_get_egpc_base(struct gk20a *g) | ||
3604 | { | ||
3605 | return EGPC_PRI_BASE; | ||
3606 | } | ||
3607 | |||
3608 | void gr_gv11b_init_gpc_mmu(struct gk20a *g) | ||
3609 | { | ||
3610 | u32 temp; | ||
3611 | |||
3612 | nvgpu_log_info(g, "initialize gpc mmu"); | ||
3613 | |||
3614 | if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { | ||
3615 | /* Bypass MMU check for non-secure boot. For | ||
3616 | * secure-boot,this register write has no-effect */ | ||
3617 | gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffff); | ||
3618 | } | ||
3619 | temp = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
3620 | temp &= gr_gpcs_pri_mmu_ctrl_vm_pg_size_m() | | ||
3621 | gr_gpcs_pri_mmu_ctrl_use_pdb_big_page_size_m() | | ||
3622 | gr_gpcs_pri_mmu_ctrl_vol_fault_m() | | ||
3623 | gr_gpcs_pri_mmu_ctrl_comp_fault_m() | | ||
3624 | gr_gpcs_pri_mmu_ctrl_miss_gran_m() | | ||
3625 | gr_gpcs_pri_mmu_ctrl_cache_mode_m() | | ||
3626 | gr_gpcs_pri_mmu_ctrl_mmu_aperture_m() | | ||
3627 | gr_gpcs_pri_mmu_ctrl_mmu_vol_m() | | ||
3628 | gr_gpcs_pri_mmu_ctrl_mmu_disable_m(); | ||
3629 | gk20a_writel(g, gr_gpcs_pri_mmu_ctrl_r(), temp); | ||
3630 | gk20a_writel(g, gr_gpcs_pri_mmu_pm_unit_mask_r(), 0); | ||
3631 | gk20a_writel(g, gr_gpcs_pri_mmu_pm_req_mask_r(), 0); | ||
3632 | |||
3633 | gk20a_writel(g, gr_gpcs_pri_mmu_debug_ctrl_r(), | ||
3634 | gk20a_readl(g, fb_mmu_debug_ctrl_r())); | ||
3635 | gk20a_writel(g, gr_gpcs_pri_mmu_debug_wr_r(), | ||
3636 | gk20a_readl(g, fb_mmu_debug_wr_r())); | ||
3637 | gk20a_writel(g, gr_gpcs_pri_mmu_debug_rd_r(), | ||
3638 | gk20a_readl(g, fb_mmu_debug_rd_r())); | ||
3639 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gr_gv11b.h b/drivers/gpu/nvgpu/gv11b/gr_gv11b.h new file mode 100644 index 00000000..b6ba231e --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gr_gv11b.h | |||
@@ -0,0 +1,215 @@ | |||
1 | /* | ||
2 | * GV11B GPU GR | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_GR_GV11B_H_ | ||
26 | #define _NVGPU_GR_GV11B_H_ | ||
27 | |||
28 | #define EGPC_PRI_BASE 0x580000 | ||
29 | #define EGPC_PRI_SHARED_BASE 0x480000 | ||
30 | |||
31 | #define PRI_BROADCAST_FLAGS_SMPC BIT(17) | ||
32 | |||
33 | #define GV11B_ZBC_TYPE_STENCIL T19X_ZBC | ||
34 | #define ZBC_STENCIL_CLEAR_FMT_INVAILD 0 | ||
35 | #define ZBC_STENCIL_CLEAR_FMT_U8 1 | ||
36 | |||
37 | struct zbc_s_table { | ||
38 | u32 stencil; | ||
39 | u32 format; | ||
40 | u32 ref_cnt; | ||
41 | }; | ||
42 | |||
43 | struct gk20a; | ||
44 | struct zbc_entry; | ||
45 | struct zbc_query_params; | ||
46 | struct channel_ctx_gk20a; | ||
47 | struct nvgpu_warpstate; | ||
48 | struct nvgpu_gr_sm_error_state; | ||
49 | |||
50 | enum { | ||
51 | VOLTA_CHANNEL_GPFIFO_A = 0xC36F, | ||
52 | VOLTA_A = 0xC397, | ||
53 | VOLTA_COMPUTE_A = 0xC3C0, | ||
54 | VOLTA_DMA_COPY_A = 0xC3B5, | ||
55 | }; | ||
56 | |||
57 | #define NVC397_SET_SHADER_EXCEPTIONS 0x1528 | ||
58 | #define NVC397_SET_CIRCULAR_BUFFER_SIZE 0x1280 | ||
59 | #define NVC397_SET_ALPHA_CIRCULAR_BUFFER_SIZE 0x02dc | ||
60 | #define NVC397_SET_GO_IDLE_TIMEOUT 0x022c | ||
61 | #define NVC397_SET_TEX_IN_DBG 0x10bc | ||
62 | #define NVC397_SET_SKEDCHECK 0x10c0 | ||
63 | #define NVC397_SET_BES_CROP_DEBUG3 0x10c4 | ||
64 | |||
65 | #define NVC397_SET_TEX_IN_DBG_TSL1_RVCH_INVALIDATE 0x1 | ||
66 | #define NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_LD 0x2 | ||
67 | #define NVC397_SET_TEX_IN_DBG_SM_L1TAG_CTRL_CACHE_SURFACE_ST 0x4 | ||
68 | |||
69 | #define NVC397_SET_SKEDCHECK_18_MASK 0x3 | ||
70 | #define NVC397_SET_SKEDCHECK_18_DEFAULT 0x0 | ||
71 | #define NVC397_SET_SKEDCHECK_18_DISABLE 0x1 | ||
72 | #define NVC397_SET_SKEDCHECK_18_ENABLE 0x2 | ||
73 | |||
74 | #define NVC3C0_SET_SKEDCHECK 0x23c | ||
75 | |||
76 | #define NVA297_SET_SHADER_EXCEPTIONS_ENABLE_FALSE 0 | ||
77 | |||
78 | int gr_gv11b_alloc_buffer(struct vm_gk20a *vm, size_t size, | ||
79 | struct nvgpu_mem *mem); | ||
80 | /*zcull*/ | ||
81 | void gr_gv11b_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries, | ||
82 | u32 *zcull_map_tiles); | ||
83 | void gr_gv11b_create_sysfs(struct gk20a *g); | ||
84 | |||
85 | bool gr_gv11b_is_valid_class(struct gk20a *g, u32 class_num); | ||
86 | bool gr_gv11b_is_valid_gfx_class(struct gk20a *g, u32 class_num); | ||
87 | bool gr_gv11b_is_valid_compute_class(struct gk20a *g, u32 class_num); | ||
88 | void gr_gv11b_enable_hww_exceptions(struct gk20a *g); | ||
89 | void gr_gv11b_enable_exceptions(struct gk20a *g); | ||
90 | int gr_gv11b_handle_tpc_sm_ecc_exception(struct gk20a *g, | ||
91 | u32 gpc, u32 tpc, | ||
92 | bool *post_event, struct channel_gk20a *fault_ch, | ||
93 | u32 *hww_global_esr); | ||
94 | int gr_gv11b_handle_gcc_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
95 | bool *post_event, struct channel_gk20a *fault_ch, | ||
96 | u32 *hww_global_esr); | ||
97 | int gr_gv11b_handle_gpc_gpcmmu_exception(struct gk20a *g, u32 gpc, | ||
98 | u32 gpc_exception); | ||
99 | int gr_gv11b_handle_gpc_gpccs_exception(struct gk20a *g, u32 gpc, | ||
100 | u32 gpc_exception); | ||
101 | void gr_gv11b_enable_gpc_exceptions(struct gk20a *g); | ||
102 | int gr_gv11b_handle_tex_exception(struct gk20a *g, u32 gpc, u32 tpc, | ||
103 | bool *post_event); | ||
104 | int gr_gv11b_zbc_s_query_table(struct gk20a *g, struct gr_gk20a *gr, | ||
105 | struct zbc_query_params *query_params); | ||
106 | bool gr_gv11b_add_zbc_type_s(struct gk20a *g, struct gr_gk20a *gr, | ||
107 | struct zbc_entry *zbc_val, int *ret_val); | ||
108 | int gr_gv11b_add_zbc_stencil(struct gk20a *g, struct gr_gk20a *gr, | ||
109 | struct zbc_entry *stencil_val, u32 index); | ||
110 | int gr_gv11b_load_stencil_default_tbl(struct gk20a *g, | ||
111 | struct gr_gk20a *gr); | ||
112 | int gr_gv11b_load_stencil_tbl(struct gk20a *g, struct gr_gk20a *gr); | ||
113 | u32 gr_gv11b_pagepool_default_size(struct gk20a *g); | ||
114 | int gr_gv11b_calc_global_ctx_buffer_size(struct gk20a *g); | ||
115 | int gr_gv11b_handle_sw_method(struct gk20a *g, u32 addr, | ||
116 | u32 class_num, u32 offset, u32 data); | ||
117 | void gr_gv11b_bundle_cb_defaults(struct gk20a *g); | ||
118 | void gr_gv11b_cb_size_default(struct gk20a *g); | ||
119 | void gr_gv11b_set_alpha_circular_buffer_size(struct gk20a *g, u32 data); | ||
120 | void gr_gv11b_set_circular_buffer_size(struct gk20a *g, u32 data); | ||
121 | int gr_gv11b_dump_gr_status_regs(struct gk20a *g, | ||
122 | struct gk20a_debug_output *o); | ||
123 | int gr_gv11b_wait_empty(struct gk20a *g, unsigned long duration_ms, | ||
124 | u32 expect_delay); | ||
125 | void gr_gv11b_commit_global_attrib_cb(struct gk20a *g, | ||
126 | struct channel_ctx_gk20a *ch_ctx, | ||
127 | u64 addr, bool patch); | ||
128 | void gr_gv11b_set_gpc_tpc_mask(struct gk20a *g, u32 gpc_index); | ||
129 | void gr_gv11b_get_access_map(struct gk20a *g, | ||
130 | u32 **whitelist, int *num_entries); | ||
131 | int gr_gv11b_pre_process_sm_exception(struct gk20a *g, | ||
132 | u32 gpc, u32 tpc, u32 sm, u32 global_esr, u32 warp_esr, | ||
133 | bool sm_debugger_attached, struct channel_gk20a *fault_ch, | ||
134 | bool *early_exit, bool *ignore_debugger); | ||
135 | int gr_gv11b_handle_fecs_error(struct gk20a *g, | ||
136 | struct channel_gk20a *__ch, | ||
137 | struct gr_gk20a_isr_data *isr_data); | ||
138 | int gr_gv11b_setup_rop_mapping(struct gk20a *g, struct gr_gk20a *gr); | ||
139 | int gr_gv11b_init_sw_veid_bundle(struct gk20a *g); | ||
140 | void gr_gv11b_detect_sm_arch(struct gk20a *g); | ||
141 | void gr_gv11b_program_sm_id_numbering(struct gk20a *g, | ||
142 | u32 gpc, u32 tpc, u32 smid); | ||
143 | int gr_gv11b_load_smid_config(struct gk20a *g); | ||
144 | int gr_gv11b_commit_inst(struct channel_gk20a *c, u64 gpu_va); | ||
145 | int gr_gv11b_commit_global_timeslice(struct gk20a *g, struct channel_gk20a *c); | ||
146 | void gr_gv11b_write_zcull_ptr(struct gk20a *g, | ||
147 | struct nvgpu_mem *mem, u64 gpu_va); | ||
148 | void gr_gv11b_write_pm_ptr(struct gk20a *g, | ||
149 | struct nvgpu_mem *mem, u64 gpu_va); | ||
150 | void gr_gv11b_init_elcg_mode(struct gk20a *g, u32 mode, u32 engine); | ||
151 | void gr_gv11b_load_tpc_mask(struct gk20a *g); | ||
152 | void gr_gv11b_set_preemption_buffer_va(struct gk20a *g, | ||
153 | struct nvgpu_mem *mem, u64 gpu_va); | ||
154 | int gr_gv11b_init_fs_state(struct gk20a *g); | ||
155 | void gv11b_gr_get_esr_sm_sel(struct gk20a *g, u32 gpc, u32 tpc, | ||
156 | u32 *esr_sm_sel); | ||
157 | int gv11b_gr_sm_trigger_suspend(struct gk20a *g); | ||
158 | void gv11b_gr_bpt_reg_info(struct gk20a *g, struct nvgpu_warpstate *w_state); | ||
159 | int gv11b_gr_update_sm_error_state(struct gk20a *g, | ||
160 | struct channel_gk20a *ch, u32 sm_id, | ||
161 | struct nvgpu_gr_sm_error_state *sm_error_state); | ||
162 | int gv11b_gr_set_sm_debug_mode(struct gk20a *g, | ||
163 | struct channel_gk20a *ch, u64 sms, bool enable); | ||
164 | int gv11b_gr_record_sm_error_state(struct gk20a *g, u32 gpc, u32 tpc); | ||
165 | void gv11b_gr_set_hww_esr_report_mask(struct gk20a *g); | ||
166 | bool gv11b_gr_sm_debugger_attached(struct gk20a *g); | ||
167 | void gv11b_gr_suspend_single_sm(struct gk20a *g, | ||
168 | u32 gpc, u32 tpc, u32 sm, | ||
169 | u32 global_esr_mask, bool check_errors); | ||
170 | void gv11b_gr_suspend_all_sms(struct gk20a *g, | ||
171 | u32 global_esr_mask, bool check_errors); | ||
172 | void gv11b_gr_resume_single_sm(struct gk20a *g, | ||
173 | u32 gpc, u32 tpc, u32 sm); | ||
174 | void gv11b_gr_resume_all_sms(struct gk20a *g); | ||
175 | int gv11b_gr_resume_from_pause(struct gk20a *g); | ||
176 | u32 gv11b_gr_get_sm_hww_warp_esr(struct gk20a *g, | ||
177 | u32 gpc, u32 tpc, u32 sm); | ||
178 | u32 gv11b_gr_get_sm_hww_global_esr(struct gk20a *g, | ||
179 | u32 gpc, u32 tpc, u32 sm); | ||
180 | u32 gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g); | ||
181 | int gv11b_gr_wait_for_sm_lock_down(struct gk20a *g, | ||
182 | u32 gpc, u32 tpc, u32 sm, | ||
183 | u32 global_esr_mask, bool check_errors); | ||
184 | int gv11b_gr_lock_down_sm(struct gk20a *g, | ||
185 | u32 gpc, u32 tpc, u32 sm, u32 global_esr_mask, | ||
186 | bool check_errors); | ||
187 | void gv11b_gr_clear_sm_hww(struct gk20a *g, u32 gpc, u32 tpc, u32 sm, | ||
188 | u32 global_esr); | ||
189 | int gr_gv11b_handle_tpc_mpc_exception(struct gk20a *g, | ||
190 | u32 gpc, u32 tpc, bool *post_event); | ||
191 | void gv11b_gr_init_ovr_sm_dsm_perf(void); | ||
192 | void gv11b_gr_init_sm_dsm_reg_info(void); | ||
193 | void gv11b_gr_get_sm_dsm_perf_regs(struct gk20a *g, | ||
194 | u32 *num_sm_dsm_perf_regs, | ||
195 | u32 **sm_dsm_perf_regs, | ||
196 | u32 *perf_register_stride); | ||
197 | void gv11b_gr_get_sm_dsm_perf_ctrl_regs(struct gk20a *g, | ||
198 | u32 *num_sm_dsm_perf_ctrl_regs, | ||
199 | u32 **sm_dsm_perf_ctrl_regs, | ||
200 | u32 *ctrl_register_stride); | ||
201 | void gv11b_gr_get_ovr_perf_regs(struct gk20a *g, u32 *num_ovr_perf_regs, | ||
202 | u32 **ovr_perf_regs); | ||
203 | void gv11b_gr_access_smpc_reg(struct gk20a *g, u32 quad, u32 offset); | ||
204 | bool gv11b_gr_pri_is_egpc_addr(struct gk20a *g, u32 addr); | ||
205 | bool gv11b_gr_pri_is_etpc_addr(struct gk20a *g, u32 addr); | ||
206 | void gv11b_gr_get_egpc_etpc_num(struct gk20a *g, u32 addr, | ||
207 | u32 *egpc_num, u32 *etpc_num); | ||
208 | int gv11b_gr_decode_egpc_addr(struct gk20a *g, u32 addr, int *addr_type, | ||
209 | u32 *gpc_num, u32 *tpc_num, u32 *broadcast_flags); | ||
210 | void gv11b_gr_egpc_etpc_priv_addr_table(struct gk20a *g, u32 addr, | ||
211 | u32 gpc, u32 broadcast_flags, u32 *priv_addr_table, u32 *t); | ||
212 | u32 gv11b_gr_get_egpc_base(struct gk20a *g); | ||
213 | void gr_gv11b_init_gpc_mmu(struct gk20a *g); | ||
214 | |||
215 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b.c b/drivers/gpu/nvgpu/gv11b/gv11b.c new file mode 100644 index 00000000..211755e5 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gv11b.c | |||
@@ -0,0 +1,38 @@ | |||
1 | /* | ||
2 | * GV11B Graphics | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/enabled.h> | ||
26 | #include <nvgpu/enabled_t19x.h> | ||
27 | |||
28 | #include "gk20a/gk20a.h" | ||
29 | |||
30 | #include "gv11b/gv11b.h" | ||
31 | |||
32 | int gv11b_init_gpu_characteristics(struct gk20a *g) | ||
33 | { | ||
34 | gk20a_init_gpu_characteristics(g); | ||
35 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_TSG_SUBCONTEXTS, true); | ||
36 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_IO_COHERENCE, true); | ||
37 | return 0; | ||
38 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b.h b/drivers/gpu/nvgpu/gv11b/gv11b.h new file mode 100644 index 00000000..3d5490e6 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gv11b.h | |||
@@ -0,0 +1,32 @@ | |||
1 | /* | ||
2 | * GV11B Graphics | ||
3 | * | ||
4 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef GV11B_H | ||
26 | #define GV11B_H | ||
27 | |||
28 | #include "gk20a/gk20a.h" | ||
29 | |||
30 | int gv11b_init_gpu_characteristics(struct gk20a *g); | ||
31 | |||
32 | #endif /* GV11B_H */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c new file mode 100644 index 00000000..9f6057ae --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.c | |||
@@ -0,0 +1,748 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | * | ||
22 | * This file is autogenerated. Do not edit. | ||
23 | */ | ||
24 | |||
25 | #ifndef __gv11b_gating_reglist_h__ | ||
26 | #define __gv11b_gating_reglist_h__ | ||
27 | |||
28 | #include <linux/types.h> | ||
29 | #include "gv11b_gating_reglist.h" | ||
30 | #include <nvgpu/enabled.h> | ||
31 | |||
32 | struct gating_desc { | ||
33 | u32 addr; | ||
34 | u32 prod; | ||
35 | u32 disable; | ||
36 | }; | ||
37 | /* slcg bus */ | ||
38 | static const struct gating_desc gv11b_slcg_bus[] = { | ||
39 | {.addr = 0x00001c04, .prod = 0x00000000, .disable = 0x000003fe}, | ||
40 | }; | ||
41 | |||
42 | /* slcg ce2 */ | ||
43 | static const struct gating_desc gv11b_slcg_ce2[] = { | ||
44 | {.addr = 0x00104204, .prod = 0x00000040, .disable = 0x000007fe}, | ||
45 | }; | ||
46 | |||
47 | /* slcg chiplet */ | ||
48 | static const struct gating_desc gv11b_slcg_chiplet[] = { | ||
49 | {.addr = 0x0010c07c, .prod = 0x00000000, .disable = 0x00000007}, | ||
50 | {.addr = 0x0010e07c, .prod = 0x00000000, .disable = 0x00000007}, | ||
51 | {.addr = 0x0010d07c, .prod = 0x00000000, .disable = 0x00000007}, | ||
52 | {.addr = 0x0010e17c, .prod = 0x00000000, .disable = 0x00000007}, | ||
53 | }; | ||
54 | |||
55 | /* slcg fb */ | ||
56 | static const struct gating_desc gv11b_slcg_fb[] = { | ||
57 | {.addr = 0x00100d14, .prod = 0x00000000, .disable = 0xfffffffe}, | ||
58 | {.addr = 0x00100c9c, .prod = 0x00000000, .disable = 0x000001fe}, | ||
59 | }; | ||
60 | |||
61 | /* slcg fifo */ | ||
62 | static const struct gating_desc gv11b_slcg_fifo[] = { | ||
63 | {.addr = 0x000026ec, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
64 | }; | ||
65 | |||
66 | /* slcg gr */ | ||
67 | static const struct gating_desc gv11b_slcg_gr[] = { | ||
68 | {.addr = 0x004041f4, .prod = 0x00000000, .disable = 0x07fffffe}, | ||
69 | {.addr = 0x00409134, .prod = 0x00020008, .disable = 0x0003fffe}, | ||
70 | {.addr = 0x00409894, .prod = 0x00000000, .disable = 0x0000fffe}, | ||
71 | {.addr = 0x004078c4, .prod = 0x00000000, .disable = 0x000001fe}, | ||
72 | {.addr = 0x00406004, .prod = 0x00000200, .disable = 0x0001fffe}, | ||
73 | {.addr = 0x00405864, .prod = 0x00000000, .disable = 0x000001fe}, | ||
74 | {.addr = 0x00405910, .prod = 0xfffffff0, .disable = 0xfffffffe}, | ||
75 | {.addr = 0x00408044, .prod = 0x00000000, .disable = 0x000007fe}, | ||
76 | {.addr = 0x00407004, .prod = 0x00000000, .disable = 0x000001fe}, | ||
77 | {.addr = 0x00405bf4, .prod = 0x00000000, .disable = 0x00000002}, | ||
78 | {.addr = 0x0041a134, .prod = 0x00020008, .disable = 0x0003fffe}, | ||
79 | {.addr = 0x0041a894, .prod = 0x00000000, .disable = 0x0000fffe}, | ||
80 | {.addr = 0x00418504, .prod = 0x00000000, .disable = 0x0007fffe}, | ||
81 | {.addr = 0x0041860c, .prod = 0x00000000, .disable = 0x000001fe}, | ||
82 | {.addr = 0x0041868c, .prod = 0x00000000, .disable = 0x0000001e}, | ||
83 | {.addr = 0x0041871c, .prod = 0x00000000, .disable = 0x000003fe}, | ||
84 | {.addr = 0x00418388, .prod = 0x00000000, .disable = 0x00000001}, | ||
85 | {.addr = 0x0041882c, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
86 | {.addr = 0x00418bc0, .prod = 0x00000000, .disable = 0x000001fe}, | ||
87 | {.addr = 0x00418974, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
88 | {.addr = 0x00418c74, .prod = 0xffffff80, .disable = 0xfffffffe}, | ||
89 | {.addr = 0x00418cf4, .prod = 0xfffffff8, .disable = 0xfffffffe}, | ||
90 | {.addr = 0x00418d74, .prod = 0xffffffe0, .disable = 0xfffffffe}, | ||
91 | {.addr = 0x00418f10, .prod = 0xffffffe0, .disable = 0xfffffffe}, | ||
92 | {.addr = 0x00418e10, .prod = 0xfffffffe, .disable = 0xfffffffe}, | ||
93 | {.addr = 0x00419024, .prod = 0x000001fe, .disable = 0x000001fe}, | ||
94 | {.addr = 0x0041889c, .prod = 0x00000000, .disable = 0x000001fe}, | ||
95 | {.addr = 0x00419d24, .prod = 0x00000000, .disable = 0x000000ff}, | ||
96 | {.addr = 0x0041986c, .prod = 0x00000104, .disable = 0x00fffffe}, | ||
97 | {.addr = 0x00419c74, .prod = 0x0000001e, .disable = 0x0000001e}, | ||
98 | {.addr = 0x00419c84, .prod = 0x0003fff8, .disable = 0x0003fffe}, | ||
99 | {.addr = 0x00419c8c, .prod = 0xffffff84, .disable = 0xfffffffe}, | ||
100 | {.addr = 0x00419c94, .prod = 0x00080040, .disable = 0x000ffffe}, | ||
101 | {.addr = 0x00419ca4, .prod = 0x00003ffe, .disable = 0x00003ffe}, | ||
102 | {.addr = 0x00419cac, .prod = 0x0001fffe, .disable = 0x0001fffe}, | ||
103 | {.addr = 0x00419a44, .prod = 0x00000008, .disable = 0x0000000e}, | ||
104 | {.addr = 0x00419a4c, .prod = 0x000001f8, .disable = 0x000001fe}, | ||
105 | {.addr = 0x00419a54, .prod = 0x0000003c, .disable = 0x0000003e}, | ||
106 | {.addr = 0x00419a5c, .prod = 0x0000000c, .disable = 0x0000000e}, | ||
107 | {.addr = 0x00419a64, .prod = 0x000001ba, .disable = 0x000001fe}, | ||
108 | {.addr = 0x00419a7c, .prod = 0x0000003c, .disable = 0x0000003e}, | ||
109 | {.addr = 0x00419a84, .prod = 0x0000000c, .disable = 0x0000000e}, | ||
110 | {.addr = 0x0041be2c, .prod = 0x04115fc0, .disable = 0xfffffffe}, | ||
111 | {.addr = 0x0041bfec, .prod = 0xfffffff0, .disable = 0xfffffffe}, | ||
112 | {.addr = 0x0041bed4, .prod = 0xfffffff8, .disable = 0xfffffffe}, | ||
113 | {.addr = 0x00408814, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
114 | {.addr = 0x00408a84, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
115 | {.addr = 0x004089ac, .prod = 0x00000000, .disable = 0x0001fffe}, | ||
116 | {.addr = 0x00408a24, .prod = 0x00000000, .disable = 0x000000ff}, | ||
117 | }; | ||
118 | |||
119 | /* slcg ltc */ | ||
120 | static const struct gating_desc gv11b_slcg_ltc[] = { | ||
121 | {.addr = 0x0017e050, .prod = 0x00000000, .disable = 0xfffffffe}, | ||
122 | {.addr = 0x0017e35c, .prod = 0x00000000, .disable = 0xfffffffe}, | ||
123 | }; | ||
124 | |||
125 | /* slcg perf */ | ||
126 | static const struct gating_desc gv11b_slcg_perf[] = { | ||
127 | {.addr = 0x00248018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
128 | {.addr = 0x00248018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
129 | {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
130 | {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
131 | {.addr = 0x00246018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
132 | {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
133 | {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
134 | {.addr = 0x00244018, .prod = 0xffffffff, .disable = 0x00000000}, | ||
135 | {.addr = 0x0024a124, .prod = 0x00000001, .disable = 0x00000000}, | ||
136 | }; | ||
137 | |||
138 | /* slcg PriRing */ | ||
139 | static const struct gating_desc gv11b_slcg_priring[] = { | ||
140 | {.addr = 0x001200a8, .prod = 0x00000000, .disable = 0x00000001}, | ||
141 | }; | ||
142 | |||
143 | /* slcg pwr_csb */ | ||
144 | static const struct gating_desc gv11b_slcg_pwr_csb[] = { | ||
145 | {.addr = 0x00000134, .prod = 0x00020008, .disable = 0x0003fffe}, | ||
146 | {.addr = 0x00000e74, .prod = 0x00000000, .disable = 0x0000000f}, | ||
147 | {.addr = 0x00000a74, .prod = 0x00004040, .disable = 0x00007ffe}, | ||
148 | {.addr = 0x000206b8, .prod = 0x00000008, .disable = 0x0000000f}, | ||
149 | }; | ||
150 | |||
151 | /* slcg pmu */ | ||
152 | static const struct gating_desc gv11b_slcg_pmu[] = { | ||
153 | {.addr = 0x0010a134, .prod = 0x00020008, .disable = 0x0003fffe}, | ||
154 | {.addr = 0x0010aa74, .prod = 0x00004040, .disable = 0x00007ffe}, | ||
155 | {.addr = 0x0010ae74, .prod = 0x00000000, .disable = 0x0000000f}, | ||
156 | }; | ||
157 | |||
158 | /* therm gr */ | ||
159 | static const struct gating_desc gv11b_slcg_therm[] = { | ||
160 | {.addr = 0x000206b8, .prod = 0x00000008, .disable = 0x0000000f}, | ||
161 | }; | ||
162 | |||
163 | /* slcg Xbar */ | ||
164 | static const struct gating_desc gv11b_slcg_xbar[] = { | ||
165 | {.addr = 0x0013c824, .prod = 0x00000000, .disable = 0x7ffffffe}, | ||
166 | {.addr = 0x0013dc08, .prod = 0x00000000, .disable = 0xfffffffe}, | ||
167 | {.addr = 0x0013c924, .prod = 0x00000000, .disable = 0x7ffffffe}, | ||
168 | {.addr = 0x0013cbe4, .prod = 0x00000000, .disable = 0x1ffffffe}, | ||
169 | {.addr = 0x0013cc04, .prod = 0x00000000, .disable = 0x1ffffffe}, | ||
170 | }; | ||
171 | |||
172 | /* blcg bus */ | ||
173 | static const struct gating_desc gv11b_blcg_bus[] = { | ||
174 | {.addr = 0x00001c00, .prod = 0x00000042, .disable = 0x00000000}, | ||
175 | }; | ||
176 | |||
177 | /* blcg ce */ | ||
178 | static const struct gating_desc gv11b_blcg_ce[] = { | ||
179 | {.addr = 0x00104200, .prod = 0x0000c242, .disable = 0x00000000}, | ||
180 | }; | ||
181 | |||
182 | /* blcg ctxsw prog */ | ||
183 | static const struct gating_desc gv11b_blcg_ctxsw_prog[] = { | ||
184 | }; | ||
185 | |||
186 | /* blcg fb */ | ||
187 | static const struct gating_desc gv11b_blcg_fb[] = { | ||
188 | {.addr = 0x00100d10, .prod = 0x0000c242, .disable = 0x00000000}, | ||
189 | {.addr = 0x00100d30, .prod = 0x0000c242, .disable = 0x00000000}, | ||
190 | {.addr = 0x00100d3c, .prod = 0x00000242, .disable = 0x00000000}, | ||
191 | {.addr = 0x00100d48, .prod = 0x0000c242, .disable = 0x00000000}, | ||
192 | {.addr = 0x00100d1c, .prod = 0x00000042, .disable = 0x00000000}, | ||
193 | {.addr = 0x00100c98, .prod = 0x00004242, .disable = 0x00000000}, | ||
194 | }; | ||
195 | |||
196 | /* blcg fifo */ | ||
197 | static const struct gating_desc gv11b_blcg_fifo[] = { | ||
198 | {.addr = 0x000026e0, .prod = 0x0000c244, .disable = 0x00000000}, | ||
199 | }; | ||
200 | |||
201 | /* blcg gr */ | ||
202 | static const struct gating_desc gv11b_blcg_gr[] = { | ||
203 | {.addr = 0x004041f0, .prod = 0x0000c646, .disable = 0x00000000}, | ||
204 | {.addr = 0x00409890, .prod = 0x0000007f, .disable = 0x00000000}, | ||
205 | {.addr = 0x004098b0, .prod = 0x0000007f, .disable = 0x00000000}, | ||
206 | {.addr = 0x004078c0, .prod = 0x00004242, .disable = 0x00000000}, | ||
207 | {.addr = 0x00406000, .prod = 0x0000c444, .disable = 0x00000000}, | ||
208 | {.addr = 0x00405860, .prod = 0x0000c242, .disable = 0x00000000}, | ||
209 | {.addr = 0x0040590c, .prod = 0x0000c444, .disable = 0x00000000}, | ||
210 | {.addr = 0x00408040, .prod = 0x0000c444, .disable = 0x00000000}, | ||
211 | {.addr = 0x00407000, .prod = 0x4000c242, .disable = 0x00000000}, | ||
212 | {.addr = 0x00405bf0, .prod = 0x0000c444, .disable = 0x00000000}, | ||
213 | {.addr = 0x0041a890, .prod = 0x0000427f, .disable = 0x00000000}, | ||
214 | {.addr = 0x0041a8b0, .prod = 0x0000007f, .disable = 0x00000000}, | ||
215 | {.addr = 0x00418500, .prod = 0x0000c244, .disable = 0x00000000}, | ||
216 | {.addr = 0x00418608, .prod = 0x0000c242, .disable = 0x00000000}, | ||
217 | {.addr = 0x00418688, .prod = 0x0000c242, .disable = 0x00000000}, | ||
218 | {.addr = 0x00418718, .prod = 0x00000042, .disable = 0x00000000}, | ||
219 | {.addr = 0x00418828, .prod = 0x00008444, .disable = 0x00000000}, | ||
220 | {.addr = 0x00418bbc, .prod = 0x0000c242, .disable = 0x00000000}, | ||
221 | {.addr = 0x00418970, .prod = 0x0000c242, .disable = 0x00000000}, | ||
222 | {.addr = 0x00418c70, .prod = 0x0000c444, .disable = 0x00000000}, | ||
223 | {.addr = 0x00418cf0, .prod = 0x0000c444, .disable = 0x00000000}, | ||
224 | {.addr = 0x00418d70, .prod = 0x0000c444, .disable = 0x00000000}, | ||
225 | {.addr = 0x00418f0c, .prod = 0x0000c444, .disable = 0x00000000}, | ||
226 | {.addr = 0x00418e0c, .prod = 0x0000c444, .disable = 0x00000000}, | ||
227 | {.addr = 0x00419020, .prod = 0x0000c242, .disable = 0x00000000}, | ||
228 | {.addr = 0x00419038, .prod = 0x00000042, .disable = 0x00000000}, | ||
229 | {.addr = 0x00418898, .prod = 0x00004242, .disable = 0x00000000}, | ||
230 | {.addr = 0x00419868, .prod = 0x00008243, .disable = 0x00000000}, | ||
231 | {.addr = 0x00419c70, .prod = 0x0000c444, .disable = 0x00000000}, | ||
232 | {.addr = 0x00419c80, .prod = 0x00004045, .disable = 0x00000000}, | ||
233 | {.addr = 0x00419c88, .prod = 0x00004043, .disable = 0x00000000}, | ||
234 | {.addr = 0x00419c90, .prod = 0x0000004a, .disable = 0x00000000}, | ||
235 | {.addr = 0x00419c98, .prod = 0x00000042, .disable = 0x00000000}, | ||
236 | {.addr = 0x00419ca0, .prod = 0x00000043, .disable = 0x00000000}, | ||
237 | {.addr = 0x00419ca8, .prod = 0x00000003, .disable = 0x00000000}, | ||
238 | {.addr = 0x00419cb0, .prod = 0x00000002, .disable = 0x00000000}, | ||
239 | {.addr = 0x00419a40, .prod = 0x00000242, .disable = 0x00000000}, | ||
240 | {.addr = 0x00419a48, .prod = 0x00000242, .disable = 0x00000000}, | ||
241 | {.addr = 0x00419a50, .prod = 0x00000242, .disable = 0x00000000}, | ||
242 | {.addr = 0x00419a58, .prod = 0x00000242, .disable = 0x00000000}, | ||
243 | {.addr = 0x00419a60, .prod = 0x00000202, .disable = 0x00000000}, | ||
244 | {.addr = 0x00419a68, .prod = 0x00000202, .disable = 0x00000000}, | ||
245 | {.addr = 0x00419a78, .prod = 0x00000242, .disable = 0x00000000}, | ||
246 | {.addr = 0x00419a80, .prod = 0x00000242, .disable = 0x00000000}, | ||
247 | {.addr = 0x0041be28, .prod = 0x00008242, .disable = 0x00000000}, | ||
248 | {.addr = 0x0041bfe8, .prod = 0x0000c444, .disable = 0x00000000}, | ||
249 | {.addr = 0x0041bed0, .prod = 0x0000c444, .disable = 0x00000000}, | ||
250 | {.addr = 0x00408810, .prod = 0x0000c242, .disable = 0x00000000}, | ||
251 | {.addr = 0x00408a80, .prod = 0x0000c242, .disable = 0x00000000}, | ||
252 | {.addr = 0x004089a8, .prod = 0x0000c242, .disable = 0x00000000}, | ||
253 | }; | ||
254 | |||
255 | /* blcg ltc */ | ||
256 | static const struct gating_desc gv11b_blcg_ltc[] = { | ||
257 | {.addr = 0x0017e030, .prod = 0x00000044, .disable = 0x00000000}, | ||
258 | {.addr = 0x0017e040, .prod = 0x00000044, .disable = 0x00000000}, | ||
259 | {.addr = 0x0017e3e0, .prod = 0x00000044, .disable = 0x00000000}, | ||
260 | {.addr = 0x0017e3c8, .prod = 0x00000044, .disable = 0x00000000}, | ||
261 | }; | ||
262 | |||
263 | /* blcg pwr_csb */ | ||
264 | static const struct gating_desc gv11b_blcg_pwr_csb[] = { | ||
265 | {.addr = 0x00000a70, .prod = 0x00000045, .disable = 0x00000000}, | ||
266 | }; | ||
267 | |||
268 | /* blcg pmu */ | ||
269 | static const struct gating_desc gv11b_blcg_pmu[] = { | ||
270 | {.addr = 0x0010aa70, .prod = 0x00000045, .disable = 0x00000000}, | ||
271 | }; | ||
272 | |||
273 | /* blcg Xbar */ | ||
274 | static const struct gating_desc gv11b_blcg_xbar[] = { | ||
275 | {.addr = 0x0013c820, .prod = 0x0001004a, .disable = 0x00000000}, | ||
276 | {.addr = 0x0013dc04, .prod = 0x0001004a, .disable = 0x00000000}, | ||
277 | {.addr = 0x0013c920, .prod = 0x0000004a, .disable = 0x00000000}, | ||
278 | {.addr = 0x0013cbe0, .prod = 0x00000042, .disable = 0x00000000}, | ||
279 | {.addr = 0x0013cc00, .prod = 0x00000042, .disable = 0x00000000}, | ||
280 | }; | ||
281 | |||
282 | /* pg gr */ | ||
283 | static const struct gating_desc gv11b_pg_gr[] = { | ||
284 | }; | ||
285 | |||
286 | /* inline functions */ | ||
287 | void gv11b_slcg_bus_load_gating_prod(struct gk20a *g, | ||
288 | bool prod) | ||
289 | { | ||
290 | u32 i; | ||
291 | u32 size = sizeof(gv11b_slcg_bus) / sizeof(struct gating_desc); | ||
292 | |||
293 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
294 | return; | ||
295 | |||
296 | for (i = 0; i < size; i++) { | ||
297 | if (prod) | ||
298 | gk20a_writel(g, gv11b_slcg_bus[i].addr, | ||
299 | gv11b_slcg_bus[i].prod); | ||
300 | else | ||
301 | gk20a_writel(g, gv11b_slcg_bus[i].addr, | ||
302 | gv11b_slcg_bus[i].disable); | ||
303 | } | ||
304 | } | ||
305 | |||
306 | void gv11b_slcg_ce2_load_gating_prod(struct gk20a *g, | ||
307 | bool prod) | ||
308 | { | ||
309 | u32 i; | ||
310 | u32 size = sizeof(gv11b_slcg_ce2) / sizeof(struct gating_desc); | ||
311 | |||
312 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
313 | return; | ||
314 | |||
315 | for (i = 0; i < size; i++) { | ||
316 | if (prod) | ||
317 | gk20a_writel(g, gv11b_slcg_ce2[i].addr, | ||
318 | gv11b_slcg_ce2[i].prod); | ||
319 | else | ||
320 | gk20a_writel(g, gv11b_slcg_ce2[i].addr, | ||
321 | gv11b_slcg_ce2[i].disable); | ||
322 | } | ||
323 | } | ||
324 | |||
325 | void gv11b_slcg_chiplet_load_gating_prod(struct gk20a *g, | ||
326 | bool prod) | ||
327 | { | ||
328 | u32 i; | ||
329 | u32 size = sizeof(gv11b_slcg_chiplet) / sizeof(struct gating_desc); | ||
330 | |||
331 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
332 | return; | ||
333 | |||
334 | for (i = 0; i < size; i++) { | ||
335 | if (prod) | ||
336 | gk20a_writel(g, gv11b_slcg_chiplet[i].addr, | ||
337 | gv11b_slcg_chiplet[i].prod); | ||
338 | else | ||
339 | gk20a_writel(g, gv11b_slcg_chiplet[i].addr, | ||
340 | gv11b_slcg_chiplet[i].disable); | ||
341 | } | ||
342 | } | ||
343 | |||
344 | void gv11b_slcg_ctxsw_firmware_load_gating_prod(struct gk20a *g, | ||
345 | bool prod) | ||
346 | { | ||
347 | } | ||
348 | |||
349 | void gv11b_slcg_fb_load_gating_prod(struct gk20a *g, | ||
350 | bool prod) | ||
351 | { | ||
352 | u32 i; | ||
353 | u32 size = sizeof(gv11b_slcg_fb) / sizeof(struct gating_desc); | ||
354 | |||
355 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
356 | return; | ||
357 | |||
358 | for (i = 0; i < size; i++) { | ||
359 | if (prod) | ||
360 | gk20a_writel(g, gv11b_slcg_fb[i].addr, | ||
361 | gv11b_slcg_fb[i].prod); | ||
362 | else | ||
363 | gk20a_writel(g, gv11b_slcg_fb[i].addr, | ||
364 | gv11b_slcg_fb[i].disable); | ||
365 | } | ||
366 | } | ||
367 | |||
368 | void gv11b_slcg_fifo_load_gating_prod(struct gk20a *g, | ||
369 | bool prod) | ||
370 | { | ||
371 | u32 i; | ||
372 | u32 size = sizeof(gv11b_slcg_fifo) / sizeof(struct gating_desc); | ||
373 | |||
374 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
375 | return; | ||
376 | |||
377 | for (i = 0; i < size; i++) { | ||
378 | if (prod) | ||
379 | gk20a_writel(g, gv11b_slcg_fifo[i].addr, | ||
380 | gv11b_slcg_fifo[i].prod); | ||
381 | else | ||
382 | gk20a_writel(g, gv11b_slcg_fifo[i].addr, | ||
383 | gv11b_slcg_fifo[i].disable); | ||
384 | } | ||
385 | } | ||
386 | |||
387 | void gr_gv11b_slcg_gr_load_gating_prod(struct gk20a *g, | ||
388 | bool prod) | ||
389 | { | ||
390 | u32 i; | ||
391 | u32 size = sizeof(gv11b_slcg_gr) / sizeof(struct gating_desc); | ||
392 | |||
393 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
394 | return; | ||
395 | |||
396 | for (i = 0; i < size; i++) { | ||
397 | if (prod) | ||
398 | gk20a_writel(g, gv11b_slcg_gr[i].addr, | ||
399 | gv11b_slcg_gr[i].prod); | ||
400 | else | ||
401 | gk20a_writel(g, gv11b_slcg_gr[i].addr, | ||
402 | gv11b_slcg_gr[i].disable); | ||
403 | } | ||
404 | } | ||
405 | |||
406 | void ltc_gv11b_slcg_ltc_load_gating_prod(struct gk20a *g, | ||
407 | bool prod) | ||
408 | { | ||
409 | u32 i; | ||
410 | u32 size = sizeof(gv11b_slcg_ltc) / sizeof(struct gating_desc); | ||
411 | |||
412 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
413 | return; | ||
414 | |||
415 | for (i = 0; i < size; i++) { | ||
416 | if (prod) | ||
417 | gk20a_writel(g, gv11b_slcg_ltc[i].addr, | ||
418 | gv11b_slcg_ltc[i].prod); | ||
419 | else | ||
420 | gk20a_writel(g, gv11b_slcg_ltc[i].addr, | ||
421 | gv11b_slcg_ltc[i].disable); | ||
422 | } | ||
423 | } | ||
424 | |||
425 | void gv11b_slcg_perf_load_gating_prod(struct gk20a *g, | ||
426 | bool prod) | ||
427 | { | ||
428 | u32 i; | ||
429 | u32 size = sizeof(gv11b_slcg_perf) / sizeof(struct gating_desc); | ||
430 | |||
431 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
432 | return; | ||
433 | |||
434 | for (i = 0; i < size; i++) { | ||
435 | if (prod) | ||
436 | gk20a_writel(g, gv11b_slcg_perf[i].addr, | ||
437 | gv11b_slcg_perf[i].prod); | ||
438 | else | ||
439 | gk20a_writel(g, gv11b_slcg_perf[i].addr, | ||
440 | gv11b_slcg_perf[i].disable); | ||
441 | } | ||
442 | } | ||
443 | |||
444 | void gv11b_slcg_priring_load_gating_prod(struct gk20a *g, | ||
445 | bool prod) | ||
446 | { | ||
447 | u32 i; | ||
448 | u32 size = sizeof(gv11b_slcg_priring) / sizeof(struct gating_desc); | ||
449 | |||
450 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
451 | return; | ||
452 | |||
453 | for (i = 0; i < size; i++) { | ||
454 | if (prod) | ||
455 | gk20a_writel(g, gv11b_slcg_priring[i].addr, | ||
456 | gv11b_slcg_priring[i].prod); | ||
457 | else | ||
458 | gk20a_writel(g, gv11b_slcg_priring[i].addr, | ||
459 | gv11b_slcg_priring[i].disable); | ||
460 | } | ||
461 | } | ||
462 | |||
463 | void gv11b_slcg_pwr_csb_load_gating_prod(struct gk20a *g, | ||
464 | bool prod) | ||
465 | { | ||
466 | u32 i; | ||
467 | u32 size = sizeof(gv11b_slcg_pwr_csb) / sizeof(struct gating_desc); | ||
468 | |||
469 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
470 | return; | ||
471 | |||
472 | for (i = 0; i < size; i++) { | ||
473 | if (prod) | ||
474 | gk20a_writel(g, gv11b_slcg_pwr_csb[i].addr, | ||
475 | gv11b_slcg_pwr_csb[i].prod); | ||
476 | else | ||
477 | gk20a_writel(g, gv11b_slcg_pwr_csb[i].addr, | ||
478 | gv11b_slcg_pwr_csb[i].disable); | ||
479 | } | ||
480 | } | ||
481 | |||
482 | void gv11b_slcg_pmu_load_gating_prod(struct gk20a *g, | ||
483 | bool prod) | ||
484 | { | ||
485 | u32 i; | ||
486 | u32 size = sizeof(gv11b_slcg_pmu) / sizeof(struct gating_desc); | ||
487 | |||
488 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
489 | return; | ||
490 | |||
491 | for (i = 0; i < size; i++) { | ||
492 | if (prod) | ||
493 | gk20a_writel(g, gv11b_slcg_pmu[i].addr, | ||
494 | gv11b_slcg_pmu[i].prod); | ||
495 | else | ||
496 | gk20a_writel(g, gv11b_slcg_pmu[i].addr, | ||
497 | gv11b_slcg_pmu[i].disable); | ||
498 | } | ||
499 | } | ||
500 | |||
501 | void gv11b_slcg_therm_load_gating_prod(struct gk20a *g, | ||
502 | bool prod) | ||
503 | { | ||
504 | u32 i; | ||
505 | u32 size = sizeof(gv11b_slcg_therm) / sizeof(struct gating_desc); | ||
506 | |||
507 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
508 | return; | ||
509 | |||
510 | for (i = 0; i < size; i++) { | ||
511 | if (prod) | ||
512 | gk20a_writel(g, gv11b_slcg_therm[i].addr, | ||
513 | gv11b_slcg_therm[i].prod); | ||
514 | else | ||
515 | gk20a_writel(g, gv11b_slcg_therm[i].addr, | ||
516 | gv11b_slcg_therm[i].disable); | ||
517 | } | ||
518 | } | ||
519 | |||
520 | void gv11b_slcg_xbar_load_gating_prod(struct gk20a *g, | ||
521 | bool prod) | ||
522 | { | ||
523 | u32 i; | ||
524 | u32 size = sizeof(gv11b_slcg_xbar) / sizeof(struct gating_desc); | ||
525 | |||
526 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) | ||
527 | return; | ||
528 | |||
529 | for (i = 0; i < size; i++) { | ||
530 | if (prod) | ||
531 | gk20a_writel(g, gv11b_slcg_xbar[i].addr, | ||
532 | gv11b_slcg_xbar[i].prod); | ||
533 | else | ||
534 | gk20a_writel(g, gv11b_slcg_xbar[i].addr, | ||
535 | gv11b_slcg_xbar[i].disable); | ||
536 | } | ||
537 | } | ||
538 | |||
539 | void gv11b_blcg_bus_load_gating_prod(struct gk20a *g, | ||
540 | bool prod) | ||
541 | { | ||
542 | u32 i; | ||
543 | u32 size = sizeof(gv11b_blcg_bus) / sizeof(struct gating_desc); | ||
544 | |||
545 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
546 | return; | ||
547 | |||
548 | for (i = 0; i < size; i++) { | ||
549 | if (prod) | ||
550 | gk20a_writel(g, gv11b_blcg_bus[i].addr, | ||
551 | gv11b_blcg_bus[i].prod); | ||
552 | else | ||
553 | gk20a_writel(g, gv11b_blcg_bus[i].addr, | ||
554 | gv11b_blcg_bus[i].disable); | ||
555 | } | ||
556 | } | ||
557 | |||
558 | void gv11b_blcg_ce_load_gating_prod(struct gk20a *g, | ||
559 | bool prod) | ||
560 | { | ||
561 | u32 i; | ||
562 | u32 size = sizeof(gv11b_blcg_ce) / sizeof(struct gating_desc); | ||
563 | |||
564 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
565 | return; | ||
566 | |||
567 | for (i = 0; i < size; i++) { | ||
568 | if (prod) | ||
569 | gk20a_writel(g, gv11b_blcg_ce[i].addr, | ||
570 | gv11b_blcg_ce[i].prod); | ||
571 | else | ||
572 | gk20a_writel(g, gv11b_blcg_ce[i].addr, | ||
573 | gv11b_blcg_ce[i].disable); | ||
574 | } | ||
575 | } | ||
576 | |||
577 | void gv11b_blcg_ctxsw_firmware_load_gating_prod(struct gk20a *g, | ||
578 | bool prod) | ||
579 | { | ||
580 | u32 i; | ||
581 | u32 size = sizeof(gv11b_blcg_ctxsw_prog) / sizeof(struct gating_desc); | ||
582 | |||
583 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
584 | return; | ||
585 | |||
586 | for (i = 0; i < size; i++) { | ||
587 | if (prod) | ||
588 | gk20a_writel(g, gv11b_blcg_ctxsw_prog[i].addr, | ||
589 | gv11b_blcg_ctxsw_prog[i].prod); | ||
590 | else | ||
591 | gk20a_writel(g, gv11b_blcg_ctxsw_prog[i].addr, | ||
592 | gv11b_blcg_ctxsw_prog[i].disable); | ||
593 | } | ||
594 | } | ||
595 | |||
596 | void gv11b_blcg_fb_load_gating_prod(struct gk20a *g, | ||
597 | bool prod) | ||
598 | { | ||
599 | u32 i; | ||
600 | u32 size = sizeof(gv11b_blcg_fb) / sizeof(struct gating_desc); | ||
601 | |||
602 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
603 | return; | ||
604 | |||
605 | for (i = 0; i < size; i++) { | ||
606 | if (prod) | ||
607 | gk20a_writel(g, gv11b_blcg_fb[i].addr, | ||
608 | gv11b_blcg_fb[i].prod); | ||
609 | else | ||
610 | gk20a_writel(g, gv11b_blcg_fb[i].addr, | ||
611 | gv11b_blcg_fb[i].disable); | ||
612 | } | ||
613 | } | ||
614 | |||
615 | void gv11b_blcg_fifo_load_gating_prod(struct gk20a *g, | ||
616 | bool prod) | ||
617 | { | ||
618 | u32 i; | ||
619 | u32 size = sizeof(gv11b_blcg_fifo) / sizeof(struct gating_desc); | ||
620 | |||
621 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
622 | return; | ||
623 | |||
624 | for (i = 0; i < size; i++) { | ||
625 | if (prod) | ||
626 | gk20a_writel(g, gv11b_blcg_fifo[i].addr, | ||
627 | gv11b_blcg_fifo[i].prod); | ||
628 | else | ||
629 | gk20a_writel(g, gv11b_blcg_fifo[i].addr, | ||
630 | gv11b_blcg_fifo[i].disable); | ||
631 | } | ||
632 | } | ||
633 | |||
634 | void gv11b_blcg_gr_load_gating_prod(struct gk20a *g, | ||
635 | bool prod) | ||
636 | { | ||
637 | u32 i; | ||
638 | u32 size = sizeof(gv11b_blcg_gr) / sizeof(struct gating_desc); | ||
639 | |||
640 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
641 | return; | ||
642 | |||
643 | for (i = 0; i < size; i++) { | ||
644 | if (prod) | ||
645 | gk20a_writel(g, gv11b_blcg_gr[i].addr, | ||
646 | gv11b_blcg_gr[i].prod); | ||
647 | else | ||
648 | gk20a_writel(g, gv11b_blcg_gr[i].addr, | ||
649 | gv11b_blcg_gr[i].disable); | ||
650 | } | ||
651 | } | ||
652 | |||
653 | void gv11b_blcg_ltc_load_gating_prod(struct gk20a *g, | ||
654 | bool prod) | ||
655 | { | ||
656 | u32 i; | ||
657 | u32 size = sizeof(gv11b_blcg_ltc) / sizeof(struct gating_desc); | ||
658 | |||
659 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
660 | return; | ||
661 | |||
662 | for (i = 0; i < size; i++) { | ||
663 | if (prod) | ||
664 | gk20a_writel(g, gv11b_blcg_ltc[i].addr, | ||
665 | gv11b_blcg_ltc[i].prod); | ||
666 | else | ||
667 | gk20a_writel(g, gv11b_blcg_ltc[i].addr, | ||
668 | gv11b_blcg_ltc[i].disable); | ||
669 | } | ||
670 | } | ||
671 | |||
672 | void gv11b_blcg_pwr_csb_load_gating_prod(struct gk20a *g, | ||
673 | bool prod) | ||
674 | { | ||
675 | u32 i; | ||
676 | u32 size = sizeof(gv11b_blcg_pwr_csb) / sizeof(struct gating_desc); | ||
677 | |||
678 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
679 | return; | ||
680 | |||
681 | for (i = 0; i < size; i++) { | ||
682 | if (prod) | ||
683 | gk20a_writel(g, gv11b_blcg_pwr_csb[i].addr, | ||
684 | gv11b_blcg_pwr_csb[i].prod); | ||
685 | else | ||
686 | gk20a_writel(g, gv11b_blcg_pwr_csb[i].addr, | ||
687 | gv11b_blcg_pwr_csb[i].disable); | ||
688 | } | ||
689 | } | ||
690 | |||
691 | void gv11b_blcg_pmu_load_gating_prod(struct gk20a *g, | ||
692 | bool prod) | ||
693 | { | ||
694 | u32 i; | ||
695 | u32 size = sizeof(gv11b_blcg_pmu) / sizeof(struct gating_desc); | ||
696 | |||
697 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
698 | return; | ||
699 | |||
700 | for (i = 0; i < size; i++) { | ||
701 | if (prod) | ||
702 | gk20a_writel(g, gv11b_blcg_pmu[i].addr, | ||
703 | gv11b_blcg_pmu[i].prod); | ||
704 | else | ||
705 | gk20a_writel(g, gv11b_blcg_pmu[i].addr, | ||
706 | gv11b_blcg_pmu[i].disable); | ||
707 | } | ||
708 | } | ||
709 | |||
710 | void gv11b_blcg_xbar_load_gating_prod(struct gk20a *g, | ||
711 | bool prod) | ||
712 | { | ||
713 | u32 i; | ||
714 | u32 size = sizeof(gv11b_blcg_xbar) / sizeof(struct gating_desc); | ||
715 | |||
716 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
717 | return; | ||
718 | |||
719 | for (i = 0; i < size; i++) { | ||
720 | if (prod) | ||
721 | gk20a_writel(g, gv11b_blcg_xbar[i].addr, | ||
722 | gv11b_blcg_xbar[i].prod); | ||
723 | else | ||
724 | gk20a_writel(g, gv11b_blcg_xbar[i].addr, | ||
725 | gv11b_blcg_xbar[i].disable); | ||
726 | } | ||
727 | } | ||
728 | |||
729 | void gr_gv11b_pg_gr_load_gating_prod(struct gk20a *g, | ||
730 | bool prod) | ||
731 | { | ||
732 | u32 i; | ||
733 | u32 size = sizeof(gv11b_pg_gr) / sizeof(struct gating_desc); | ||
734 | |||
735 | if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) | ||
736 | return; | ||
737 | |||
738 | for (i = 0; i < size; i++) { | ||
739 | if (prod) | ||
740 | gk20a_writel(g, gv11b_pg_gr[i].addr, | ||
741 | gv11b_pg_gr[i].prod); | ||
742 | else | ||
743 | gk20a_writel(g, gv11b_pg_gr[i].addr, | ||
744 | gv11b_pg_gr[i].disable); | ||
745 | } | ||
746 | } | ||
747 | |||
748 | #endif /* __gv11b_gating_reglist_h__ */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h new file mode 100644 index 00000000..233189e0 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/gv11b_gating_reglist.h | |||
@@ -0,0 +1,99 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016, NVIDIA Corporation. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #include "gk20a/gk20a.h" | ||
24 | |||
25 | void gv11b_slcg_bus_load_gating_prod(struct gk20a *g, | ||
26 | bool prod); | ||
27 | |||
28 | void gv11b_slcg_ce2_load_gating_prod(struct gk20a *g, | ||
29 | bool prod); | ||
30 | |||
31 | void gv11b_slcg_chiplet_load_gating_prod(struct gk20a *g, | ||
32 | bool prod); | ||
33 | |||
34 | void gv11b_slcg_ctxsw_firmware_load_gating_prod(struct gk20a *g, | ||
35 | bool prod); | ||
36 | |||
37 | void gv11b_slcg_fb_load_gating_prod(struct gk20a *g, | ||
38 | bool prod); | ||
39 | |||
40 | void gv11b_slcg_fifo_load_gating_prod(struct gk20a *g, | ||
41 | bool prod); | ||
42 | |||
43 | void gr_gv11b_slcg_gr_load_gating_prod(struct gk20a *g, | ||
44 | bool prod); | ||
45 | |||
46 | void ltc_gv11b_slcg_ltc_load_gating_prod(struct gk20a *g, | ||
47 | bool prod); | ||
48 | |||
49 | void gv11b_slcg_perf_load_gating_prod(struct gk20a *g, | ||
50 | bool prod); | ||
51 | |||
52 | void gv11b_slcg_priring_load_gating_prod(struct gk20a *g, | ||
53 | bool prod); | ||
54 | |||
55 | void gv11b_slcg_pwr_csb_load_gating_prod(struct gk20a *g, | ||
56 | bool prod); | ||
57 | |||
58 | void gv11b_slcg_pmu_load_gating_prod(struct gk20a *g, | ||
59 | bool prod); | ||
60 | |||
61 | void gv11b_slcg_therm_load_gating_prod(struct gk20a *g, | ||
62 | bool prod); | ||
63 | |||
64 | void gv11b_slcg_xbar_load_gating_prod(struct gk20a *g, | ||
65 | bool prod); | ||
66 | |||
67 | void gv11b_blcg_bus_load_gating_prod(struct gk20a *g, | ||
68 | bool prod); | ||
69 | |||
70 | void gv11b_blcg_ce_load_gating_prod(struct gk20a *g, | ||
71 | bool prod); | ||
72 | |||
73 | void gv11b_blcg_ctxsw_firmware_load_gating_prod(struct gk20a *g, | ||
74 | bool prod); | ||
75 | |||
76 | void gv11b_blcg_fb_load_gating_prod(struct gk20a *g, | ||
77 | bool prod); | ||
78 | |||
79 | void gv11b_blcg_fifo_load_gating_prod(struct gk20a *g, | ||
80 | bool prod); | ||
81 | |||
82 | void gv11b_blcg_gr_load_gating_prod(struct gk20a *g, | ||
83 | bool prod); | ||
84 | |||
85 | void gv11b_blcg_ltc_load_gating_prod(struct gk20a *g, | ||
86 | bool prod); | ||
87 | |||
88 | void gv11b_blcg_pwr_csb_load_gating_prod(struct gk20a *g, | ||
89 | bool prod); | ||
90 | |||
91 | void gv11b_blcg_pmu_load_gating_prod(struct gk20a *g, | ||
92 | bool prod); | ||
93 | |||
94 | void gv11b_blcg_xbar_load_gating_prod(struct gk20a *g, | ||
95 | bool prod); | ||
96 | |||
97 | void gr_gv11b_pg_gr_load_gating_prod(struct gk20a *g, | ||
98 | bool prod); | ||
99 | |||
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c new file mode 100644 index 00000000..fc059caa --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c | |||
@@ -0,0 +1,778 @@ | |||
1 | /* | ||
2 | * GV11B Tegra HAL interface | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | #include <linux/printk.h> | ||
27 | |||
28 | #include <linux/types.h> | ||
29 | #include <linux/tegra_gpu_t19x.h> | ||
30 | |||
31 | #include "gk20a/gk20a.h" | ||
32 | #include "gk20a/fifo_gk20a.h" | ||
33 | #include "gk20a/fecs_trace_gk20a.h" | ||
34 | #include "gk20a/css_gr_gk20a.h" | ||
35 | #include "gk20a/mc_gk20a.h" | ||
36 | #include "gk20a/mm_gk20a.h" | ||
37 | #include "gk20a/dbg_gpu_gk20a.h" | ||
38 | #include "gk20a/bus_gk20a.h" | ||
39 | #include "gk20a/flcn_gk20a.h" | ||
40 | #include "gk20a/regops_gk20a.h" | ||
41 | #include "gk20a/fb_gk20a.h" | ||
42 | #include "gk20a/pmu_gk20a.h" | ||
43 | #include "gk20a/gr_gk20a.h" | ||
44 | |||
45 | #include "gm20b/ltc_gm20b.h" | ||
46 | #include "gm20b/gr_gm20b.h" | ||
47 | #include "gm20b/fb_gm20b.h" | ||
48 | #include "gm20b/fifo_gm20b.h" | ||
49 | #include "gm20b/mm_gm20b.h" | ||
50 | #include "gm20b/acr_gm20b.h" | ||
51 | #include "gm20b/pmu_gm20b.h" | ||
52 | |||
53 | #include "gp10b/ltc_gp10b.h" | ||
54 | #include "gp10b/therm_gp10b.h" | ||
55 | #include "gp10b/mc_gp10b.h" | ||
56 | #include "gp10b/ce_gp10b.h" | ||
57 | #include "gp10b/priv_ring_gp10b.h" | ||
58 | #include "gp10b/fifo_gp10b.h" | ||
59 | #include "gp10b/fecs_trace_gp10b.h" | ||
60 | #include "gp10b/fb_gp10b.h" | ||
61 | #include "gp10b/mm_gp10b.h" | ||
62 | #include "gp10b/pmu_gp10b.h" | ||
63 | #include "gp10b/gr_gp10b.h" | ||
64 | |||
65 | #include "gp106/pmu_gp106.h" | ||
66 | #include "gp106/acr_gp106.h" | ||
67 | |||
68 | #include "gv100/gr_gv100.h" | ||
69 | |||
70 | #include "dbg_gpu_gv11b.h" | ||
71 | #include "hal_gv11b.h" | ||
72 | #include "css_gr_gv11b.h" | ||
73 | #include "gr_gv11b.h" | ||
74 | #include "mc_gv11b.h" | ||
75 | #include "ltc_gv11b.h" | ||
76 | #include "gv11b.h" | ||
77 | #include "ce_gv11b.h" | ||
78 | #include "gr_ctx_gv11b.h" | ||
79 | #include "mm_gv11b.h" | ||
80 | #include "pmu_gv11b.h" | ||
81 | #include "acr_gv11b.h" | ||
82 | #include "fb_gv11b.h" | ||
83 | #include "fifo_gv11b.h" | ||
84 | #include "gv11b_gating_reglist.h" | ||
85 | #include "regops_gv11b.h" | ||
86 | #include "subctx_gv11b.h" | ||
87 | #include "therm_gv11b.h" | ||
88 | |||
89 | #include <nvgpu/bus.h> | ||
90 | #include <nvgpu/debug.h> | ||
91 | #include <nvgpu/enabled.h> | ||
92 | #include <nvgpu/ctxsw_trace.h> | ||
93 | |||
94 | #include <nvgpu/hw/gv11b/hw_proj_gv11b.h> | ||
95 | #include <nvgpu/hw/gv11b/hw_fifo_gv11b.h> | ||
96 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
97 | #include <nvgpu/hw/gv11b/hw_top_gv11b.h> | ||
98 | #include <nvgpu/hw/gv11b/hw_pwr_gv11b.h> | ||
99 | #include <nvgpu/hw/gv11b/hw_fuse_gv11b.h> | ||
100 | |||
101 | int gv11b_get_litter_value(struct gk20a *g, int value) | ||
102 | { | ||
103 | int ret = EINVAL; | ||
104 | switch (value) { | ||
105 | case GPU_LIT_NUM_GPCS: | ||
106 | ret = proj_scal_litter_num_gpcs_v(); | ||
107 | break; | ||
108 | case GPU_LIT_NUM_PES_PER_GPC: | ||
109 | ret = proj_scal_litter_num_pes_per_gpc_v(); | ||
110 | break; | ||
111 | case GPU_LIT_NUM_ZCULL_BANKS: | ||
112 | ret = proj_scal_litter_num_zcull_banks_v(); | ||
113 | break; | ||
114 | case GPU_LIT_NUM_TPC_PER_GPC: | ||
115 | ret = proj_scal_litter_num_tpc_per_gpc_v(); | ||
116 | break; | ||
117 | case GPU_LIT_NUM_SM_PER_TPC: | ||
118 | ret = proj_scal_litter_num_sm_per_tpc_v(); | ||
119 | break; | ||
120 | case GPU_LIT_NUM_FBPS: | ||
121 | ret = proj_scal_litter_num_fbps_v(); | ||
122 | break; | ||
123 | case GPU_LIT_GPC_BASE: | ||
124 | ret = proj_gpc_base_v(); | ||
125 | break; | ||
126 | case GPU_LIT_GPC_STRIDE: | ||
127 | ret = proj_gpc_stride_v(); | ||
128 | break; | ||
129 | case GPU_LIT_GPC_SHARED_BASE: | ||
130 | ret = proj_gpc_shared_base_v(); | ||
131 | break; | ||
132 | case GPU_LIT_TPC_IN_GPC_BASE: | ||
133 | ret = proj_tpc_in_gpc_base_v(); | ||
134 | break; | ||
135 | case GPU_LIT_TPC_IN_GPC_STRIDE: | ||
136 | ret = proj_tpc_in_gpc_stride_v(); | ||
137 | break; | ||
138 | case GPU_LIT_TPC_IN_GPC_SHARED_BASE: | ||
139 | ret = proj_tpc_in_gpc_shared_base_v(); | ||
140 | break; | ||
141 | case GPU_LIT_PPC_IN_GPC_BASE: | ||
142 | ret = proj_ppc_in_gpc_base_v(); | ||
143 | break; | ||
144 | case GPU_LIT_PPC_IN_GPC_SHARED_BASE: | ||
145 | ret = proj_ppc_in_gpc_shared_base_v(); | ||
146 | break; | ||
147 | case GPU_LIT_PPC_IN_GPC_STRIDE: | ||
148 | ret = proj_ppc_in_gpc_stride_v(); | ||
149 | break; | ||
150 | case GPU_LIT_ROP_BASE: | ||
151 | ret = proj_rop_base_v(); | ||
152 | break; | ||
153 | case GPU_LIT_ROP_STRIDE: | ||
154 | ret = proj_rop_stride_v(); | ||
155 | break; | ||
156 | case GPU_LIT_ROP_SHARED_BASE: | ||
157 | ret = proj_rop_shared_base_v(); | ||
158 | break; | ||
159 | case GPU_LIT_HOST_NUM_ENGINES: | ||
160 | ret = proj_host_num_engines_v(); | ||
161 | break; | ||
162 | case GPU_LIT_HOST_NUM_PBDMA: | ||
163 | ret = proj_host_num_pbdma_v(); | ||
164 | break; | ||
165 | case GPU_LIT_LTC_STRIDE: | ||
166 | ret = proj_ltc_stride_v(); | ||
167 | break; | ||
168 | case GPU_LIT_LTS_STRIDE: | ||
169 | ret = proj_lts_stride_v(); | ||
170 | break; | ||
171 | case GPU_LIT_SM_PRI_STRIDE: | ||
172 | ret = proj_sm_stride_v(); | ||
173 | break; | ||
174 | case GPU_LIT_SMPC_PRI_BASE: | ||
175 | ret = proj_smpc_base_v(); | ||
176 | break; | ||
177 | case GPU_LIT_SMPC_PRI_SHARED_BASE: | ||
178 | ret = proj_smpc_shared_base_v(); | ||
179 | break; | ||
180 | case GPU_LIT_SMPC_PRI_UNIQUE_BASE: | ||
181 | ret = proj_smpc_unique_base_v(); | ||
182 | break; | ||
183 | case GPU_LIT_SMPC_PRI_STRIDE: | ||
184 | ret = proj_smpc_stride_v(); | ||
185 | break; | ||
186 | /* Even though GV11B doesn't have an FBPA unit, the HW reports one, | ||
187 | * and the microcode as a result leaves space in the context buffer | ||
188 | * for one, so make sure SW accounts for this also. | ||
189 | */ | ||
190 | case GPU_LIT_NUM_FBPAS: | ||
191 | ret = proj_scal_litter_num_fbpas_v(); | ||
192 | break; | ||
193 | /* Hardcode FBPA values other than NUM_FBPAS to 0. */ | ||
194 | case GPU_LIT_FBPA_STRIDE: | ||
195 | case GPU_LIT_FBPA_BASE: | ||
196 | case GPU_LIT_FBPA_SHARED_BASE: | ||
197 | ret = 0; | ||
198 | break; | ||
199 | case GPU_LIT_TWOD_CLASS: | ||
200 | ret = FERMI_TWOD_A; | ||
201 | break; | ||
202 | case GPU_LIT_THREED_CLASS: | ||
203 | ret = VOLTA_A; | ||
204 | break; | ||
205 | case GPU_LIT_COMPUTE_CLASS: | ||
206 | ret = VOLTA_COMPUTE_A; | ||
207 | break; | ||
208 | case GPU_LIT_GPFIFO_CLASS: | ||
209 | ret = VOLTA_CHANNEL_GPFIFO_A; | ||
210 | break; | ||
211 | case GPU_LIT_I2M_CLASS: | ||
212 | ret = KEPLER_INLINE_TO_MEMORY_B; | ||
213 | break; | ||
214 | case GPU_LIT_DMA_COPY_CLASS: | ||
215 | ret = VOLTA_DMA_COPY_A; | ||
216 | break; | ||
217 | |||
218 | default: | ||
219 | nvgpu_err(g, "Missing definition %d", value); | ||
220 | BUG(); | ||
221 | break; | ||
222 | } | ||
223 | |||
224 | return ret; | ||
225 | } | ||
226 | |||
227 | static const struct gpu_ops gv11b_ops = { | ||
228 | .ltc = { | ||
229 | .determine_L2_size_bytes = gp10b_determine_L2_size_bytes, | ||
230 | .set_zbc_s_entry = gv11b_ltc_set_zbc_stencil_entry, | ||
231 | .set_zbc_color_entry = gm20b_ltc_set_zbc_color_entry, | ||
232 | .set_zbc_depth_entry = gm20b_ltc_set_zbc_depth_entry, | ||
233 | .init_cbc = NULL, | ||
234 | .init_fs_state = gv11b_ltc_init_fs_state, | ||
235 | .init_comptags = gp10b_ltc_init_comptags, | ||
236 | .cbc_ctrl = gm20b_ltc_cbc_ctrl, | ||
237 | .isr = gv11b_ltc_isr, | ||
238 | .cbc_fix_config = gv11b_ltc_cbc_fix_config, | ||
239 | .flush = gm20b_flush_ltc, | ||
240 | .set_enabled = gp10b_ltc_set_enabled, | ||
241 | }, | ||
242 | .ce2 = { | ||
243 | .isr_stall = gv11b_ce_isr, | ||
244 | .isr_nonstall = gp10b_ce_nonstall_isr, | ||
245 | .get_num_pce = gv11b_ce_get_num_pce, | ||
246 | }, | ||
247 | .gr = { | ||
248 | .get_patch_slots = gr_gv100_get_patch_slots, | ||
249 | .init_gpc_mmu = gr_gv11b_init_gpc_mmu, | ||
250 | .bundle_cb_defaults = gr_gv11b_bundle_cb_defaults, | ||
251 | .cb_size_default = gr_gv11b_cb_size_default, | ||
252 | .calc_global_ctx_buffer_size = | ||
253 | gr_gv11b_calc_global_ctx_buffer_size, | ||
254 | .commit_global_attrib_cb = gr_gv11b_commit_global_attrib_cb, | ||
255 | .commit_global_bundle_cb = gr_gp10b_commit_global_bundle_cb, | ||
256 | .commit_global_cb_manager = gr_gp10b_commit_global_cb_manager, | ||
257 | .commit_global_pagepool = gr_gp10b_commit_global_pagepool, | ||
258 | .handle_sw_method = gr_gv11b_handle_sw_method, | ||
259 | .set_alpha_circular_buffer_size = | ||
260 | gr_gv11b_set_alpha_circular_buffer_size, | ||
261 | .set_circular_buffer_size = gr_gv11b_set_circular_buffer_size, | ||
262 | .enable_hww_exceptions = gr_gv11b_enable_hww_exceptions, | ||
263 | .is_valid_class = gr_gv11b_is_valid_class, | ||
264 | .is_valid_gfx_class = gr_gv11b_is_valid_gfx_class, | ||
265 | .is_valid_compute_class = gr_gv11b_is_valid_compute_class, | ||
266 | .get_sm_dsm_perf_regs = gv11b_gr_get_sm_dsm_perf_regs, | ||
267 | .get_sm_dsm_perf_ctrl_regs = gv11b_gr_get_sm_dsm_perf_ctrl_regs, | ||
268 | .init_fs_state = gr_gv11b_init_fs_state, | ||
269 | .set_hww_esr_report_mask = gv11b_gr_set_hww_esr_report_mask, | ||
270 | .falcon_load_ucode = gr_gm20b_load_ctxsw_ucode_segments, | ||
271 | .load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode, | ||
272 | .set_gpc_tpc_mask = gr_gv11b_set_gpc_tpc_mask, | ||
273 | .get_gpc_tpc_mask = gr_gm20b_get_gpc_tpc_mask, | ||
274 | .free_channel_ctx = gk20a_free_channel_ctx, | ||
275 | .alloc_obj_ctx = gk20a_alloc_obj_ctx, | ||
276 | .bind_ctxsw_zcull = gr_gk20a_bind_ctxsw_zcull, | ||
277 | .get_zcull_info = gr_gk20a_get_zcull_info, | ||
278 | .is_tpc_addr = gr_gm20b_is_tpc_addr, | ||
279 | .get_tpc_num = gr_gm20b_get_tpc_num, | ||
280 | .detect_sm_arch = gr_gv11b_detect_sm_arch, | ||
281 | .add_zbc_color = gr_gp10b_add_zbc_color, | ||
282 | .add_zbc_depth = gr_gp10b_add_zbc_depth, | ||
283 | .zbc_set_table = gk20a_gr_zbc_set_table, | ||
284 | .zbc_query_table = gr_gk20a_query_zbc, | ||
285 | .pmu_save_zbc = gk20a_pmu_save_zbc, | ||
286 | .add_zbc = gr_gk20a_add_zbc, | ||
287 | .pagepool_default_size = gr_gv11b_pagepool_default_size, | ||
288 | .init_ctx_state = gr_gp10b_init_ctx_state, | ||
289 | .alloc_gr_ctx = gr_gp10b_alloc_gr_ctx, | ||
290 | .free_gr_ctx = gr_gp10b_free_gr_ctx, | ||
291 | .update_ctxsw_preemption_mode = | ||
292 | gr_gp10b_update_ctxsw_preemption_mode, | ||
293 | .dump_gr_regs = gr_gv11b_dump_gr_status_regs, | ||
294 | .update_pc_sampling = gr_gm20b_update_pc_sampling, | ||
295 | .get_fbp_en_mask = gr_gm20b_get_fbp_en_mask, | ||
296 | .get_max_ltc_per_fbp = gr_gm20b_get_max_ltc_per_fbp, | ||
297 | .get_max_lts_per_ltc = gr_gm20b_get_max_lts_per_ltc, | ||
298 | .get_rop_l2_en_mask = gr_gm20b_rop_l2_en_mask, | ||
299 | .get_max_fbps_count = gr_gm20b_get_max_fbps_count, | ||
300 | .init_sm_dsm_reg_info = gv11b_gr_init_sm_dsm_reg_info, | ||
301 | .wait_empty = gr_gv11b_wait_empty, | ||
302 | .init_cyclestats = gr_gm20b_init_cyclestats, | ||
303 | .set_sm_debug_mode = gv11b_gr_set_sm_debug_mode, | ||
304 | .enable_cde_in_fecs = gr_gm20b_enable_cde_in_fecs, | ||
305 | .bpt_reg_info = gv11b_gr_bpt_reg_info, | ||
306 | .get_access_map = gr_gv11b_get_access_map, | ||
307 | .handle_fecs_error = gr_gv11b_handle_fecs_error, | ||
308 | .handle_sm_exception = gr_gk20a_handle_sm_exception, | ||
309 | .handle_tex_exception = gr_gv11b_handle_tex_exception, | ||
310 | .enable_gpc_exceptions = gr_gv11b_enable_gpc_exceptions, | ||
311 | .enable_exceptions = gr_gv11b_enable_exceptions, | ||
312 | .get_lrf_tex_ltc_dram_override = get_ecc_override_val, | ||
313 | .update_smpc_ctxsw_mode = gr_gk20a_update_smpc_ctxsw_mode, | ||
314 | .update_hwpm_ctxsw_mode = gr_gk20a_update_hwpm_ctxsw_mode, | ||
315 | .record_sm_error_state = gv11b_gr_record_sm_error_state, | ||
316 | .update_sm_error_state = gv11b_gr_update_sm_error_state, | ||
317 | .clear_sm_error_state = gm20b_gr_clear_sm_error_state, | ||
318 | .suspend_contexts = gr_gp10b_suspend_contexts, | ||
319 | .resume_contexts = gr_gk20a_resume_contexts, | ||
320 | .get_preemption_mode_flags = gr_gp10b_get_preemption_mode_flags, | ||
321 | .init_sm_id_table = gr_gv100_init_sm_id_table, | ||
322 | .load_smid_config = gr_gv11b_load_smid_config, | ||
323 | .program_sm_id_numbering = gr_gv11b_program_sm_id_numbering, | ||
324 | .is_ltcs_ltss_addr = gr_gm20b_is_ltcs_ltss_addr, | ||
325 | .is_ltcn_ltss_addr = gr_gm20b_is_ltcn_ltss_addr, | ||
326 | .split_lts_broadcast_addr = gr_gm20b_split_lts_broadcast_addr, | ||
327 | .split_ltc_broadcast_addr = gr_gm20b_split_ltc_broadcast_addr, | ||
328 | .setup_rop_mapping = gr_gv11b_setup_rop_mapping, | ||
329 | .program_zcull_mapping = gr_gv11b_program_zcull_mapping, | ||
330 | .commit_global_timeslice = gr_gv11b_commit_global_timeslice, | ||
331 | .commit_inst = gr_gv11b_commit_inst, | ||
332 | .write_zcull_ptr = gr_gv11b_write_zcull_ptr, | ||
333 | .write_pm_ptr = gr_gv11b_write_pm_ptr, | ||
334 | .init_elcg_mode = gr_gv11b_init_elcg_mode, | ||
335 | .load_tpc_mask = gr_gv11b_load_tpc_mask, | ||
336 | .inval_icache = gr_gk20a_inval_icache, | ||
337 | .trigger_suspend = gv11b_gr_sm_trigger_suspend, | ||
338 | .wait_for_pause = gr_gk20a_wait_for_pause, | ||
339 | .resume_from_pause = gv11b_gr_resume_from_pause, | ||
340 | .clear_sm_errors = gr_gk20a_clear_sm_errors, | ||
341 | .tpc_enabled_exceptions = gr_gk20a_tpc_enabled_exceptions, | ||
342 | .get_esr_sm_sel = gv11b_gr_get_esr_sm_sel, | ||
343 | .sm_debugger_attached = gv11b_gr_sm_debugger_attached, | ||
344 | .suspend_single_sm = gv11b_gr_suspend_single_sm, | ||
345 | .suspend_all_sms = gv11b_gr_suspend_all_sms, | ||
346 | .resume_single_sm = gv11b_gr_resume_single_sm, | ||
347 | .resume_all_sms = gv11b_gr_resume_all_sms, | ||
348 | .get_sm_hww_warp_esr = gv11b_gr_get_sm_hww_warp_esr, | ||
349 | .get_sm_hww_global_esr = gv11b_gr_get_sm_hww_global_esr, | ||
350 | .get_sm_no_lock_down_hww_global_esr_mask = | ||
351 | gv11b_gr_get_sm_no_lock_down_hww_global_esr_mask, | ||
352 | .lock_down_sm = gv11b_gr_lock_down_sm, | ||
353 | .wait_for_sm_lock_down = gv11b_gr_wait_for_sm_lock_down, | ||
354 | .clear_sm_hww = gv11b_gr_clear_sm_hww, | ||
355 | .init_ovr_sm_dsm_perf = gv11b_gr_init_ovr_sm_dsm_perf, | ||
356 | .get_ovr_perf_regs = gv11b_gr_get_ovr_perf_regs, | ||
357 | .disable_rd_coalesce = gm20a_gr_disable_rd_coalesce, | ||
358 | .set_boosted_ctx = gr_gp10b_set_boosted_ctx, | ||
359 | .set_preemption_mode = gr_gp10b_set_preemption_mode, | ||
360 | .set_czf_bypass = NULL, | ||
361 | .pre_process_sm_exception = gr_gv11b_pre_process_sm_exception, | ||
362 | .set_preemption_buffer_va = gr_gv11b_set_preemption_buffer_va, | ||
363 | .init_preemption_state = NULL, | ||
364 | .update_boosted_ctx = gr_gp10b_update_boosted_ctx, | ||
365 | .set_bes_crop_debug3 = gr_gp10b_set_bes_crop_debug3, | ||
366 | .create_gr_sysfs = gr_gv11b_create_sysfs, | ||
367 | .set_ctxsw_preemption_mode = gr_gp10b_set_ctxsw_preemption_mode, | ||
368 | .is_etpc_addr = gv11b_gr_pri_is_etpc_addr, | ||
369 | .egpc_etpc_priv_addr_table = gv11b_gr_egpc_etpc_priv_addr_table, | ||
370 | .handle_tpc_mpc_exception = gr_gv11b_handle_tpc_mpc_exception, | ||
371 | .zbc_s_query_table = gr_gv11b_zbc_s_query_table, | ||
372 | .load_zbc_s_default_tbl = gr_gv11b_load_stencil_default_tbl, | ||
373 | .handle_gpc_gpcmmu_exception = | ||
374 | gr_gv11b_handle_gpc_gpcmmu_exception, | ||
375 | .add_zbc_type_s = gr_gv11b_add_zbc_type_s, | ||
376 | .get_egpc_base = gv11b_gr_get_egpc_base, | ||
377 | .get_egpc_etpc_num = gv11b_gr_get_egpc_etpc_num, | ||
378 | .handle_gpc_gpccs_exception = | ||
379 | gr_gv11b_handle_gpc_gpccs_exception, | ||
380 | .load_zbc_s_tbl = gr_gv11b_load_stencil_tbl, | ||
381 | .access_smpc_reg = gv11b_gr_access_smpc_reg, | ||
382 | .is_egpc_addr = gv11b_gr_pri_is_egpc_addr, | ||
383 | .add_zbc_s = gr_gv11b_add_zbc_stencil, | ||
384 | .handle_gcc_exception = gr_gv11b_handle_gcc_exception, | ||
385 | .init_sw_veid_bundle = gr_gv11b_init_sw_veid_bundle, | ||
386 | .handle_tpc_sm_ecc_exception = | ||
387 | gr_gv11b_handle_tpc_sm_ecc_exception, | ||
388 | .decode_egpc_addr = gv11b_gr_decode_egpc_addr, | ||
389 | .init_ctxsw_hdr_data = gr_gp10b_init_ctxsw_hdr_data, | ||
390 | }, | ||
391 | .fb = { | ||
392 | .reset = gv11b_fb_reset, | ||
393 | .init_hw = gk20a_fb_init_hw, | ||
394 | .init_fs_state = gv11b_fb_init_fs_state, | ||
395 | .init_cbc = gv11b_fb_init_cbc, | ||
396 | .set_mmu_page_size = gm20b_fb_set_mmu_page_size, | ||
397 | .set_use_full_comp_tag_line = | ||
398 | gm20b_fb_set_use_full_comp_tag_line, | ||
399 | .compression_page_size = gp10b_fb_compression_page_size, | ||
400 | .compressible_page_size = gp10b_fb_compressible_page_size, | ||
401 | .vpr_info_fetch = gm20b_fb_vpr_info_fetch, | ||
402 | .dump_vpr_wpr_info = gm20b_fb_dump_vpr_wpr_info, | ||
403 | .read_wpr_info = gm20b_fb_read_wpr_info, | ||
404 | .is_debug_mode_enabled = gm20b_fb_debug_mode_enabled, | ||
405 | .set_debug_mode = gm20b_fb_set_debug_mode, | ||
406 | .tlb_invalidate = gk20a_fb_tlb_invalidate, | ||
407 | .hub_isr = gv11b_fb_hub_isr, | ||
408 | .mem_unlock = NULL, | ||
409 | }, | ||
410 | .clock_gating = { | ||
411 | .slcg_bus_load_gating_prod = | ||
412 | gv11b_slcg_bus_load_gating_prod, | ||
413 | .slcg_ce2_load_gating_prod = | ||
414 | gv11b_slcg_ce2_load_gating_prod, | ||
415 | .slcg_chiplet_load_gating_prod = | ||
416 | gv11b_slcg_chiplet_load_gating_prod, | ||
417 | .slcg_ctxsw_firmware_load_gating_prod = | ||
418 | gv11b_slcg_ctxsw_firmware_load_gating_prod, | ||
419 | .slcg_fb_load_gating_prod = | ||
420 | gv11b_slcg_fb_load_gating_prod, | ||
421 | .slcg_fifo_load_gating_prod = | ||
422 | gv11b_slcg_fifo_load_gating_prod, | ||
423 | .slcg_gr_load_gating_prod = | ||
424 | gr_gv11b_slcg_gr_load_gating_prod, | ||
425 | .slcg_ltc_load_gating_prod = | ||
426 | ltc_gv11b_slcg_ltc_load_gating_prod, | ||
427 | .slcg_perf_load_gating_prod = | ||
428 | gv11b_slcg_perf_load_gating_prod, | ||
429 | .slcg_priring_load_gating_prod = | ||
430 | gv11b_slcg_priring_load_gating_prod, | ||
431 | .slcg_pmu_load_gating_prod = | ||
432 | gv11b_slcg_pmu_load_gating_prod, | ||
433 | .slcg_therm_load_gating_prod = | ||
434 | gv11b_slcg_therm_load_gating_prod, | ||
435 | .slcg_xbar_load_gating_prod = | ||
436 | gv11b_slcg_xbar_load_gating_prod, | ||
437 | .blcg_bus_load_gating_prod = | ||
438 | gv11b_blcg_bus_load_gating_prod, | ||
439 | .blcg_ce_load_gating_prod = | ||
440 | gv11b_blcg_ce_load_gating_prod, | ||
441 | .blcg_ctxsw_firmware_load_gating_prod = | ||
442 | gv11b_blcg_ctxsw_firmware_load_gating_prod, | ||
443 | .blcg_fb_load_gating_prod = | ||
444 | gv11b_blcg_fb_load_gating_prod, | ||
445 | .blcg_fifo_load_gating_prod = | ||
446 | gv11b_blcg_fifo_load_gating_prod, | ||
447 | .blcg_gr_load_gating_prod = | ||
448 | gv11b_blcg_gr_load_gating_prod, | ||
449 | .blcg_ltc_load_gating_prod = | ||
450 | gv11b_blcg_ltc_load_gating_prod, | ||
451 | .blcg_pwr_csb_load_gating_prod = | ||
452 | gv11b_blcg_pwr_csb_load_gating_prod, | ||
453 | .blcg_pmu_load_gating_prod = | ||
454 | gv11b_blcg_pmu_load_gating_prod, | ||
455 | .blcg_xbar_load_gating_prod = | ||
456 | gv11b_blcg_xbar_load_gating_prod, | ||
457 | .pg_gr_load_gating_prod = | ||
458 | gr_gv11b_pg_gr_load_gating_prod, | ||
459 | }, | ||
460 | .fifo = { | ||
461 | .get_preempt_timeout = gv11b_fifo_get_preempt_timeout, | ||
462 | .init_fifo_setup_hw = gv11b_init_fifo_setup_hw, | ||
463 | .bind_channel = channel_gm20b_bind, | ||
464 | .unbind_channel = channel_gv11b_unbind, | ||
465 | .disable_channel = gk20a_fifo_disable_channel, | ||
466 | .enable_channel = gk20a_fifo_enable_channel, | ||
467 | .alloc_inst = gk20a_fifo_alloc_inst, | ||
468 | .free_inst = gk20a_fifo_free_inst, | ||
469 | .setup_ramfc = channel_gv11b_setup_ramfc, | ||
470 | .channel_set_timeslice = gk20a_fifo_set_timeslice, | ||
471 | .default_timeslice_us = gk20a_fifo_default_timeslice_us, | ||
472 | .setup_userd = gk20a_fifo_setup_userd, | ||
473 | .userd_gp_get = gv11b_userd_gp_get, | ||
474 | .userd_gp_put = gv11b_userd_gp_put, | ||
475 | .userd_pb_get = gv11b_userd_pb_get, | ||
476 | .pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val, | ||
477 | .preempt_channel = gv11b_fifo_preempt_channel, | ||
478 | .preempt_tsg = gv11b_fifo_preempt_tsg, | ||
479 | .enable_tsg = gv11b_fifo_enable_tsg, | ||
480 | .disable_tsg = gk20a_disable_tsg, | ||
481 | .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, | ||
482 | .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, | ||
483 | .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, | ||
484 | .update_runlist = gk20a_fifo_update_runlist, | ||
485 | .trigger_mmu_fault = NULL, | ||
486 | .get_mmu_fault_info = NULL, | ||
487 | .wait_engine_idle = gk20a_fifo_wait_engine_idle, | ||
488 | .get_num_fifos = gv11b_fifo_get_num_fifos, | ||
489 | .get_pbdma_signature = gp10b_fifo_get_pbdma_signature, | ||
490 | .set_runlist_interleave = gk20a_fifo_set_runlist_interleave, | ||
491 | .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, | ||
492 | .force_reset_ch = gk20a_fifo_force_reset_ch, | ||
493 | .engine_enum_from_type = gp10b_fifo_engine_enum_from_type, | ||
494 | .device_info_data_parse = gp10b_device_info_data_parse, | ||
495 | .eng_runlist_base_size = fifo_eng_runlist_base__size_1_v, | ||
496 | .init_engine_info = gk20a_fifo_init_engine_info, | ||
497 | .runlist_entry_size = ram_rl_entry_size_v, | ||
498 | .get_tsg_runlist_entry = gv11b_get_tsg_runlist_entry, | ||
499 | .get_ch_runlist_entry = gv11b_get_ch_runlist_entry, | ||
500 | .is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc, | ||
501 | .dump_pbdma_status = gk20a_dump_pbdma_status, | ||
502 | .dump_eng_status = gv11b_dump_eng_status, | ||
503 | .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc, | ||
504 | .intr_0_error_mask = gv11b_fifo_intr_0_error_mask, | ||
505 | .is_preempt_pending = gv11b_fifo_is_preempt_pending, | ||
506 | .init_pbdma_intr_descs = gv11b_fifo_init_pbdma_intr_descs, | ||
507 | .reset_enable_hw = gv11b_init_fifo_reset_enable_hw, | ||
508 | .teardown_ch_tsg = gv11b_fifo_teardown_ch_tsg, | ||
509 | .handle_sched_error = gv11b_fifo_handle_sched_error, | ||
510 | .handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0, | ||
511 | .handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1, | ||
512 | .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers, | ||
513 | .deinit_eng_method_buffers = | ||
514 | gv11b_fifo_deinit_eng_method_buffers, | ||
515 | .tsg_bind_channel = gk20a_tsg_bind_channel, | ||
516 | .tsg_unbind_channel = gk20a_tsg_unbind_channel, | ||
517 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
518 | .alloc_syncpt_buf = gv11b_fifo_alloc_syncpt_buf, | ||
519 | .free_syncpt_buf = gv11b_fifo_free_syncpt_buf, | ||
520 | .add_syncpt_wait_cmd = gv11b_fifo_add_syncpt_wait_cmd, | ||
521 | .get_syncpt_wait_cmd_size = gv11b_fifo_get_syncpt_wait_cmd_size, | ||
522 | .add_syncpt_incr_cmd = gv11b_fifo_add_syncpt_incr_cmd, | ||
523 | .get_syncpt_incr_cmd_size = gv11b_fifo_get_syncpt_incr_cmd_size, | ||
524 | #endif | ||
525 | .resetup_ramfc = NULL, | ||
526 | .device_info_fault_id = top_device_info_data_fault_id_enum_v, | ||
527 | .free_channel_ctx_header = gv11b_free_subctx_header, | ||
528 | .preempt_ch_tsg = gv11b_fifo_preempt_ch_tsg, | ||
529 | .handle_ctxsw_timeout = gv11b_fifo_handle_ctxsw_timeout, | ||
530 | }, | ||
531 | .gr_ctx = { | ||
532 | .get_netlist_name = gr_gv11b_get_netlist_name, | ||
533 | .is_fw_defined = gr_gv11b_is_firmware_defined, | ||
534 | }, | ||
535 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
536 | .fecs_trace = { | ||
537 | .alloc_user_buffer = NULL, | ||
538 | .free_user_buffer = NULL, | ||
539 | .mmap_user_buffer = NULL, | ||
540 | .init = NULL, | ||
541 | .deinit = NULL, | ||
542 | .enable = NULL, | ||
543 | .disable = NULL, | ||
544 | .is_enabled = NULL, | ||
545 | .reset = NULL, | ||
546 | .flush = NULL, | ||
547 | .poll = NULL, | ||
548 | .bind_channel = NULL, | ||
549 | .unbind_channel = NULL, | ||
550 | .max_entries = NULL, | ||
551 | }, | ||
552 | #endif /* CONFIG_GK20A_CTXSW_TRACE */ | ||
553 | .mm = { | ||
554 | .support_sparse = gm20b_mm_support_sparse, | ||
555 | .gmmu_map = gk20a_locked_gmmu_map, | ||
556 | .gmmu_unmap = gk20a_locked_gmmu_unmap, | ||
557 | .vm_bind_channel = gk20a_vm_bind_channel, | ||
558 | .fb_flush = gk20a_mm_fb_flush, | ||
559 | .l2_invalidate = gk20a_mm_l2_invalidate, | ||
560 | .l2_flush = gv11b_mm_l2_flush, | ||
561 | .cbc_clean = gk20a_mm_cbc_clean, | ||
562 | .set_big_page_size = gm20b_mm_set_big_page_size, | ||
563 | .get_big_page_sizes = gm20b_mm_get_big_page_sizes, | ||
564 | .get_default_big_page_size = gp10b_mm_get_default_big_page_size, | ||
565 | .gpu_phys_addr = gv11b_gpu_phys_addr, | ||
566 | .get_iommu_bit = gp10b_mm_get_iommu_bit, | ||
567 | .get_mmu_levels = gp10b_mm_get_mmu_levels, | ||
568 | .init_pdb = gp10b_mm_init_pdb, | ||
569 | .init_mm_setup_hw = gv11b_init_mm_setup_hw, | ||
570 | .is_bar1_supported = gv11b_mm_is_bar1_supported, | ||
571 | .alloc_inst_block = gk20a_alloc_inst_block, | ||
572 | .init_inst_block = gv11b_init_inst_block, | ||
573 | .mmu_fault_pending = gv11b_mm_mmu_fault_pending, | ||
574 | .get_kind_invalid = gm20b_get_kind_invalid, | ||
575 | .get_kind_pitch = gm20b_get_kind_pitch, | ||
576 | .init_bar2_vm = gb10b_init_bar2_vm, | ||
577 | .init_bar2_mm_hw_setup = gv11b_init_bar2_mm_hw_setup, | ||
578 | .remove_bar2_vm = gv11b_mm_remove_bar2_vm, | ||
579 | .fault_info_mem_destroy = gv11b_mm_fault_info_mem_destroy, | ||
580 | }, | ||
581 | .therm = { | ||
582 | .init_therm_setup_hw = gp10b_init_therm_setup_hw, | ||
583 | .elcg_init_idle_filters = gv11b_elcg_init_idle_filters, | ||
584 | }, | ||
585 | .pmu = { | ||
586 | .pmu_setup_elpg = gp10b_pmu_setup_elpg, | ||
587 | .pmu_get_queue_head = pwr_pmu_queue_head_r, | ||
588 | .pmu_get_queue_head_size = pwr_pmu_queue_head__size_1_v, | ||
589 | .pmu_get_queue_tail = pwr_pmu_queue_tail_r, | ||
590 | .pmu_get_queue_tail_size = pwr_pmu_queue_tail__size_1_v, | ||
591 | .pmu_queue_head = gk20a_pmu_queue_head, | ||
592 | .pmu_queue_tail = gk20a_pmu_queue_tail, | ||
593 | .pmu_msgq_tail = gk20a_pmu_msgq_tail, | ||
594 | .pmu_mutex_size = pwr_pmu_mutex__size_1_v, | ||
595 | .pmu_mutex_acquire = gk20a_pmu_mutex_acquire, | ||
596 | .pmu_mutex_release = gk20a_pmu_mutex_release, | ||
597 | .write_dmatrfbase = gp10b_write_dmatrfbase, | ||
598 | .pmu_elpg_statistics = gp106_pmu_elpg_statistics, | ||
599 | .pmu_pg_init_param = gv11b_pg_gr_init, | ||
600 | .pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list, | ||
601 | .pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list, | ||
602 | .dump_secure_fuses = pmu_dump_security_fuses_gp10b, | ||
603 | .reset_engine = gp106_pmu_engine_reset, | ||
604 | .is_engine_in_reset = gp106_pmu_is_engine_in_reset, | ||
605 | .pmu_nsbootstrap = gv11b_pmu_bootstrap, | ||
606 | .pmu_pg_set_sub_feature_mask = gv11b_pg_set_subfeature_mask, | ||
607 | .is_pmu_supported = gv11b_is_pmu_supported, | ||
608 | }, | ||
609 | .regops = { | ||
610 | .get_global_whitelist_ranges = | ||
611 | gv11b_get_global_whitelist_ranges, | ||
612 | .get_global_whitelist_ranges_count = | ||
613 | gv11b_get_global_whitelist_ranges_count, | ||
614 | .get_context_whitelist_ranges = | ||
615 | gv11b_get_context_whitelist_ranges, | ||
616 | .get_context_whitelist_ranges_count = | ||
617 | gv11b_get_context_whitelist_ranges_count, | ||
618 | .get_runcontrol_whitelist = gv11b_get_runcontrol_whitelist, | ||
619 | .get_runcontrol_whitelist_count = | ||
620 | gv11b_get_runcontrol_whitelist_count, | ||
621 | .get_runcontrol_whitelist_ranges = | ||
622 | gv11b_get_runcontrol_whitelist_ranges, | ||
623 | .get_runcontrol_whitelist_ranges_count = | ||
624 | gv11b_get_runcontrol_whitelist_ranges_count, | ||
625 | .get_qctl_whitelist = gv11b_get_qctl_whitelist, | ||
626 | .get_qctl_whitelist_count = gv11b_get_qctl_whitelist_count, | ||
627 | .get_qctl_whitelist_ranges = gv11b_get_qctl_whitelist_ranges, | ||
628 | .get_qctl_whitelist_ranges_count = | ||
629 | gv11b_get_qctl_whitelist_ranges_count, | ||
630 | .apply_smpc_war = gv11b_apply_smpc_war, | ||
631 | }, | ||
632 | .mc = { | ||
633 | .intr_enable = mc_gv11b_intr_enable, | ||
634 | .intr_unit_config = mc_gp10b_intr_unit_config, | ||
635 | .isr_stall = mc_gp10b_isr_stall, | ||
636 | .intr_stall = mc_gp10b_intr_stall, | ||
637 | .intr_stall_pause = mc_gp10b_intr_stall_pause, | ||
638 | .intr_stall_resume = mc_gp10b_intr_stall_resume, | ||
639 | .intr_nonstall = mc_gp10b_intr_nonstall, | ||
640 | .intr_nonstall_pause = mc_gp10b_intr_nonstall_pause, | ||
641 | .intr_nonstall_resume = mc_gp10b_intr_nonstall_resume, | ||
642 | .enable = gk20a_mc_enable, | ||
643 | .disable = gk20a_mc_disable, | ||
644 | .reset = gk20a_mc_reset, | ||
645 | .boot_0 = gk20a_mc_boot_0, | ||
646 | .is_intr1_pending = mc_gp10b_is_intr1_pending, | ||
647 | .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, | ||
648 | }, | ||
649 | .debug = { | ||
650 | .show_dump = gk20a_debug_show_dump, | ||
651 | }, | ||
652 | .dbg_session_ops = { | ||
653 | .exec_reg_ops = exec_regops_gk20a, | ||
654 | .dbg_set_powergate = dbg_set_powergate, | ||
655 | .check_and_set_global_reservation = | ||
656 | nvgpu_check_and_set_global_reservation, | ||
657 | .check_and_set_context_reservation = | ||
658 | nvgpu_check_and_set_context_reservation, | ||
659 | .release_profiler_reservation = | ||
660 | nvgpu_release_profiler_reservation, | ||
661 | .perfbuffer_enable = gv11b_perfbuf_enable_locked, | ||
662 | .perfbuffer_disable = gv11b_perfbuf_disable_locked, | ||
663 | }, | ||
664 | .bus = { | ||
665 | .init_hw = gk20a_bus_init_hw, | ||
666 | .isr = gk20a_bus_isr, | ||
667 | .read_ptimer = gk20a_read_ptimer, | ||
668 | .get_timestamps_zipper = nvgpu_get_timestamps_zipper, | ||
669 | .bar1_bind = NULL, | ||
670 | }, | ||
671 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
672 | .css = { | ||
673 | .enable_snapshot = gv11b_css_hw_enable_snapshot, | ||
674 | .disable_snapshot = gv11b_css_hw_disable_snapshot, | ||
675 | .check_data_available = gv11b_css_hw_check_data_available, | ||
676 | .set_handled_snapshots = css_hw_set_handled_snapshots, | ||
677 | .allocate_perfmon_ids = css_gr_allocate_perfmon_ids, | ||
678 | .release_perfmon_ids = css_gr_release_perfmon_ids, | ||
679 | }, | ||
680 | #endif | ||
681 | .falcon = { | ||
682 | .falcon_hal_sw_init = gk20a_falcon_hal_sw_init, | ||
683 | }, | ||
684 | .priv_ring = { | ||
685 | .isr = gp10b_priv_ring_isr, | ||
686 | }, | ||
687 | .chip_init_gpu_characteristics = gv11b_init_gpu_characteristics, | ||
688 | .get_litter_value = gv11b_get_litter_value, | ||
689 | }; | ||
690 | |||
691 | int gv11b_init_hal(struct gk20a *g) | ||
692 | { | ||
693 | struct gpu_ops *gops = &g->ops; | ||
694 | u32 val; | ||
695 | bool priv_security; | ||
696 | |||
697 | gops->ltc = gv11b_ops.ltc; | ||
698 | gops->ce2 = gv11b_ops.ce2; | ||
699 | gops->gr = gv11b_ops.gr; | ||
700 | gops->fb = gv11b_ops.fb; | ||
701 | gops->clock_gating = gv11b_ops.clock_gating; | ||
702 | gops->fifo = gv11b_ops.fifo; | ||
703 | gops->gr_ctx = gv11b_ops.gr_ctx; | ||
704 | gops->mm = gv11b_ops.mm; | ||
705 | #ifdef CONFIG_GK20A_CTXSW_TRACE | ||
706 | gops->fecs_trace = gv11b_ops.fecs_trace; | ||
707 | #endif | ||
708 | gops->therm = gv11b_ops.therm; | ||
709 | gops->pmu = gv11b_ops.pmu; | ||
710 | gops->regops = gv11b_ops.regops; | ||
711 | gops->mc = gv11b_ops.mc; | ||
712 | gops->debug = gv11b_ops.debug; | ||
713 | gops->dbg_session_ops = gv11b_ops.dbg_session_ops; | ||
714 | gops->bus = gv11b_ops.bus; | ||
715 | #if defined(CONFIG_GK20A_CYCLE_STATS) | ||
716 | gops->css = gv11b_ops.css; | ||
717 | #endif | ||
718 | gops->falcon = gv11b_ops.falcon; | ||
719 | gops->priv_ring = gv11b_ops.priv_ring; | ||
720 | |||
721 | /* Lone functions */ | ||
722 | gops->chip_init_gpu_characteristics = | ||
723 | gv11b_ops.chip_init_gpu_characteristics; | ||
724 | gops->get_litter_value = gv11b_ops.get_litter_value; | ||
725 | |||
726 | val = gk20a_readl(g, fuse_opt_priv_sec_en_r()); | ||
727 | if (val) { | ||
728 | priv_security = true; | ||
729 | pr_err("priv security is enabled\n"); | ||
730 | } else { | ||
731 | priv_security = false; | ||
732 | pr_err("priv security is disabled\n"); | ||
733 | } | ||
734 | __nvgpu_set_enabled(g, NVGPU_GR_USE_DMA_FOR_FW_BOOTSTRAP, false); | ||
735 | __nvgpu_set_enabled(g, NVGPU_SEC_PRIVSECURITY, priv_security); | ||
736 | __nvgpu_set_enabled(g, NVGPU_SEC_SECUREGPCCS, priv_security); | ||
737 | |||
738 | /* priv security dependent ops */ | ||
739 | if (nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { | ||
740 | /* Add in ops from gm20b acr */ | ||
741 | gops->pmu.prepare_ucode = gp106_prepare_ucode_blob, | ||
742 | gops->pmu.pmu_setup_hw_and_bootstrap = gv11b_bootstrap_hs_flcn, | ||
743 | gops->pmu.get_wpr = gm20b_wpr_info, | ||
744 | gops->pmu.alloc_blob_space = gm20b_alloc_blob_space, | ||
745 | gops->pmu.pmu_populate_loader_cfg = | ||
746 | gp106_pmu_populate_loader_cfg, | ||
747 | gops->pmu.flcn_populate_bl_dmem_desc = | ||
748 | gp106_flcn_populate_bl_dmem_desc, | ||
749 | gops->pmu.falcon_wait_for_halt = pmu_wait_for_halt, | ||
750 | gops->pmu.falcon_clear_halt_interrupt_status = | ||
751 | clear_halt_interrupt_status, | ||
752 | gops->pmu.init_falcon_setup_hw = gv11b_init_pmu_setup_hw1, | ||
753 | |||
754 | gops->pmu.init_wpr_region = gm20b_pmu_init_acr; | ||
755 | gops->pmu.load_lsfalcon_ucode = gp10b_load_falcon_ucode; | ||
756 | gops->pmu.is_lazy_bootstrap = gv11b_is_lazy_bootstrap, | ||
757 | gops->pmu.is_priv_load = gv11b_is_priv_load, | ||
758 | |||
759 | gops->gr.load_ctxsw_ucode = gr_gm20b_load_ctxsw_ucode; | ||
760 | } else { | ||
761 | /* Inherit from gk20a */ | ||
762 | gops->pmu.prepare_ucode = nvgpu_pmu_prepare_ns_ucode_blob, | ||
763 | gops->pmu.pmu_setup_hw_and_bootstrap = gk20a_init_pmu_setup_hw1, | ||
764 | |||
765 | gops->pmu.load_lsfalcon_ucode = NULL; | ||
766 | gops->pmu.init_wpr_region = NULL; | ||
767 | gops->pmu.pmu_setup_hw_and_bootstrap = gp10b_init_pmu_setup_hw1; | ||
768 | |||
769 | gops->gr.load_ctxsw_ucode = gr_gk20a_load_ctxsw_ucode; | ||
770 | } | ||
771 | |||
772 | __nvgpu_set_enabled(g, NVGPU_PMU_FECS_BOOTSTRAP_DONE, false); | ||
773 | g->bootstrap_owner = LSF_BOOTSTRAP_OWNER_DEFAULT; | ||
774 | |||
775 | g->name = "gv11b"; | ||
776 | |||
777 | return 0; | ||
778 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.h b/drivers/gpu/nvgpu/gv11b/hal_gv11b.h new file mode 100644 index 00000000..668353dc --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.h | |||
@@ -0,0 +1,31 @@ | |||
1 | /* | ||
2 | * GV11B Tegra HAL interface | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef _NVGPU_HAL_GV11B_H | ||
26 | #define _NVGPU_HAL_GV11B_H | ||
27 | struct gk20a; | ||
28 | |||
29 | int gv11b_init_hal(struct gk20a *gops); | ||
30 | int gv11b_get_litter_value(struct gk20a *g, int value); | ||
31 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c new file mode 100644 index 00000000..a199e024 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.c | |||
@@ -0,0 +1,205 @@ | |||
1 | /* | ||
2 | * GV11B LTC | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include "gk20a/gk20a.h" | ||
26 | #include "gp10b/ltc_gp10b.h" | ||
27 | |||
28 | #include "ltc_gv11b.h" | ||
29 | |||
30 | #include <nvgpu/hw/gv11b/hw_ltc_gv11b.h> | ||
31 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
32 | #include <nvgpu/hw/gv11b/hw_top_gv11b.h> | ||
33 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
34 | #include <nvgpu/hw/gv11b/hw_pri_ringmaster_gv11b.h> | ||
35 | |||
36 | /* | ||
37 | * Sets the ZBC stencil for the passed index. | ||
38 | */ | ||
39 | void gv11b_ltc_set_zbc_stencil_entry(struct gk20a *g, | ||
40 | struct zbc_entry *stencil_val, | ||
41 | u32 index) | ||
42 | { | ||
43 | u32 real_index = index + GK20A_STARTOF_ZBC_TABLE; | ||
44 | |||
45 | gk20a_writel(g, ltc_ltcs_ltss_dstg_zbc_index_r(), | ||
46 | ltc_ltcs_ltss_dstg_zbc_index_address_f(real_index)); | ||
47 | |||
48 | gk20a_writel(g, ltc_ltcs_ltss_dstg_zbc_stencil_clear_value_r(), | ||
49 | stencil_val->depth); | ||
50 | |||
51 | gk20a_readl(g, ltc_ltcs_ltss_dstg_zbc_index_r()); | ||
52 | } | ||
53 | |||
54 | void gv11b_ltc_init_fs_state(struct gk20a *g) | ||
55 | { | ||
56 | u32 ltc_intr; | ||
57 | u32 reg; | ||
58 | |||
59 | gk20a_dbg_info("initialize gv11b l2"); | ||
60 | |||
61 | g->ops.mc.reset(g, mc_enable_pfb_enabled_f() | | ||
62 | mc_enable_l2_enabled_f()); | ||
63 | |||
64 | reg = gk20a_readl(g, mc_elpg_enable_r()); | ||
65 | reg |= mc_elpg_enable_l2_enabled_f(); | ||
66 | gk20a_writel(g, mc_elpg_enable_r(), reg); | ||
67 | |||
68 | g->max_ltc_count = gk20a_readl(g, top_num_ltcs_r()); | ||
69 | g->ltc_count = gk20a_readl(g, pri_ringmaster_enum_ltc_r()); | ||
70 | gk20a_dbg_info("%u ltcs out of %u", g->ltc_count, g->max_ltc_count); | ||
71 | |||
72 | gk20a_writel(g, ltc_ltcs_ltss_dstg_cfg0_r(), | ||
73 | gk20a_readl(g, ltc_ltc0_lts0_dstg_cfg0_r()) | | ||
74 | ltc_ltcs_ltss_dstg_cfg0_vdc_4to2_disable_m()); | ||
75 | |||
76 | /* Disable LTC interrupts */ | ||
77 | reg = gk20a_readl(g, ltc_ltcs_ltss_intr_r()); | ||
78 | reg &= ~ltc_ltcs_ltss_intr_en_evicted_cb_m(); | ||
79 | reg &= ~ltc_ltcs_ltss_intr_en_illegal_compstat_access_m(); | ||
80 | gk20a_writel(g, ltc_ltcs_ltss_intr_r(), reg); | ||
81 | |||
82 | /* Enable ECC interrupts */ | ||
83 | ltc_intr = gk20a_readl(g, ltc_ltcs_ltss_intr_r()); | ||
84 | ltc_intr |= ltc_ltcs_ltss_intr_en_ecc_sec_error_enabled_f() | | ||
85 | ltc_ltcs_ltss_intr_en_ecc_ded_error_enabled_f(); | ||
86 | gk20a_writel(g, ltc_ltcs_ltss_intr_r(), | ||
87 | ltc_intr); | ||
88 | } | ||
89 | |||
90 | void gv11b_ltc_isr(struct gk20a *g) | ||
91 | { | ||
92 | u32 mc_intr, ltc_intr3; | ||
93 | unsigned int ltc, slice; | ||
94 | u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE); | ||
95 | u32 lts_stride = nvgpu_get_litter_value(g, GPU_LIT_LTS_STRIDE); | ||
96 | u32 ecc_status, ecc_addr, corrected_cnt, uncorrected_cnt; | ||
97 | u32 corrected_delta, uncorrected_delta; | ||
98 | u32 corrected_overflow, uncorrected_overflow; | ||
99 | u32 ltc_corrected, ltc_uncorrected; | ||
100 | |||
101 | mc_intr = gk20a_readl(g, mc_intr_ltc_r()); | ||
102 | for (ltc = 0; ltc < g->ltc_count; ltc++) { | ||
103 | if ((mc_intr & 1 << ltc) == 0) | ||
104 | continue; | ||
105 | ltc_corrected = ltc_uncorrected = 0; | ||
106 | |||
107 | for (slice = 0; slice < g->gr.slices_per_ltc; slice++) { | ||
108 | u32 offset = ltc_stride * ltc + lts_stride * slice; | ||
109 | ltc_intr3 = gk20a_readl(g, ltc_ltc0_lts0_intr3_r() + | ||
110 | offset); | ||
111 | |||
112 | /* Detect and handle ECC PARITY errors */ | ||
113 | |||
114 | if (ltc_intr3 & | ||
115 | (ltc_ltcs_ltss_intr3_ecc_uncorrected_m() | | ||
116 | ltc_ltcs_ltss_intr3_ecc_corrected_m())) { | ||
117 | |||
118 | ecc_status = gk20a_readl(g, | ||
119 | ltc_ltc0_lts0_l2_cache_ecc_status_r() + | ||
120 | offset); | ||
121 | ecc_addr = gk20a_readl(g, | ||
122 | ltc_ltc0_lts0_l2_cache_ecc_address_r() + | ||
123 | offset); | ||
124 | corrected_cnt = gk20a_readl(g, | ||
125 | ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_r() + offset); | ||
126 | uncorrected_cnt = gk20a_readl(g, | ||
127 | ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_r() + offset); | ||
128 | |||
129 | corrected_delta = | ||
130 | ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_total_v(corrected_cnt); | ||
131 | uncorrected_delta = | ||
132 | ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_total_v(uncorrected_cnt); | ||
133 | corrected_overflow = ecc_status & | ||
134 | ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_total_counter_overflow_m(); | ||
135 | |||
136 | uncorrected_overflow = ecc_status & | ||
137 | ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
138 | |||
139 | /* clear the interrupt */ | ||
140 | if ((corrected_delta > 0) || corrected_overflow) { | ||
141 | gk20a_writel(g, ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_r() + offset, 0); | ||
142 | } | ||
143 | if ((uncorrected_delta > 0) || uncorrected_overflow) { | ||
144 | gk20a_writel(g, | ||
145 | ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_r() + offset, 0); | ||
146 | } | ||
147 | |||
148 | gk20a_writel(g, ltc_ltc0_lts0_l2_cache_ecc_status_r() + offset, | ||
149 | ltc_ltc0_lts0_l2_cache_ecc_status_reset_task_f()); | ||
150 | |||
151 | /* update counters per slice */ | ||
152 | if (corrected_overflow) | ||
153 | corrected_delta += (0x1UL << ltc_ltc0_lts0_l2_cache_ecc_corrected_err_count_total_s()); | ||
154 | if (uncorrected_overflow) | ||
155 | uncorrected_delta += (0x1UL << ltc_ltc0_lts0_l2_cache_ecc_uncorrected_err_count_total_s()); | ||
156 | |||
157 | ltc_corrected += corrected_delta; | ||
158 | ltc_uncorrected += uncorrected_delta; | ||
159 | nvgpu_log(g, gpu_dbg_intr, | ||
160 | "ltc:%d lts: %d cache ecc interrupt intr: 0x%x", ltc, slice, ltc_intr3); | ||
161 | |||
162 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_rstg_m()) | ||
163 | nvgpu_log(g, gpu_dbg_intr, "rstg ecc error corrected"); | ||
164 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_rstg_m()) | ||
165 | nvgpu_log(g, gpu_dbg_intr, "rstg ecc error uncorrected"); | ||
166 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_tstg_m()) | ||
167 | nvgpu_log(g, gpu_dbg_intr, "tstg ecc error corrected"); | ||
168 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_tstg_m()) | ||
169 | nvgpu_log(g, gpu_dbg_intr, "tstg ecc error uncorrected"); | ||
170 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_corrected_err_dstg_m()) | ||
171 | nvgpu_log(g, gpu_dbg_intr, "dstg ecc error corrected"); | ||
172 | if (ecc_status & ltc_ltc0_lts0_l2_cache_ecc_status_uncorrected_err_dstg_m()) | ||
173 | nvgpu_log(g, gpu_dbg_intr, "dstg ecc error uncorrected"); | ||
174 | |||
175 | if (corrected_overflow || uncorrected_overflow) | ||
176 | nvgpu_info(g, "ecc counter overflow!"); | ||
177 | |||
178 | nvgpu_log(g, gpu_dbg_intr, | ||
179 | "ecc error address: 0x%x", ecc_addr); | ||
180 | |||
181 | } | ||
182 | |||
183 | } | ||
184 | g->ecc.ltc.t19x.l2_cache_corrected_err_count.counters[ltc] += | ||
185 | ltc_corrected; | ||
186 | g->ecc.ltc.t19x.l2_cache_uncorrected_err_count.counters[ltc] += | ||
187 | ltc_uncorrected; | ||
188 | |||
189 | } | ||
190 | |||
191 | /* fallback to other interrupts */ | ||
192 | gp10b_ltc_isr(g); | ||
193 | } | ||
194 | |||
195 | u32 gv11b_ltc_cbc_fix_config(struct gk20a *g, int base) | ||
196 | { | ||
197 | u32 val = gk20a_readl(g, ltc_ltcs_ltss_cbc_num_active_ltcs_r()); | ||
198 | |||
199 | if (ltc_ltcs_ltss_cbc_num_active_ltcs__v(val) == 2) | ||
200 | return base * 2; | ||
201 | else if (ltc_ltcs_ltss_cbc_num_active_ltcs__v(val) != 1) { | ||
202 | nvgpu_err(g, "Invalid number of active ltcs: %08x", val); | ||
203 | } | ||
204 | return base; | ||
205 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h new file mode 100644 index 00000000..9b46e74c --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/ltc_gv11b.h | |||
@@ -0,0 +1,34 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef LTC_GV11B_H | ||
24 | #define LTC_GV11B_H | ||
25 | struct gk20a; | ||
26 | |||
27 | void gv11b_ltc_set_zbc_stencil_entry(struct gk20a *g, | ||
28 | struct zbc_entry *stencil_val, | ||
29 | u32 index); | ||
30 | void gv11b_ltc_init_fs_state(struct gk20a *g); | ||
31 | void gv11b_ltc_isr(struct gk20a *g); | ||
32 | u32 gv11b_ltc_cbc_fix_config(struct gk20a *g, int base); | ||
33 | |||
34 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/mc_gv11b.c b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c new file mode 100644 index 00000000..74c5c4d6 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/mc_gv11b.c | |||
@@ -0,0 +1,92 @@ | |||
1 | /* | ||
2 | * GV11B master | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | |||
29 | #include "gp10b/mc_gp10b.h" | ||
30 | |||
31 | #include "mc_gv11b.h" | ||
32 | #include "fb_gv11b.h" | ||
33 | |||
34 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
35 | |||
36 | void mc_gv11b_intr_enable(struct gk20a *g) | ||
37 | { | ||
38 | u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g); | ||
39 | |||
40 | gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), | ||
41 | 0xffffffff); | ||
42 | gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), | ||
43 | 0xffffffff); | ||
44 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL); | ||
45 | |||
46 | g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] = | ||
47 | mc_intr_pfifo_pending_f() | | ||
48 | mc_intr_hub_pending_f() | | ||
49 | mc_intr_priv_ring_pending_f() | | ||
50 | mc_intr_pbus_pending_f() | | ||
51 | mc_intr_ltc_pending_f() | | ||
52 | eng_intr_mask; | ||
53 | |||
54 | g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = | ||
55 | mc_intr_pfifo_pending_f() | ||
56 | | eng_intr_mask; | ||
57 | |||
58 | /* TODO: Enable PRI faults for HUB ECC err intr */ | ||
59 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types); | ||
60 | |||
61 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), | ||
62 | g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]); | ||
63 | |||
64 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), | ||
65 | g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); | ||
66 | |||
67 | } | ||
68 | |||
69 | bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0) | ||
70 | { | ||
71 | return ((mc_intr_0 & mc_intr_hub_pending_f()) ? true : false); | ||
72 | } | ||
73 | |||
74 | bool gv11b_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id) | ||
75 | { | ||
76 | u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); | ||
77 | u32 stall_intr, eng_intr_mask; | ||
78 | |||
79 | eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id); | ||
80 | if (mc_intr_0 & eng_intr_mask) | ||
81 | return true; | ||
82 | |||
83 | stall_intr = mc_intr_pfifo_pending_f() | | ||
84 | mc_intr_hub_pending_f() | | ||
85 | mc_intr_priv_ring_pending_f() | | ||
86 | mc_intr_pbus_pending_f() | | ||
87 | mc_intr_ltc_pending_f(); | ||
88 | if (mc_intr_0 & stall_intr) | ||
89 | return true; | ||
90 | |||
91 | return false; | ||
92 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/mc_gv11b.h b/drivers/gpu/nvgpu/gv11b/mc_gv11b.h new file mode 100644 index 00000000..eb9d0e4e --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/mc_gv11b.h | |||
@@ -0,0 +1,30 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef MC_GV11B_H | ||
24 | #define MC_GV11B_H | ||
25 | struct gk20a; | ||
26 | |||
27 | void mc_gv11b_intr_enable(struct gk20a *g); | ||
28 | bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0); | ||
29 | bool gv11b_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id); | ||
30 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.c b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c new file mode 100644 index 00000000..fdc506ac --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.c | |||
@@ -0,0 +1,330 @@ | |||
1 | /* | ||
2 | * GV11B MMU | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/pm_runtime.h> | ||
26 | |||
27 | #include <nvgpu/kmem.h> | ||
28 | #include <nvgpu/dma.h> | ||
29 | #include <nvgpu/log.h> | ||
30 | #include <nvgpu/mm.h> | ||
31 | |||
32 | #include "gk20a/gk20a.h" | ||
33 | #include "gk20a/mm_gk20a.h" | ||
34 | |||
35 | #include "gp10b/mm_gp10b.h" | ||
36 | #include "gp10b/mc_gp10b.h" | ||
37 | |||
38 | #include "mm_gv11b.h" | ||
39 | #include "fb_gv11b.h" | ||
40 | |||
41 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
42 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
43 | #include <nvgpu/hw/gv11b/hw_bus_gv11b.h> | ||
44 | |||
45 | #define NVGPU_L3_ALLOC_BIT BIT(36) | ||
46 | |||
47 | bool gv11b_mm_is_bar1_supported(struct gk20a *g) | ||
48 | { | ||
49 | return false; | ||
50 | } | ||
51 | |||
52 | void gv11b_init_inst_block(struct nvgpu_mem *inst_block, | ||
53 | struct vm_gk20a *vm, u32 big_page_size) | ||
54 | { | ||
55 | struct gk20a *g = gk20a_from_vm(vm); | ||
56 | |||
57 | gk20a_dbg_info("inst block phys = 0x%llx, kv = 0x%p", | ||
58 | nvgpu_inst_block_addr(g, inst_block), inst_block->cpu_va); | ||
59 | |||
60 | g->ops.mm.init_pdb(g, inst_block, vm); | ||
61 | |||
62 | if (big_page_size && g->ops.mm.set_big_page_size) | ||
63 | g->ops.mm.set_big_page_size(g, inst_block, big_page_size); | ||
64 | } | ||
65 | |||
66 | bool gv11b_mm_mmu_fault_pending(struct gk20a *g) | ||
67 | { | ||
68 | return gv11b_fb_mmu_fault_pending(g); | ||
69 | } | ||
70 | |||
71 | void gv11b_mm_fault_info_mem_destroy(struct gk20a *g) | ||
72 | { | ||
73 | nvgpu_log_fn(g, " "); | ||
74 | |||
75 | nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); | ||
76 | |||
77 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_OTHER | | ||
78 | HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY); | ||
79 | |||
80 | nvgpu_kfree(g, g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY]); | ||
81 | |||
82 | g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY] = NULL; | ||
83 | g->mm.fault_info[FAULT_TYPE_REPLAY] = NULL; | ||
84 | |||
85 | nvgpu_mutex_release(&g->mm.hub_isr_mutex); | ||
86 | nvgpu_mutex_destroy(&g->mm.hub_isr_mutex); | ||
87 | } | ||
88 | |||
89 | static int gv11b_mm_mmu_fault_info_buf_init(struct gk20a *g, | ||
90 | u32 *hub_intr_types) | ||
91 | { | ||
92 | struct mmu_fault_info *fault_info_mem; | ||
93 | |||
94 | fault_info_mem = nvgpu_kzalloc(g, sizeof(struct mmu_fault_info) * | ||
95 | FAULT_TYPE_NUM); | ||
96 | if (!fault_info_mem) { | ||
97 | nvgpu_log_info(g, "failed to alloc shadow fault info"); | ||
98 | return -ENOMEM; | ||
99 | } | ||
100 | /* shadow buffer for copying mmu fault info */ | ||
101 | g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY] = | ||
102 | &fault_info_mem[FAULT_TYPE_OTHER_AND_NONREPLAY]; | ||
103 | |||
104 | g->mm.fault_info[FAULT_TYPE_REPLAY] = | ||
105 | &fault_info_mem[FAULT_TYPE_REPLAY]; | ||
106 | |||
107 | *hub_intr_types |= HUB_INTR_TYPE_OTHER; | ||
108 | return 0; | ||
109 | } | ||
110 | |||
111 | static void gv11b_mm_mmu_hw_fault_buf_init(struct gk20a *g, | ||
112 | u32 *hub_intr_types) | ||
113 | { | ||
114 | struct vm_gk20a *vm = g->mm.bar2.vm; | ||
115 | int err = 0; | ||
116 | size_t fb_size; | ||
117 | |||
118 | /* Max entries take care of 1 entry used for full detection */ | ||
119 | fb_size = (g->ops.fifo.get_num_fifos(g) + 1) * | ||
120 | gmmu_fault_buf_size_v(); | ||
121 | |||
122 | err = nvgpu_dma_alloc_map_sys(vm, fb_size, | ||
123 | &g->mm.hw_fault_buf[FAULT_TYPE_OTHER_AND_NONREPLAY]); | ||
124 | if (err) { | ||
125 | nvgpu_err(g, | ||
126 | "Error in hw mmu fault buf [0] alloc in bar2 vm "); | ||
127 | /* Fault will be snapped in pri reg but not in buffer */ | ||
128 | return; | ||
129 | } | ||
130 | |||
131 | g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] = | ||
132 | HW_FAULT_BUF_STATUS_ALLOC_TRUE; | ||
133 | *hub_intr_types |= HUB_INTR_TYPE_NONREPLAY; | ||
134 | |||
135 | err = nvgpu_dma_alloc_map_sys(vm, fb_size, | ||
136 | &g->mm.hw_fault_buf[FAULT_TYPE_REPLAY]); | ||
137 | if (err) { | ||
138 | nvgpu_err(g, | ||
139 | "Error in hw mmu fault buf [1] alloc in bar2 vm "); | ||
140 | /* Fault will be snapped in pri reg but not in buffer */ | ||
141 | return; | ||
142 | } | ||
143 | g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] = | ||
144 | HW_FAULT_BUF_STATUS_ALLOC_TRUE; | ||
145 | *hub_intr_types |= HUB_INTR_TYPE_REPLAY; | ||
146 | } | ||
147 | |||
148 | static void gv11b_mm_mmu_hw_fault_buf_deinit(struct gk20a *g) | ||
149 | { | ||
150 | struct vm_gk20a *vm = g->mm.bar2.vm; | ||
151 | |||
152 | nvgpu_log_fn(g, " "); | ||
153 | |||
154 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_NONREPLAY | | ||
155 | HUB_INTR_TYPE_REPLAY); | ||
156 | |||
157 | g->mm.hub_intr_types &= (~(HUB_INTR_TYPE_NONREPLAY | | ||
158 | HUB_INTR_TYPE_REPLAY)); | ||
159 | |||
160 | if ((gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX))) { | ||
161 | gv11b_fb_fault_buf_set_state_hw(g, NONREPLAY_REG_INDEX, | ||
162 | FAULT_BUF_DISABLED); | ||
163 | } | ||
164 | |||
165 | if ((gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX))) { | ||
166 | gv11b_fb_fault_buf_set_state_hw(g, REPLAY_REG_INDEX, | ||
167 | FAULT_BUF_DISABLED); | ||
168 | } | ||
169 | |||
170 | if (g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] == | ||
171 | HW_FAULT_BUF_STATUS_ALLOC_TRUE) { | ||
172 | nvgpu_dma_unmap_free(vm, | ||
173 | &g->mm.hw_fault_buf[FAULT_TYPE_OTHER_AND_NONREPLAY]); | ||
174 | g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] = | ||
175 | HW_FAULT_BUF_STATUS_ALLOC_FALSE; | ||
176 | } | ||
177 | |||
178 | if (g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] == | ||
179 | HW_FAULT_BUF_STATUS_ALLOC_TRUE) { | ||
180 | nvgpu_dma_unmap_free(vm, | ||
181 | &g->mm.hw_fault_buf[FAULT_TYPE_REPLAY]); | ||
182 | g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] = | ||
183 | HW_FAULT_BUF_STATUS_ALLOC_FALSE; | ||
184 | } | ||
185 | } | ||
186 | |||
187 | void gv11b_mm_remove_bar2_vm(struct gk20a *g) | ||
188 | { | ||
189 | struct mm_gk20a *mm = &g->mm; | ||
190 | |||
191 | nvgpu_log_fn(g, " "); | ||
192 | |||
193 | gv11b_mm_mmu_hw_fault_buf_deinit(g); | ||
194 | |||
195 | nvgpu_free_inst_block(g, &mm->bar2.inst_block); | ||
196 | nvgpu_vm_put(mm->bar2.vm); | ||
197 | } | ||
198 | |||
199 | static void gv11b_mm_mmu_fault_setup_hw(struct gk20a *g) | ||
200 | { | ||
201 | if (g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] == | ||
202 | HW_FAULT_BUF_STATUS_ALLOC_TRUE) { | ||
203 | gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX); | ||
204 | } | ||
205 | if (g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] == | ||
206 | HW_FAULT_BUF_STATUS_ALLOC_TRUE) { | ||
207 | gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX); | ||
208 | } | ||
209 | } | ||
210 | |||
211 | static int gv11b_mm_mmu_fault_setup_sw(struct gk20a *g) | ||
212 | { | ||
213 | int err; | ||
214 | |||
215 | nvgpu_log_fn(g, " "); | ||
216 | |||
217 | nvgpu_mutex_init(&g->mm.hub_isr_mutex); | ||
218 | |||
219 | g->mm.hw_fault_buf_status[NONREPLAY_REG_INDEX] = | ||
220 | HW_FAULT_BUF_STATUS_ALLOC_FALSE; | ||
221 | g->mm.hw_fault_buf_status[REPLAY_REG_INDEX] = | ||
222 | HW_FAULT_BUF_STATUS_ALLOC_FALSE; | ||
223 | |||
224 | g->mm.hub_intr_types = HUB_INTR_TYPE_ECC_UNCORRECTED; | ||
225 | |||
226 | err = gv11b_mm_mmu_fault_info_buf_init(g, &g->mm.hub_intr_types); | ||
227 | |||
228 | if (!err) | ||
229 | gv11b_mm_mmu_hw_fault_buf_init(g, &g->mm.hub_intr_types); | ||
230 | |||
231 | return err; | ||
232 | } | ||
233 | |||
234 | int gv11b_init_mm_setup_hw(struct gk20a *g) | ||
235 | { | ||
236 | int err = 0; | ||
237 | |||
238 | nvgpu_log_fn(g, " "); | ||
239 | |||
240 | g->ops.fb.set_mmu_page_size(g); | ||
241 | g->ops.fb.init_hw(g); | ||
242 | |||
243 | err = g->ops.mm.init_bar2_mm_hw_setup(g); | ||
244 | if (err) | ||
245 | return err; | ||
246 | |||
247 | if (gk20a_mm_fb_flush(g) || gk20a_mm_fb_flush(g)) | ||
248 | return -EBUSY; | ||
249 | |||
250 | err = gv11b_mm_mmu_fault_setup_sw(g); | ||
251 | if (!err) | ||
252 | gv11b_mm_mmu_fault_setup_hw(g); | ||
253 | |||
254 | nvgpu_log_fn(g, "end"); | ||
255 | |||
256 | return err; | ||
257 | } | ||
258 | |||
259 | void gv11b_mm_l2_flush(struct gk20a *g, bool invalidate) | ||
260 | { | ||
261 | nvgpu_log(g, gpu_dbg_fn, "gv11b_mm_l2_flush"); | ||
262 | |||
263 | g->ops.mm.fb_flush(g); | ||
264 | gk20a_mm_l2_flush(g, invalidate); | ||
265 | g->ops.mm.fb_flush(g); | ||
266 | } | ||
267 | |||
268 | /* | ||
269 | * On Volta the GPU determines whether to do L3 allocation for a mapping by | ||
270 | * checking bit 36 of the phsyical address. So if a mapping should allocte lines | ||
271 | * in the L3 this bit must be set. | ||
272 | */ | ||
273 | u64 gv11b_gpu_phys_addr(struct gk20a *g, | ||
274 | struct nvgpu_gmmu_attrs *attrs, u64 phys) | ||
275 | { | ||
276 | if (attrs && attrs->t19x_attrs.l3_alloc) | ||
277 | return phys | NVGPU_L3_ALLOC_BIT; | ||
278 | |||
279 | return phys; | ||
280 | } | ||
281 | |||
282 | int gv11b_init_bar2_mm_hw_setup(struct gk20a *g) | ||
283 | { | ||
284 | struct mm_gk20a *mm = &g->mm; | ||
285 | struct nvgpu_mem *inst_block = &mm->bar2.inst_block; | ||
286 | u64 inst_pa = nvgpu_inst_block_addr(g, inst_block); | ||
287 | u32 reg_val; | ||
288 | struct nvgpu_timeout timeout; | ||
289 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
290 | |||
291 | nvgpu_log_fn(g, " "); | ||
292 | |||
293 | g->ops.fb.set_mmu_page_size(g); | ||
294 | |||
295 | inst_pa = (u32)(inst_pa >> bus_bar2_block_ptr_shift_v()); | ||
296 | nvgpu_log_info(g, "bar2 inst block ptr: 0x%08x", (u32)inst_pa); | ||
297 | |||
298 | gk20a_writel(g, bus_bar2_block_r(), | ||
299 | nvgpu_aperture_mask(g, inst_block, | ||
300 | bus_bar2_block_target_sys_mem_ncoh_f(), | ||
301 | bus_bar2_block_target_vid_mem_f()) | | ||
302 | bus_bar2_block_mode_virtual_f() | | ||
303 | bus_bar2_block_ptr_f(inst_pa)); | ||
304 | |||
305 | /* This is needed as BAR1 support is removed and there is no way | ||
306 | * to know if gpu successfully accessed memory. | ||
307 | * To avoid deadlocks and non-deterministic virtual address translation | ||
308 | * behavior, after writing BAR2_BLOCK to bind BAR2 to a virtual address | ||
309 | * space, SW must ensure that the bind has completed prior to issuing | ||
310 | * any further BAR2 requests by polling for both | ||
311 | * BUS_BIND_STATUS_BAR2_PENDING to return to EMPTY and | ||
312 | * BUS_BIND_STATUS_BAR2_OUTSTANDING to return to FALSE | ||
313 | */ | ||
314 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
315 | NVGPU_TIMER_CPU_TIMER); | ||
316 | nvgpu_log_info(g, "check bar2 bind status"); | ||
317 | do { | ||
318 | reg_val = gk20a_readl(g, bus_bind_status_r()); | ||
319 | |||
320 | if (!((reg_val & bus_bind_status_bar2_pending_busy_f()) || | ||
321 | (reg_val & bus_bind_status_bar2_outstanding_true_f()))) | ||
322 | return 0; | ||
323 | |||
324 | nvgpu_usleep_range(delay, delay * 2); | ||
325 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
326 | } while (!nvgpu_timeout_expired_msg(&timeout, "bar2 bind timedout")); | ||
327 | |||
328 | nvgpu_err(g, "bar2 bind failed. gpu unable to access memory"); | ||
329 | return -EBUSY; | ||
330 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/mm_gv11b.h b/drivers/gpu/nvgpu/gv11b/mm_gv11b.h new file mode 100644 index 00000000..d830b7cc --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/mm_gv11b.h | |||
@@ -0,0 +1,46 @@ | |||
1 | /* | ||
2 | * GV11B MM | ||
3 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
4 | * | ||
5 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
6 | * copy of this software and associated documentation files (the "Software"), | ||
7 | * to deal in the Software without restriction, including without limitation | ||
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
9 | * and/or sell copies of the Software, and to permit persons to whom the | ||
10 | * Software is furnished to do so, subject to the following conditions: | ||
11 | * | ||
12 | * The above copyright notice and this permission notice shall be included in | ||
13 | * all copies or substantial portions of the Software. | ||
14 | * | ||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
21 | * DEALINGS IN THE SOFTWARE. | ||
22 | */ | ||
23 | |||
24 | #ifndef MM_GV11B_H | ||
25 | #define MM_GV11B_H | ||
26 | |||
27 | #define HW_FAULT_BUF_STATUS_ALLOC_TRUE 1 | ||
28 | #define HW_FAULT_BUF_STATUS_ALLOC_FALSE 0 | ||
29 | |||
30 | struct gk20a; | ||
31 | struct nvgpu_mem; | ||
32 | struct vm_gk20a; | ||
33 | |||
34 | bool gv11b_mm_is_bar1_supported(struct gk20a *g); | ||
35 | void gv11b_init_inst_block(struct nvgpu_mem *inst_block, | ||
36 | struct vm_gk20a *vm, u32 big_page_size); | ||
37 | bool gv11b_mm_mmu_fault_pending(struct gk20a *g); | ||
38 | void gv11b_mm_remove_bar2_vm(struct gk20a *g); | ||
39 | int gv11b_init_mm_setup_hw(struct gk20a *g); | ||
40 | int gv11b_init_bar2_mm_hw_setup(struct gk20a *g); | ||
41 | void gv11b_mm_l2_flush(struct gk20a *g, bool invalidate); | ||
42 | u64 gv11b_gpu_phys_addr(struct gk20a *g, | ||
43 | struct nvgpu_gmmu_attrs *attrs, u64 phys); | ||
44 | void gv11b_mm_fault_info_mem_destroy(struct gk20a *g); | ||
45 | |||
46 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c b/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c new file mode 100644 index 00000000..95d82254 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/platform_gv11b_tegra.c | |||
@@ -0,0 +1,549 @@ | |||
1 | /* | ||
2 | * GV11B Tegra Platform Interface | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/of_platform.h> | ||
26 | #include <linux/debugfs.h> | ||
27 | #include <linux/dma-buf.h> | ||
28 | #include <linux/nvmap.h> | ||
29 | #include <linux/reset.h> | ||
30 | #include <linux/hashtable.h> | ||
31 | #include <linux/clk.h> | ||
32 | #include <nvgpu/nvhost.h> | ||
33 | #include <nvgpu/nvhost_t19x.h> | ||
34 | |||
35 | #include <uapi/linux/nvgpu.h> | ||
36 | |||
37 | #include <soc/tegra/tegra_bpmp.h> | ||
38 | #include <soc/tegra/tegra_powergate.h> | ||
39 | |||
40 | #include "gk20a/gk20a.h" | ||
41 | #include "common/linux/platform_gk20a.h" | ||
42 | #include "common/linux/clk.h" | ||
43 | |||
44 | #include "gp10b/platform_gp10b.h" | ||
45 | #include "common/linux/platform_gp10b_tegra.h" | ||
46 | |||
47 | #include "common/linux/os_linux.h" | ||
48 | #include "common/linux/platform_gk20a_tegra.h" | ||
49 | #include "gr_gv11b.h" | ||
50 | #include "nvgpu_gpuid_t19x.h" | ||
51 | |||
52 | static void gr_gv11b_remove_sysfs(struct device *dev); | ||
53 | |||
54 | static int gv11b_tegra_probe(struct device *dev) | ||
55 | { | ||
56 | struct gk20a_platform *platform = dev_get_drvdata(dev); | ||
57 | #ifdef CONFIG_TEGRA_GK20A_NVHOST | ||
58 | struct gk20a *g = platform->g; | ||
59 | int err = 0; | ||
60 | |||
61 | err = nvgpu_get_nvhost_dev(g); | ||
62 | if (err) { | ||
63 | dev_err(dev, "host1x device not available"); | ||
64 | return err; | ||
65 | } | ||
66 | |||
67 | err = nvgpu_nvhost_syncpt_unit_interface_get_aperture( | ||
68 | g->nvhost_dev, | ||
69 | &g->syncpt_unit_base, | ||
70 | &g->syncpt_unit_size); | ||
71 | if (err) { | ||
72 | dev_err(dev, "Failed to get syncpt interface"); | ||
73 | return -ENOSYS; | ||
74 | } | ||
75 | g->syncpt_size = nvgpu_nvhost_syncpt_unit_interface_get_byte_offset(1); | ||
76 | gk20a_dbg_info("syncpt_unit_base %llx syncpt_unit_size %zx size %x\n", | ||
77 | g->syncpt_unit_base, g->syncpt_unit_size, | ||
78 | g->syncpt_size); | ||
79 | #endif | ||
80 | |||
81 | platform->bypass_smmu = !device_is_iommuable(dev); | ||
82 | platform->disable_bigpage = platform->bypass_smmu; | ||
83 | |||
84 | platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close | ||
85 | = false; | ||
86 | platform->g->gr.t18x.ctx_vars.dump_ctxsw_stats_on_channel_close | ||
87 | = false; | ||
88 | |||
89 | platform->g->gr.t18x.ctx_vars.force_preemption_gfxp = false; | ||
90 | platform->g->gr.t18x.ctx_vars.force_preemption_cilp = false; | ||
91 | |||
92 | gp10b_tegra_get_clocks(dev); | ||
93 | nvgpu_linux_init_clk_support(platform->g); | ||
94 | |||
95 | return 0; | ||
96 | } | ||
97 | |||
98 | static int gv11b_tegra_remove(struct device *dev) | ||
99 | { | ||
100 | gp10b_tegra_remove(dev); | ||
101 | |||
102 | gr_gv11b_remove_sysfs(dev); | ||
103 | |||
104 | return 0; | ||
105 | } | ||
106 | |||
107 | static bool gv11b_tegra_is_railgated(struct device *dev) | ||
108 | { | ||
109 | bool ret = false; | ||
110 | #ifdef TEGRA194_POWER_DOMAIN_GPU | ||
111 | struct gk20a *g = get_gk20a(dev); | ||
112 | |||
113 | if (tegra_bpmp_running()) { | ||
114 | nvgpu_log(g, gpu_dbg_info, "bpmp running"); | ||
115 | ret = !tegra_powergate_is_powered(TEGRA194_POWER_DOMAIN_GPU); | ||
116 | |||
117 | nvgpu_log(g, gpu_dbg_info, "railgated? %s", ret ? "yes" : "no"); | ||
118 | } else { | ||
119 | nvgpu_log(g, gpu_dbg_info, "bpmp not running"); | ||
120 | } | ||
121 | #endif | ||
122 | return ret; | ||
123 | } | ||
124 | |||
125 | static int gv11b_tegra_railgate(struct device *dev) | ||
126 | { | ||
127 | #ifdef TEGRA194_POWER_DOMAIN_GPU | ||
128 | struct gk20a_platform *platform = gk20a_get_platform(dev); | ||
129 | struct gk20a *g = get_gk20a(dev); | ||
130 | int i; | ||
131 | |||
132 | if (tegra_bpmp_running()) { | ||
133 | nvgpu_log(g, gpu_dbg_info, "bpmp running"); | ||
134 | if (!tegra_powergate_is_powered(TEGRA194_POWER_DOMAIN_GPU)) { | ||
135 | nvgpu_log(g, gpu_dbg_info, "powergate is not powered"); | ||
136 | return 0; | ||
137 | } | ||
138 | nvgpu_log(g, gpu_dbg_info, "clk_disable_unprepare"); | ||
139 | for (i = 0; i < platform->num_clks; i++) { | ||
140 | if (platform->clk[i]) | ||
141 | clk_disable_unprepare(platform->clk[i]); | ||
142 | } | ||
143 | nvgpu_log(g, gpu_dbg_info, "powergate_partition"); | ||
144 | tegra_powergate_partition(TEGRA194_POWER_DOMAIN_GPU); | ||
145 | } else { | ||
146 | nvgpu_log(g, gpu_dbg_info, "bpmp not running"); | ||
147 | } | ||
148 | #endif | ||
149 | return 0; | ||
150 | } | ||
151 | |||
152 | static int gv11b_tegra_unrailgate(struct device *dev) | ||
153 | { | ||
154 | int ret = 0; | ||
155 | #ifdef TEGRA194_POWER_DOMAIN_GPU | ||
156 | struct gk20a_platform *platform = gk20a_get_platform(dev); | ||
157 | struct gk20a *g = get_gk20a(dev); | ||
158 | int i; | ||
159 | |||
160 | if (tegra_bpmp_running()) { | ||
161 | nvgpu_log(g, gpu_dbg_info, "bpmp running"); | ||
162 | ret = tegra_unpowergate_partition(TEGRA194_POWER_DOMAIN_GPU); | ||
163 | if (ret) { | ||
164 | nvgpu_log(g, gpu_dbg_info, | ||
165 | "unpowergate partition failed"); | ||
166 | return ret; | ||
167 | } | ||
168 | nvgpu_log(g, gpu_dbg_info, "clk_prepare_enable"); | ||
169 | for (i = 0; i < platform->num_clks; i++) { | ||
170 | if (platform->clk[i]) | ||
171 | clk_prepare_enable(platform->clk[i]); | ||
172 | } | ||
173 | } else { | ||
174 | nvgpu_log(g, gpu_dbg_info, "bpmp not running"); | ||
175 | } | ||
176 | #endif | ||
177 | return ret; | ||
178 | } | ||
179 | |||
180 | static int gv11b_tegra_suspend(struct device *dev) | ||
181 | { | ||
182 | return 0; | ||
183 | } | ||
184 | |||
185 | struct gk20a_platform t19x_gpu_tegra_platform = { | ||
186 | .has_syncpoints = true, | ||
187 | |||
188 | /* power management configuration */ | ||
189 | |||
190 | /* ptimer src frequency in hz*/ | ||
191 | .ptimer_src_freq = 31250000, | ||
192 | |||
193 | .probe = gv11b_tegra_probe, | ||
194 | .remove = gv11b_tegra_remove, | ||
195 | |||
196 | .enable_slcg = false, | ||
197 | .enable_blcg = false, | ||
198 | .enable_elcg = false, | ||
199 | .can_slcg = false, | ||
200 | .can_blcg = false, | ||
201 | .can_elcg = false, | ||
202 | |||
203 | /* power management callbacks */ | ||
204 | .suspend = gv11b_tegra_suspend, | ||
205 | .railgate = gv11b_tegra_railgate, | ||
206 | .unrailgate = gv11b_tegra_unrailgate, | ||
207 | .is_railgated = gv11b_tegra_is_railgated, | ||
208 | |||
209 | .busy = gk20a_tegra_busy, | ||
210 | .idle = gk20a_tegra_idle, | ||
211 | |||
212 | .dump_platform_dependencies = gk20a_tegra_debug_dump, | ||
213 | |||
214 | .soc_name = "tegra19x", | ||
215 | |||
216 | .honors_aperture = true, | ||
217 | .unified_memory = true, | ||
218 | |||
219 | .reset_assert = gp10b_tegra_reset_assert, | ||
220 | .reset_deassert = gp10b_tegra_reset_deassert, | ||
221 | }; | ||
222 | |||
223 | static struct device_attribute *dev_attr_sm_l1_tag_ecc_corrected_err_count_array; | ||
224 | static struct device_attribute *dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array; | ||
225 | static struct device_attribute *dev_attr_sm_cbu_ecc_corrected_err_count_array; | ||
226 | static struct device_attribute *dev_attr_sm_cbu_ecc_uncorrected_err_count_array; | ||
227 | static struct device_attribute *dev_attr_sm_l1_data_ecc_corrected_err_count_array; | ||
228 | static struct device_attribute *dev_attr_sm_l1_data_ecc_uncorrected_err_count_array; | ||
229 | static struct device_attribute *dev_attr_sm_icache_ecc_corrected_err_count_array; | ||
230 | static struct device_attribute *dev_attr_sm_icache_ecc_uncorrected_err_count_array; | ||
231 | static struct device_attribute *dev_attr_gcc_l15_ecc_corrected_err_count_array; | ||
232 | static struct device_attribute *dev_attr_gcc_l15_ecc_uncorrected_err_count_array; | ||
233 | static struct device_attribute *dev_attr_mmu_l1tlb_ecc_corrected_err_count_array; | ||
234 | static struct device_attribute *dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array; | ||
235 | |||
236 | static struct device_attribute *dev_attr_fecs_ecc_corrected_err_count_array; | ||
237 | static struct device_attribute *dev_attr_fecs_ecc_uncorrected_err_count_array; | ||
238 | static struct device_attribute *dev_attr_gpccs_ecc_corrected_err_count_array; | ||
239 | static struct device_attribute *dev_attr_gpccs_ecc_uncorrected_err_count_array; | ||
240 | |||
241 | static struct device_attribute *dev_attr_l2_cache_ecc_corrected_err_count_array; | ||
242 | static struct device_attribute *dev_attr_l2_cache_ecc_uncorrected_err_count_array; | ||
243 | |||
244 | static struct device_attribute *dev_attr_mmu_l2tlb_ecc_corrected_err_count_array; | ||
245 | static struct device_attribute *dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array; | ||
246 | static struct device_attribute *dev_attr_mmu_hubtlb_ecc_corrected_err_count_array; | ||
247 | static struct device_attribute *dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array; | ||
248 | static struct device_attribute *dev_attr_mmu_fillunit_ecc_corrected_err_count_array; | ||
249 | static struct device_attribute *dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array; | ||
250 | |||
251 | void gr_gv11b_create_sysfs(struct gk20a *g) | ||
252 | { | ||
253 | struct device *dev = dev_from_gk20a(g); | ||
254 | int error = 0; | ||
255 | /* This stat creation function is called on GR init. GR can get | ||
256 | initialized multiple times but we only need to create the ECC | ||
257 | stats once. Therefore, add the following check to avoid | ||
258 | creating duplicate stat sysfs nodes. */ | ||
259 | if (g->ecc.gr.t19x.sm_l1_tag_corrected_err_count.counters != NULL) | ||
260 | return; | ||
261 | |||
262 | gr_gp10b_create_sysfs(g); | ||
263 | |||
264 | error |= gr_gp10b_ecc_stat_create(dev, | ||
265 | 0, | ||
266 | "sm_l1_tag_ecc_corrected_err_count", | ||
267 | &g->ecc.gr.t19x.sm_l1_tag_corrected_err_count, | ||
268 | &dev_attr_sm_l1_tag_ecc_corrected_err_count_array); | ||
269 | |||
270 | error |= gr_gp10b_ecc_stat_create(dev, | ||
271 | 0, | ||
272 | "sm_l1_tag_ecc_uncorrected_err_count", | ||
273 | &g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count, | ||
274 | &dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array); | ||
275 | |||
276 | error |= gr_gp10b_ecc_stat_create(dev, | ||
277 | 0, | ||
278 | "sm_cbu_ecc_corrected_err_count", | ||
279 | &g->ecc.gr.t19x.sm_cbu_corrected_err_count, | ||
280 | &dev_attr_sm_cbu_ecc_corrected_err_count_array); | ||
281 | |||
282 | error |= gr_gp10b_ecc_stat_create(dev, | ||
283 | 0, | ||
284 | "sm_cbu_ecc_uncorrected_err_count", | ||
285 | &g->ecc.gr.t19x.sm_cbu_uncorrected_err_count, | ||
286 | &dev_attr_sm_cbu_ecc_uncorrected_err_count_array); | ||
287 | |||
288 | error |= gr_gp10b_ecc_stat_create(dev, | ||
289 | 0, | ||
290 | "sm_l1_data_ecc_corrected_err_count", | ||
291 | &g->ecc.gr.t19x.sm_l1_data_corrected_err_count, | ||
292 | &dev_attr_sm_l1_data_ecc_corrected_err_count_array); | ||
293 | |||
294 | error |= gr_gp10b_ecc_stat_create(dev, | ||
295 | 0, | ||
296 | "sm_l1_data_ecc_uncorrected_err_count", | ||
297 | &g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count, | ||
298 | &dev_attr_sm_l1_data_ecc_uncorrected_err_count_array); | ||
299 | |||
300 | error |= gr_gp10b_ecc_stat_create(dev, | ||
301 | 0, | ||
302 | "sm_icache_ecc_corrected_err_count", | ||
303 | &g->ecc.gr.t19x.sm_icache_corrected_err_count, | ||
304 | &dev_attr_sm_icache_ecc_corrected_err_count_array); | ||
305 | |||
306 | error |= gr_gp10b_ecc_stat_create(dev, | ||
307 | 0, | ||
308 | "sm_icache_ecc_uncorrected_err_count", | ||
309 | &g->ecc.gr.t19x.sm_icache_uncorrected_err_count, | ||
310 | &dev_attr_sm_icache_ecc_uncorrected_err_count_array); | ||
311 | |||
312 | error |= gr_gp10b_ecc_stat_create(dev, | ||
313 | 0, | ||
314 | "gcc_l15_ecc_corrected_err_count", | ||
315 | &g->ecc.gr.t19x.gcc_l15_corrected_err_count, | ||
316 | &dev_attr_gcc_l15_ecc_corrected_err_count_array); | ||
317 | |||
318 | error |= gr_gp10b_ecc_stat_create(dev, | ||
319 | 0, | ||
320 | "gcc_l15_ecc_uncorrected_err_count", | ||
321 | &g->ecc.gr.t19x.gcc_l15_uncorrected_err_count, | ||
322 | &dev_attr_gcc_l15_ecc_uncorrected_err_count_array); | ||
323 | |||
324 | error |= gp10b_ecc_stat_create(dev, | ||
325 | g->ltc_count, | ||
326 | "ltc", | ||
327 | "l2_cache_uncorrected_err_count", | ||
328 | &g->ecc.ltc.t19x.l2_cache_uncorrected_err_count, | ||
329 | &dev_attr_l2_cache_ecc_uncorrected_err_count_array); | ||
330 | |||
331 | error |= gp10b_ecc_stat_create(dev, | ||
332 | g->ltc_count, | ||
333 | "ltc", | ||
334 | "l2_cache_corrected_err_count", | ||
335 | &g->ecc.ltc.t19x.l2_cache_corrected_err_count, | ||
336 | &dev_attr_l2_cache_ecc_corrected_err_count_array); | ||
337 | |||
338 | error |= gp10b_ecc_stat_create(dev, | ||
339 | 1, | ||
340 | "gpc", | ||
341 | "fecs_ecc_uncorrected_err_count", | ||
342 | &g->ecc.gr.t19x.fecs_uncorrected_err_count, | ||
343 | &dev_attr_fecs_ecc_uncorrected_err_count_array); | ||
344 | |||
345 | error |= gp10b_ecc_stat_create(dev, | ||
346 | 1, | ||
347 | "gpc", | ||
348 | "fecs_ecc_corrected_err_count", | ||
349 | &g->ecc.gr.t19x.fecs_corrected_err_count, | ||
350 | &dev_attr_fecs_ecc_corrected_err_count_array); | ||
351 | |||
352 | error |= gp10b_ecc_stat_create(dev, | ||
353 | g->gr.gpc_count, | ||
354 | "gpc", | ||
355 | "gpccs_ecc_uncorrected_err_count", | ||
356 | &g->ecc.gr.t19x.gpccs_uncorrected_err_count, | ||
357 | &dev_attr_gpccs_ecc_uncorrected_err_count_array); | ||
358 | |||
359 | error |= gp10b_ecc_stat_create(dev, | ||
360 | g->gr.gpc_count, | ||
361 | "gpc", | ||
362 | "gpccs_ecc_corrected_err_count", | ||
363 | &g->ecc.gr.t19x.gpccs_corrected_err_count, | ||
364 | &dev_attr_gpccs_ecc_corrected_err_count_array); | ||
365 | |||
366 | error |= gp10b_ecc_stat_create(dev, | ||
367 | g->gr.gpc_count, | ||
368 | "gpc", | ||
369 | "mmu_l1tlb_ecc_uncorrected_err_count", | ||
370 | &g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count, | ||
371 | &dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array); | ||
372 | |||
373 | error |= gp10b_ecc_stat_create(dev, | ||
374 | g->gr.gpc_count, | ||
375 | "gpc", | ||
376 | "mmu_l1tlb_ecc_corrected_err_count", | ||
377 | &g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count, | ||
378 | &dev_attr_mmu_l1tlb_ecc_corrected_err_count_array); | ||
379 | |||
380 | error |= gp10b_ecc_stat_create(dev, | ||
381 | 1, | ||
382 | "eng", | ||
383 | "mmu_l2tlb_ecc_uncorrected_err_count", | ||
384 | &g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count, | ||
385 | &dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array); | ||
386 | |||
387 | error |= gp10b_ecc_stat_create(dev, | ||
388 | 1, | ||
389 | "eng", | ||
390 | "mmu_l2tlb_ecc_corrected_err_count", | ||
391 | &g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count, | ||
392 | &dev_attr_mmu_l2tlb_ecc_corrected_err_count_array); | ||
393 | |||
394 | error |= gp10b_ecc_stat_create(dev, | ||
395 | 1, | ||
396 | "eng", | ||
397 | "mmu_hubtlb_ecc_uncorrected_err_count", | ||
398 | &g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count, | ||
399 | &dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array); | ||
400 | |||
401 | error |= gp10b_ecc_stat_create(dev, | ||
402 | 1, | ||
403 | "eng", | ||
404 | "mmu_hubtlb_ecc_corrected_err_count", | ||
405 | &g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count, | ||
406 | &dev_attr_mmu_hubtlb_ecc_corrected_err_count_array); | ||
407 | |||
408 | error |= gp10b_ecc_stat_create(dev, | ||
409 | 1, | ||
410 | "eng", | ||
411 | "mmu_fillunit_ecc_uncorrected_err_count", | ||
412 | &g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count, | ||
413 | &dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array); | ||
414 | |||
415 | error |= gp10b_ecc_stat_create(dev, | ||
416 | 1, | ||
417 | "eng", | ||
418 | "mmu_fillunit_ecc_corrected_err_count", | ||
419 | &g->ecc.eng.t19x.mmu_fillunit_corrected_err_count, | ||
420 | &dev_attr_mmu_fillunit_ecc_corrected_err_count_array); | ||
421 | |||
422 | if (error) | ||
423 | dev_err(dev, "Failed to create gv11b sysfs attributes!\n"); | ||
424 | } | ||
425 | |||
426 | static void gr_gv11b_remove_sysfs(struct device *dev) | ||
427 | { | ||
428 | struct gk20a *g = get_gk20a(dev); | ||
429 | |||
430 | gr_gp10b_ecc_stat_remove(dev, | ||
431 | 0, | ||
432 | &g->ecc.gr.t19x.sm_l1_tag_corrected_err_count, | ||
433 | dev_attr_sm_l1_tag_ecc_corrected_err_count_array); | ||
434 | |||
435 | gr_gp10b_ecc_stat_remove(dev, | ||
436 | 0, | ||
437 | &g->ecc.gr.t19x.sm_l1_tag_uncorrected_err_count, | ||
438 | dev_attr_sm_l1_tag_ecc_uncorrected_err_count_array); | ||
439 | |||
440 | gr_gp10b_ecc_stat_remove(dev, | ||
441 | 0, | ||
442 | &g->ecc.gr.t19x.sm_cbu_corrected_err_count, | ||
443 | dev_attr_sm_cbu_ecc_corrected_err_count_array); | ||
444 | |||
445 | gr_gp10b_ecc_stat_remove(dev, | ||
446 | 0, | ||
447 | &g->ecc.gr.t19x.sm_cbu_uncorrected_err_count, | ||
448 | dev_attr_sm_cbu_ecc_uncorrected_err_count_array); | ||
449 | |||
450 | gr_gp10b_ecc_stat_remove(dev, | ||
451 | 0, | ||
452 | &g->ecc.gr.t19x.sm_l1_data_corrected_err_count, | ||
453 | dev_attr_sm_l1_data_ecc_corrected_err_count_array); | ||
454 | |||
455 | gr_gp10b_ecc_stat_remove(dev, | ||
456 | 0, | ||
457 | &g->ecc.gr.t19x.sm_l1_data_uncorrected_err_count, | ||
458 | dev_attr_sm_l1_data_ecc_uncorrected_err_count_array); | ||
459 | |||
460 | gr_gp10b_ecc_stat_remove(dev, | ||
461 | 0, | ||
462 | &g->ecc.gr.t19x.sm_icache_corrected_err_count, | ||
463 | dev_attr_sm_icache_ecc_corrected_err_count_array); | ||
464 | |||
465 | gr_gp10b_ecc_stat_remove(dev, | ||
466 | 0, | ||
467 | &g->ecc.gr.t19x.sm_icache_uncorrected_err_count, | ||
468 | dev_attr_sm_icache_ecc_uncorrected_err_count_array); | ||
469 | |||
470 | gr_gp10b_ecc_stat_remove(dev, | ||
471 | 0, | ||
472 | &g->ecc.gr.t19x.gcc_l15_corrected_err_count, | ||
473 | dev_attr_gcc_l15_ecc_corrected_err_count_array); | ||
474 | |||
475 | gr_gp10b_ecc_stat_remove(dev, | ||
476 | 0, | ||
477 | &g->ecc.gr.t19x.gcc_l15_uncorrected_err_count, | ||
478 | dev_attr_gcc_l15_ecc_uncorrected_err_count_array); | ||
479 | |||
480 | gp10b_ecc_stat_remove(dev, | ||
481 | g->ltc_count, | ||
482 | &g->ecc.ltc.t19x.l2_cache_uncorrected_err_count, | ||
483 | dev_attr_l2_cache_ecc_uncorrected_err_count_array); | ||
484 | |||
485 | gp10b_ecc_stat_remove(dev, | ||
486 | g->ltc_count, | ||
487 | &g->ecc.ltc.t19x.l2_cache_corrected_err_count, | ||
488 | dev_attr_l2_cache_ecc_corrected_err_count_array); | ||
489 | |||
490 | gp10b_ecc_stat_remove(dev, | ||
491 | 1, | ||
492 | &g->ecc.gr.t19x.fecs_uncorrected_err_count, | ||
493 | dev_attr_fecs_ecc_uncorrected_err_count_array); | ||
494 | |||
495 | gp10b_ecc_stat_remove(dev, | ||
496 | 1, | ||
497 | &g->ecc.gr.t19x.fecs_corrected_err_count, | ||
498 | dev_attr_fecs_ecc_corrected_err_count_array); | ||
499 | |||
500 | gp10b_ecc_stat_remove(dev, | ||
501 | g->gr.gpc_count, | ||
502 | &g->ecc.gr.t19x.gpccs_uncorrected_err_count, | ||
503 | dev_attr_gpccs_ecc_uncorrected_err_count_array); | ||
504 | |||
505 | gp10b_ecc_stat_remove(dev, | ||
506 | g->gr.gpc_count, | ||
507 | &g->ecc.gr.t19x.gpccs_corrected_err_count, | ||
508 | dev_attr_gpccs_ecc_corrected_err_count_array); | ||
509 | |||
510 | gp10b_ecc_stat_remove(dev, | ||
511 | g->gr.gpc_count, | ||
512 | &g->ecc.gr.t19x.mmu_l1tlb_uncorrected_err_count, | ||
513 | dev_attr_mmu_l1tlb_ecc_uncorrected_err_count_array); | ||
514 | |||
515 | gp10b_ecc_stat_remove(dev, | ||
516 | g->gr.gpc_count, | ||
517 | &g->ecc.gr.t19x.mmu_l1tlb_corrected_err_count, | ||
518 | dev_attr_mmu_l1tlb_ecc_corrected_err_count_array); | ||
519 | |||
520 | gp10b_ecc_stat_remove(dev, | ||
521 | 1, | ||
522 | &g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count, | ||
523 | dev_attr_mmu_l2tlb_ecc_uncorrected_err_count_array); | ||
524 | |||
525 | gp10b_ecc_stat_remove(dev, | ||
526 | 1, | ||
527 | &g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count, | ||
528 | dev_attr_mmu_l2tlb_ecc_corrected_err_count_array); | ||
529 | |||
530 | gp10b_ecc_stat_remove(dev, | ||
531 | 1, | ||
532 | &g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count, | ||
533 | dev_attr_mmu_hubtlb_ecc_uncorrected_err_count_array); | ||
534 | |||
535 | gp10b_ecc_stat_remove(dev, | ||
536 | 1, | ||
537 | &g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count, | ||
538 | dev_attr_mmu_hubtlb_ecc_corrected_err_count_array); | ||
539 | |||
540 | gp10b_ecc_stat_remove(dev, | ||
541 | 1, | ||
542 | &g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count, | ||
543 | dev_attr_mmu_fillunit_ecc_uncorrected_err_count_array); | ||
544 | |||
545 | gp10b_ecc_stat_remove(dev, | ||
546 | 1, | ||
547 | &g->ecc.eng.t19x.mmu_fillunit_corrected_err_count, | ||
548 | dev_attr_mmu_fillunit_ecc_corrected_err_count_array); | ||
549 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c new file mode 100644 index 00000000..2c7b6457 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.c | |||
@@ -0,0 +1,283 @@ | |||
1 | /* | ||
2 | * GV11B PMU | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/delay.h> /* for udelay */ | ||
26 | #include <linux/clk.h> | ||
27 | |||
28 | #include <soc/tegra/fuse.h> | ||
29 | |||
30 | #include <nvgpu/pmu.h> | ||
31 | #include <nvgpu/falcon.h> | ||
32 | #include <nvgpu/enabled.h> | ||
33 | #include <nvgpu/mm.h> | ||
34 | |||
35 | #include "gk20a/gk20a.h" | ||
36 | |||
37 | #include "gp10b/pmu_gp10b.h" | ||
38 | #include "gp106/pmu_gp106.h" | ||
39 | |||
40 | #include "pmu_gv11b.h" | ||
41 | #include "acr_gv11b.h" | ||
42 | |||
43 | #include <nvgpu/hw/gv11b/hw_pwr_gv11b.h> | ||
44 | |||
45 | #define gv11b_dbg_pmu(fmt, arg...) \ | ||
46 | gk20a_dbg(gpu_dbg_pmu, fmt, ##arg) | ||
47 | |||
48 | #define ALIGN_4KB 12 | ||
49 | |||
50 | bool gv11b_is_pmu_supported(struct gk20a *g) | ||
51 | { | ||
52 | return true; | ||
53 | } | ||
54 | |||
55 | bool gv11b_is_lazy_bootstrap(u32 falcon_id) | ||
56 | { | ||
57 | bool enable_status = false; | ||
58 | |||
59 | switch (falcon_id) { | ||
60 | case LSF_FALCON_ID_FECS: | ||
61 | enable_status = true; | ||
62 | break; | ||
63 | case LSF_FALCON_ID_GPCCS: | ||
64 | enable_status = true; | ||
65 | break; | ||
66 | default: | ||
67 | break; | ||
68 | } | ||
69 | |||
70 | return enable_status; | ||
71 | } | ||
72 | |||
73 | bool gv11b_is_priv_load(u32 falcon_id) | ||
74 | { | ||
75 | bool enable_status = false; | ||
76 | |||
77 | switch (falcon_id) { | ||
78 | case LSF_FALCON_ID_FECS: | ||
79 | enable_status = true; | ||
80 | break; | ||
81 | case LSF_FALCON_ID_GPCCS: | ||
82 | enable_status = true; | ||
83 | break; | ||
84 | default: | ||
85 | break; | ||
86 | } | ||
87 | |||
88 | return enable_status; | ||
89 | } | ||
90 | |||
91 | int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu) | ||
92 | { | ||
93 | struct gk20a *g = gk20a_from_pmu(pmu); | ||
94 | struct mm_gk20a *mm = &g->mm; | ||
95 | struct pmu_ucode_desc *desc = pmu->desc; | ||
96 | u64 addr_code_lo, addr_data_lo, addr_load_lo; | ||
97 | u64 addr_code_hi, addr_data_hi, addr_load_hi; | ||
98 | u32 i, blocks, addr_args; | ||
99 | |||
100 | gk20a_dbg_fn(""); | ||
101 | |||
102 | gk20a_writel(g, pwr_falcon_itfen_r(), | ||
103 | gk20a_readl(g, pwr_falcon_itfen_r()) | | ||
104 | pwr_falcon_itfen_ctxen_enable_f()); | ||
105 | |||
106 | gk20a_writel(g, pwr_pmu_new_instblk_r(), | ||
107 | pwr_pmu_new_instblk_ptr_f( | ||
108 | nvgpu_inst_block_addr(g, &mm->pmu.inst_block) >> ALIGN_4KB) | ||
109 | | pwr_pmu_new_instblk_valid_f(1) | ||
110 | | pwr_pmu_new_instblk_target_sys_ncoh_f()); | ||
111 | |||
112 | /* TBD: load all other surfaces */ | ||
113 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_size( | ||
114 | pmu, GK20A_PMU_TRACE_BUFSIZE); | ||
115 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_base(pmu); | ||
116 | g->ops.pmu_ver.set_pmu_cmdline_args_trace_dma_idx( | ||
117 | pmu, GK20A_PMU_DMAIDX_VIRT); | ||
118 | |||
119 | g->ops.pmu_ver.set_pmu_cmdline_args_cpu_freq(pmu, | ||
120 | g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_PWRCLK)); | ||
121 | |||
122 | addr_args = (pwr_falcon_hwcfg_dmem_size_v( | ||
123 | gk20a_readl(g, pwr_falcon_hwcfg_r())) | ||
124 | << GK20A_PMU_DMEM_BLKSIZE2) - | ||
125 | g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu); | ||
126 | |||
127 | nvgpu_flcn_copy_to_dmem(pmu->flcn, addr_args, | ||
128 | (u8 *)(g->ops.pmu_ver.get_pmu_cmdline_args_ptr(pmu)), | ||
129 | g->ops.pmu_ver.get_pmu_cmdline_args_size(pmu), 0); | ||
130 | |||
131 | gk20a_writel(g, pwr_falcon_dmemc_r(0), | ||
132 | pwr_falcon_dmemc_offs_f(0) | | ||
133 | pwr_falcon_dmemc_blk_f(0) | | ||
134 | pwr_falcon_dmemc_aincw_f(1)); | ||
135 | |||
136 | addr_code_lo = u64_lo32((pmu->ucode.gpu_va + | ||
137 | desc->app_start_offset + | ||
138 | desc->app_resident_code_offset) >> 8); | ||
139 | |||
140 | addr_code_hi = u64_hi32((pmu->ucode.gpu_va + | ||
141 | desc->app_start_offset + | ||
142 | desc->app_resident_code_offset) >> 8); | ||
143 | addr_data_lo = u64_lo32((pmu->ucode.gpu_va + | ||
144 | desc->app_start_offset + | ||
145 | desc->app_resident_data_offset) >> 8); | ||
146 | addr_data_hi = u64_hi32((pmu->ucode.gpu_va + | ||
147 | desc->app_start_offset + | ||
148 | desc->app_resident_data_offset) >> 8); | ||
149 | addr_load_lo = u64_lo32((pmu->ucode.gpu_va + | ||
150 | desc->bootloader_start_offset) >> 8); | ||
151 | addr_load_hi = u64_hi32((pmu->ucode.gpu_va + | ||
152 | desc->bootloader_start_offset) >> 8); | ||
153 | |||
154 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
155 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
156 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
157 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
158 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
159 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
160 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
161 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
162 | gk20a_writel(g, pwr_falcon_dmemd_r(0), GK20A_PMU_DMAIDX_UCODE); | ||
163 | gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code_lo << 8); | ||
164 | gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_code_hi); | ||
165 | gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_offset); | ||
166 | gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_code_size); | ||
167 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
168 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x0); | ||
169 | gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_imem_entry); | ||
170 | gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data_lo << 8); | ||
171 | gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_data_hi); | ||
172 | gk20a_writel(g, pwr_falcon_dmemd_r(0), desc->app_resident_data_size); | ||
173 | gk20a_writel(g, pwr_falcon_dmemd_r(0), 0x1); | ||
174 | gk20a_writel(g, pwr_falcon_dmemd_r(0), addr_args); | ||
175 | |||
176 | g->ops.pmu.write_dmatrfbase(g, | ||
177 | addr_load_lo - (desc->bootloader_imem_offset >> 8)); | ||
178 | |||
179 | blocks = ((desc->bootloader_size + 0xFF) & ~0xFF) >> 8; | ||
180 | |||
181 | for (i = 0; i < blocks; i++) { | ||
182 | gk20a_writel(g, pwr_falcon_dmatrfmoffs_r(), | ||
183 | desc->bootloader_imem_offset + (i << 8)); | ||
184 | gk20a_writel(g, pwr_falcon_dmatrffboffs_r(), | ||
185 | desc->bootloader_imem_offset + (i << 8)); | ||
186 | gk20a_writel(g, pwr_falcon_dmatrfcmd_r(), | ||
187 | pwr_falcon_dmatrfcmd_imem_f(1) | | ||
188 | pwr_falcon_dmatrfcmd_write_f(0) | | ||
189 | pwr_falcon_dmatrfcmd_size_f(6) | | ||
190 | pwr_falcon_dmatrfcmd_ctxdma_f(GK20A_PMU_DMAIDX_UCODE)); | ||
191 | } | ||
192 | |||
193 | nvgpu_flcn_bootstrap(pmu->flcn, desc->bootloader_entry_point); | ||
194 | |||
195 | gk20a_writel(g, pwr_falcon_os_r(), desc->app_version); | ||
196 | |||
197 | return 0; | ||
198 | } | ||
199 | |||
200 | static void pmu_handle_pg_sub_feature_msg(struct gk20a *g, struct pmu_msg *msg, | ||
201 | void *param, u32 handle, u32 status) | ||
202 | { | ||
203 | gk20a_dbg_fn(""); | ||
204 | |||
205 | if (status != 0) { | ||
206 | nvgpu_err(g, "Sub-feature mask update cmd aborted\n"); | ||
207 | return; | ||
208 | } | ||
209 | |||
210 | gv11b_dbg_pmu("sub-feature mask update is acknowledged from PMU %x\n", | ||
211 | msg->msg.pg.msg_type); | ||
212 | } | ||
213 | |||
214 | static void pmu_handle_pg_param_msg(struct gk20a *g, struct pmu_msg *msg, | ||
215 | void *param, u32 handle, u32 status) | ||
216 | { | ||
217 | gk20a_dbg_fn(""); | ||
218 | |||
219 | if (status != 0) { | ||
220 | nvgpu_err(g, "GR PARAM cmd aborted\n"); | ||
221 | return; | ||
222 | } | ||
223 | |||
224 | gv11b_dbg_pmu("GR PARAM is acknowledged from PMU %x\n", | ||
225 | msg->msg.pg.msg_type); | ||
226 | } | ||
227 | |||
228 | int gv11b_pg_gr_init(struct gk20a *g, u32 pg_engine_id) | ||
229 | { | ||
230 | struct nvgpu_pmu *pmu = &g->pmu; | ||
231 | struct pmu_cmd cmd; | ||
232 | u32 seq; | ||
233 | |||
234 | if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { | ||
235 | memset(&cmd, 0, sizeof(struct pmu_cmd)); | ||
236 | cmd.hdr.unit_id = PMU_UNIT_PG; | ||
237 | cmd.hdr.size = PMU_CMD_HDR_SIZE + | ||
238 | sizeof(struct pmu_pg_cmd_gr_init_param_v1); | ||
239 | cmd.cmd.pg.gr_init_param_v1.cmd_type = | ||
240 | PMU_PG_CMD_ID_PG_PARAM; | ||
241 | cmd.cmd.pg.gr_init_param_v1.sub_cmd_id = | ||
242 | PMU_PG_PARAM_CMD_GR_INIT_PARAM; | ||
243 | cmd.cmd.pg.gr_init_param_v1.featuremask = | ||
244 | PMU_PG_FEATURE_GR_POWER_GATING_ENABLED; | ||
245 | |||
246 | gv11b_dbg_pmu("cmd post PMU_PG_CMD_ID_PG_PARAM_INIT\n"); | ||
247 | nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ, | ||
248 | pmu_handle_pg_param_msg, pmu, &seq, ~0); | ||
249 | |||
250 | } else | ||
251 | return -EINVAL; | ||
252 | |||
253 | return 0; | ||
254 | } | ||
255 | |||
256 | int gv11b_pg_set_subfeature_mask(struct gk20a *g, u32 pg_engine_id) | ||
257 | { | ||
258 | struct nvgpu_pmu *pmu = &g->pmu; | ||
259 | struct pmu_cmd cmd; | ||
260 | u32 seq; | ||
261 | |||
262 | if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { | ||
263 | memset(&cmd, 0, sizeof(struct pmu_cmd)); | ||
264 | cmd.hdr.unit_id = PMU_UNIT_PG; | ||
265 | cmd.hdr.size = PMU_CMD_HDR_SIZE + | ||
266 | sizeof(struct pmu_pg_cmd_sub_feature_mask_update); | ||
267 | cmd.cmd.pg.sf_mask_update.cmd_type = | ||
268 | PMU_PG_CMD_ID_PG_PARAM; | ||
269 | cmd.cmd.pg.sf_mask_update.sub_cmd_id = | ||
270 | PMU_PG_PARAM_CMD_SUB_FEATURE_MASK_UPDATE; | ||
271 | cmd.cmd.pg.sf_mask_update.ctrl_id = | ||
272 | PMU_PG_ELPG_ENGINE_ID_GRAPHICS; | ||
273 | cmd.cmd.pg.sf_mask_update.enabled_mask = | ||
274 | PMU_PG_FEATURE_GR_POWER_GATING_ENABLED; | ||
275 | |||
276 | gv11b_dbg_pmu("cmd post PMU_PG_CMD_SUB_FEATURE_MASK_UPDATE\n"); | ||
277 | nvgpu_pmu_cmd_post(g, &cmd, NULL, NULL, PMU_COMMAND_QUEUE_HPQ, | ||
278 | pmu_handle_pg_sub_feature_msg, pmu, &seq, ~0); | ||
279 | } else | ||
280 | return -EINVAL; | ||
281 | |||
282 | return 0; | ||
283 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h new file mode 100644 index 00000000..809970ff --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/pmu_gv11b.h | |||
@@ -0,0 +1,37 @@ | |||
1 | /* | ||
2 | * GV11B PMU | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #ifndef __PMU_GV11B_H_ | ||
26 | #define __PMU_GV11B_H_ | ||
27 | |||
28 | struct gk20a; | ||
29 | |||
30 | bool gv11b_is_pmu_supported(struct gk20a *g); | ||
31 | int gv11b_pmu_bootstrap(struct nvgpu_pmu *pmu); | ||
32 | int gv11b_pg_gr_init(struct gk20a *g, u32 pg_engine_id); | ||
33 | int gv11b_pg_set_subfeature_mask(struct gk20a *g, u32 pg_engine_id); | ||
34 | bool gv11b_is_lazy_bootstrap(u32 falcon_id); | ||
35 | bool gv11b_is_priv_load(u32 falcon_id); | ||
36 | |||
37 | #endif /*__PMU_GV11B_H_*/ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/regops_gv11b.c b/drivers/gpu/nvgpu/gv11b/regops_gv11b.c new file mode 100644 index 00000000..c356785e --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/regops_gv11b.c | |||
@@ -0,0 +1,1548 @@ | |||
1 | /* | ||
2 | * Tegra GV11b GPU Driver Register Ops | ||
3 | * | ||
4 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/slab.h> | ||
26 | #include <linux/err.h> | ||
27 | #include <linux/bsearch.h> | ||
28 | #include <uapi/linux/nvgpu.h> | ||
29 | |||
30 | #include "gk20a/gk20a.h" | ||
31 | #include "gk20a/dbg_gpu_gk20a.h" | ||
32 | #include "gk20a/regops_gk20a.h" | ||
33 | #include "regops_gv11b.h" | ||
34 | |||
35 | static const struct regop_offset_range gv11b_global_whitelist_ranges[] = { | ||
36 | { 0x000004f0, 1}, | ||
37 | { 0x00001a00, 1}, | ||
38 | { 0x00009400, 1}, | ||
39 | { 0x00009410, 1}, | ||
40 | { 0x00009480, 1}, | ||
41 | { 0x00020200, 32}, | ||
42 | { 0x00021c04, 2}, | ||
43 | { 0x00021c14, 3}, | ||
44 | { 0x00021c24, 71}, | ||
45 | { 0x00021d44, 1}, | ||
46 | { 0x00021d4c, 1}, | ||
47 | { 0x00021d54, 1}, | ||
48 | { 0x00021d5c, 1}, | ||
49 | { 0x00021d68, 19}, | ||
50 | { 0x00021dbc, 16}, | ||
51 | { 0x00022430, 7}, | ||
52 | { 0x00022450, 1}, | ||
53 | { 0x0002245c, 2}, | ||
54 | { 0x00070000, 5}, | ||
55 | { 0x000840a8, 1}, | ||
56 | { 0x00084b5c, 1}, | ||
57 | { 0x000870a8, 1}, | ||
58 | { 0x000884e0, 1}, | ||
59 | { 0x00100c18, 3}, | ||
60 | { 0x00100c84, 1}, | ||
61 | { 0x0010a0a8, 1}, | ||
62 | { 0x0010a4f0, 1}, | ||
63 | { 0x0013c808, 2}, | ||
64 | { 0x0013cc14, 1}, | ||
65 | { 0x0013ec18, 1}, | ||
66 | { 0x00140028, 1}, | ||
67 | { 0x00140280, 1}, | ||
68 | { 0x001402a0, 1}, | ||
69 | { 0x00140350, 1}, | ||
70 | { 0x00140480, 1}, | ||
71 | { 0x001404a0, 1}, | ||
72 | { 0x00140550, 1}, | ||
73 | { 0x00140680, 1}, | ||
74 | { 0x001406a0, 1}, | ||
75 | { 0x00140750, 1}, | ||
76 | { 0x00142028, 1}, | ||
77 | { 0x00142280, 1}, | ||
78 | { 0x001422a0, 1}, | ||
79 | { 0x00142350, 1}, | ||
80 | { 0x00142480, 1}, | ||
81 | { 0x001424a0, 1}, | ||
82 | { 0x00142550, 1}, | ||
83 | { 0x00142680, 1}, | ||
84 | { 0x001426a0, 1}, | ||
85 | { 0x00142750, 1}, | ||
86 | { 0x0017e028, 1}, | ||
87 | { 0x0017e280, 1}, | ||
88 | { 0x0017e294, 1}, | ||
89 | { 0x0017e29c, 2}, | ||
90 | { 0x0017e2ac, 1}, | ||
91 | { 0x0017e350, 1}, | ||
92 | { 0x0017e39c, 1}, | ||
93 | { 0x0017e480, 1}, | ||
94 | { 0x0017e4a0, 1}, | ||
95 | { 0x0017e550, 1}, | ||
96 | { 0x0017e680, 1}, | ||
97 | { 0x0017e6a0, 1}, | ||
98 | { 0x0017e750, 1}, | ||
99 | { 0x00180040, 41}, | ||
100 | { 0x001800ec, 1}, | ||
101 | { 0x001800f8, 7}, | ||
102 | { 0x00180120, 2}, | ||
103 | { 0x00180240, 41}, | ||
104 | { 0x001802ec, 1}, | ||
105 | { 0x001802f8, 7}, | ||
106 | { 0x00180320, 2}, | ||
107 | { 0x00180440, 41}, | ||
108 | { 0x001804ec, 1}, | ||
109 | { 0x001804f8, 7}, | ||
110 | { 0x00180520, 2}, | ||
111 | { 0x00180640, 41}, | ||
112 | { 0x001806ec, 1}, | ||
113 | { 0x001806f8, 7}, | ||
114 | { 0x00180720, 2}, | ||
115 | { 0x00180840, 41}, | ||
116 | { 0x001808ec, 1}, | ||
117 | { 0x001808f8, 7}, | ||
118 | { 0x00180920, 2}, | ||
119 | { 0x00180a40, 41}, | ||
120 | { 0x00180aec, 1}, | ||
121 | { 0x00180af8, 7}, | ||
122 | { 0x00180b20, 2}, | ||
123 | { 0x00180c40, 41}, | ||
124 | { 0x00180cec, 1}, | ||
125 | { 0x00180cf8, 2}, | ||
126 | { 0x00180d04, 4}, | ||
127 | { 0x00180d20, 2}, | ||
128 | { 0x00180e40, 41}, | ||
129 | { 0x00180eec, 1}, | ||
130 | { 0x00180ef8, 2}, | ||
131 | { 0x00180f04, 4}, | ||
132 | { 0x00180f20, 2}, | ||
133 | { 0x00181040, 41}, | ||
134 | { 0x001810ec, 1}, | ||
135 | { 0x001810f8, 2}, | ||
136 | { 0x00181104, 4}, | ||
137 | { 0x00181120, 2}, | ||
138 | { 0x00181240, 41}, | ||
139 | { 0x001812ec, 1}, | ||
140 | { 0x001812f8, 2}, | ||
141 | { 0x00181304, 4}, | ||
142 | { 0x00181320, 2}, | ||
143 | { 0x00181440, 41}, | ||
144 | { 0x001814ec, 1}, | ||
145 | { 0x001814f8, 2}, | ||
146 | { 0x00181504, 4}, | ||
147 | { 0x00181520, 2}, | ||
148 | { 0x00181640, 41}, | ||
149 | { 0x001816ec, 1}, | ||
150 | { 0x001816f8, 2}, | ||
151 | { 0x00181704, 4}, | ||
152 | { 0x00181720, 2}, | ||
153 | { 0x00181840, 41}, | ||
154 | { 0x001818ec, 1}, | ||
155 | { 0x001818f8, 2}, | ||
156 | { 0x00181904, 4}, | ||
157 | { 0x00181920, 2}, | ||
158 | { 0x00181a40, 41}, | ||
159 | { 0x00181aec, 1}, | ||
160 | { 0x00181af8, 2}, | ||
161 | { 0x00181b04, 4}, | ||
162 | { 0x00181b20, 2}, | ||
163 | { 0x00181c40, 41}, | ||
164 | { 0x00181cec, 1}, | ||
165 | { 0x00181cf8, 2}, | ||
166 | { 0x00181d04, 4}, | ||
167 | { 0x00181d20, 2}, | ||
168 | { 0x00181e40, 41}, | ||
169 | { 0x00181eec, 1}, | ||
170 | { 0x00181ef8, 2}, | ||
171 | { 0x00181f04, 4}, | ||
172 | { 0x00181f20, 2}, | ||
173 | { 0x00182040, 41}, | ||
174 | { 0x001820ec, 1}, | ||
175 | { 0x001820f8, 2}, | ||
176 | { 0x00182104, 4}, | ||
177 | { 0x00182120, 2}, | ||
178 | { 0x00182240, 41}, | ||
179 | { 0x001822ec, 1}, | ||
180 | { 0x001822f8, 2}, | ||
181 | { 0x00182304, 4}, | ||
182 | { 0x00182320, 2}, | ||
183 | { 0x00182440, 41}, | ||
184 | { 0x001824ec, 1}, | ||
185 | { 0x001824f8, 2}, | ||
186 | { 0x00182504, 4}, | ||
187 | { 0x00182520, 2}, | ||
188 | { 0x00182640, 41}, | ||
189 | { 0x001826ec, 1}, | ||
190 | { 0x001826f8, 2}, | ||
191 | { 0x00182704, 4}, | ||
192 | { 0x00182720, 2}, | ||
193 | { 0x00182840, 41}, | ||
194 | { 0x001828ec, 1}, | ||
195 | { 0x001828f8, 2}, | ||
196 | { 0x00182904, 4}, | ||
197 | { 0x00182920, 2}, | ||
198 | { 0x00182a40, 41}, | ||
199 | { 0x00182aec, 1}, | ||
200 | { 0x00182af8, 2}, | ||
201 | { 0x00182b04, 4}, | ||
202 | { 0x00182b20, 2}, | ||
203 | { 0x00182c40, 41}, | ||
204 | { 0x00182cec, 1}, | ||
205 | { 0x00182cf8, 2}, | ||
206 | { 0x00182d04, 4}, | ||
207 | { 0x00182d20, 2}, | ||
208 | { 0x00182e40, 41}, | ||
209 | { 0x00182eec, 1}, | ||
210 | { 0x00182ef8, 2}, | ||
211 | { 0x00182f04, 4}, | ||
212 | { 0x00182f20, 2}, | ||
213 | { 0x00183040, 41}, | ||
214 | { 0x001830ec, 1}, | ||
215 | { 0x001830f8, 2}, | ||
216 | { 0x00183104, 4}, | ||
217 | { 0x00183120, 2}, | ||
218 | { 0x00183240, 41}, | ||
219 | { 0x001832ec, 1}, | ||
220 | { 0x001832f8, 2}, | ||
221 | { 0x00183304, 4}, | ||
222 | { 0x00183320, 2}, | ||
223 | { 0x00183440, 41}, | ||
224 | { 0x001834ec, 1}, | ||
225 | { 0x001834f8, 2}, | ||
226 | { 0x00183504, 4}, | ||
227 | { 0x00183520, 2}, | ||
228 | { 0x00183640, 41}, | ||
229 | { 0x001836ec, 1}, | ||
230 | { 0x001836f8, 2}, | ||
231 | { 0x00183704, 4}, | ||
232 | { 0x00183720, 2}, | ||
233 | { 0x00183840, 41}, | ||
234 | { 0x001838ec, 1}, | ||
235 | { 0x001838f8, 2}, | ||
236 | { 0x00183904, 4}, | ||
237 | { 0x00183920, 2}, | ||
238 | { 0x00183a40, 41}, | ||
239 | { 0x00183aec, 1}, | ||
240 | { 0x00183af8, 2}, | ||
241 | { 0x00183b04, 4}, | ||
242 | { 0x00183b20, 2}, | ||
243 | { 0x00183c40, 41}, | ||
244 | { 0x00183cec, 1}, | ||
245 | { 0x00183cf8, 2}, | ||
246 | { 0x00183d04, 4}, | ||
247 | { 0x00183d20, 2}, | ||
248 | { 0x00183e40, 41}, | ||
249 | { 0x00183eec, 1}, | ||
250 | { 0x00183ef8, 2}, | ||
251 | { 0x00183f04, 4}, | ||
252 | { 0x00183f20, 2}, | ||
253 | { 0x001c80a8, 1}, | ||
254 | { 0x001c9100, 1}, | ||
255 | { 0x001cc0a8, 1}, | ||
256 | { 0x001cd100, 1}, | ||
257 | { 0x001d00a8, 1}, | ||
258 | { 0x001d1100, 1}, | ||
259 | { 0x00200040, 41}, | ||
260 | { 0x002000ec, 1}, | ||
261 | { 0x002000f8, 7}, | ||
262 | { 0x00200120, 2}, | ||
263 | { 0x00200240, 41}, | ||
264 | { 0x002002ec, 1}, | ||
265 | { 0x002002f8, 7}, | ||
266 | { 0x00200320, 2}, | ||
267 | { 0x00200440, 41}, | ||
268 | { 0x002004ec, 1}, | ||
269 | { 0x002004f8, 7}, | ||
270 | { 0x00200520, 2}, | ||
271 | { 0x00200640, 41}, | ||
272 | { 0x002006ec, 1}, | ||
273 | { 0x002006f8, 7}, | ||
274 | { 0x00200720, 2}, | ||
275 | { 0x00200840, 41}, | ||
276 | { 0x002008ec, 1}, | ||
277 | { 0x002008f8, 2}, | ||
278 | { 0x00200904, 4}, | ||
279 | { 0x00200920, 2}, | ||
280 | { 0x00200a40, 41}, | ||
281 | { 0x00200aec, 1}, | ||
282 | { 0x00200af8, 2}, | ||
283 | { 0x00200b04, 4}, | ||
284 | { 0x00200b20, 2}, | ||
285 | { 0x00200c40, 41}, | ||
286 | { 0x00200cec, 1}, | ||
287 | { 0x00200cf8, 2}, | ||
288 | { 0x00200d04, 4}, | ||
289 | { 0x00200d20, 2}, | ||
290 | { 0x00200e40, 41}, | ||
291 | { 0x00200eec, 1}, | ||
292 | { 0x00200ef8, 2}, | ||
293 | { 0x00200f04, 4}, | ||
294 | { 0x00200f20, 2}, | ||
295 | { 0x00201040, 41}, | ||
296 | { 0x002010ec, 1}, | ||
297 | { 0x002010f8, 2}, | ||
298 | { 0x00201104, 4}, | ||
299 | { 0x00201120, 2}, | ||
300 | { 0x00201240, 41}, | ||
301 | { 0x002012ec, 1}, | ||
302 | { 0x002012f8, 2}, | ||
303 | { 0x00201304, 4}, | ||
304 | { 0x00201320, 2}, | ||
305 | { 0x00201440, 41}, | ||
306 | { 0x002014ec, 1}, | ||
307 | { 0x002014f8, 2}, | ||
308 | { 0x00201504, 4}, | ||
309 | { 0x00201520, 2}, | ||
310 | { 0x00201640, 41}, | ||
311 | { 0x002016ec, 1}, | ||
312 | { 0x002016f8, 2}, | ||
313 | { 0x00201704, 4}, | ||
314 | { 0x00201720, 2}, | ||
315 | { 0x00201840, 41}, | ||
316 | { 0x002018ec, 1}, | ||
317 | { 0x002018f8, 2}, | ||
318 | { 0x00201904, 4}, | ||
319 | { 0x00201920, 2}, | ||
320 | { 0x00201a40, 41}, | ||
321 | { 0x00201aec, 1}, | ||
322 | { 0x00201af8, 2}, | ||
323 | { 0x00201b04, 4}, | ||
324 | { 0x00201b20, 2}, | ||
325 | { 0x00201c40, 41}, | ||
326 | { 0x00201cec, 1}, | ||
327 | { 0x00201cf8, 2}, | ||
328 | { 0x00201d04, 4}, | ||
329 | { 0x00201d20, 2}, | ||
330 | { 0x00201e40, 41}, | ||
331 | { 0x00201eec, 1}, | ||
332 | { 0x00201ef8, 2}, | ||
333 | { 0x00201f04, 4}, | ||
334 | { 0x00201f20, 2}, | ||
335 | { 0x00202040, 41}, | ||
336 | { 0x002020ec, 1}, | ||
337 | { 0x002020f8, 2}, | ||
338 | { 0x00202104, 4}, | ||
339 | { 0x00202120, 2}, | ||
340 | { 0x00202240, 41}, | ||
341 | { 0x002022ec, 1}, | ||
342 | { 0x002022f8, 2}, | ||
343 | { 0x00202304, 4}, | ||
344 | { 0x00202320, 2}, | ||
345 | { 0x00202440, 41}, | ||
346 | { 0x002024ec, 1}, | ||
347 | { 0x002024f8, 2}, | ||
348 | { 0x00202504, 4}, | ||
349 | { 0x00202520, 2}, | ||
350 | { 0x00202640, 41}, | ||
351 | { 0x002026ec, 1}, | ||
352 | { 0x002026f8, 2}, | ||
353 | { 0x00202704, 4}, | ||
354 | { 0x00202720, 2}, | ||
355 | { 0x00202840, 41}, | ||
356 | { 0x002028ec, 1}, | ||
357 | { 0x002028f8, 2}, | ||
358 | { 0x00202904, 4}, | ||
359 | { 0x00202920, 2}, | ||
360 | { 0x00202a40, 41}, | ||
361 | { 0x00202aec, 1}, | ||
362 | { 0x00202af8, 2}, | ||
363 | { 0x00202b04, 4}, | ||
364 | { 0x00202b20, 2}, | ||
365 | { 0x00202c40, 41}, | ||
366 | { 0x00202cec, 1}, | ||
367 | { 0x00202cf8, 2}, | ||
368 | { 0x00202d04, 4}, | ||
369 | { 0x00202d20, 2}, | ||
370 | { 0x00202e40, 41}, | ||
371 | { 0x00202eec, 1}, | ||
372 | { 0x00202ef8, 2}, | ||
373 | { 0x00202f04, 4}, | ||
374 | { 0x00202f20, 2}, | ||
375 | { 0x00203040, 41}, | ||
376 | { 0x002030ec, 1}, | ||
377 | { 0x002030f8, 2}, | ||
378 | { 0x00203104, 4}, | ||
379 | { 0x00203120, 2}, | ||
380 | { 0x00203240, 41}, | ||
381 | { 0x002032ec, 1}, | ||
382 | { 0x002032f8, 2}, | ||
383 | { 0x00203304, 4}, | ||
384 | { 0x00203320, 2}, | ||
385 | { 0x00203440, 41}, | ||
386 | { 0x002034ec, 1}, | ||
387 | { 0x002034f8, 2}, | ||
388 | { 0x00203504, 4}, | ||
389 | { 0x00203520, 2}, | ||
390 | { 0x00203640, 41}, | ||
391 | { 0x002036ec, 1}, | ||
392 | { 0x002036f8, 2}, | ||
393 | { 0x00203704, 4}, | ||
394 | { 0x00203720, 2}, | ||
395 | { 0x00203840, 41}, | ||
396 | { 0x002038ec, 1}, | ||
397 | { 0x002038f8, 2}, | ||
398 | { 0x00203904, 4}, | ||
399 | { 0x00203920, 2}, | ||
400 | { 0x00203a40, 41}, | ||
401 | { 0x00203aec, 1}, | ||
402 | { 0x00203af8, 2}, | ||
403 | { 0x00203b04, 4}, | ||
404 | { 0x00203b20, 2}, | ||
405 | { 0x00203c40, 41}, | ||
406 | { 0x00203cec, 1}, | ||
407 | { 0x00203cf8, 2}, | ||
408 | { 0x00203d04, 4}, | ||
409 | { 0x00203d20, 2}, | ||
410 | { 0x00203e40, 41}, | ||
411 | { 0x00203eec, 1}, | ||
412 | { 0x00203ef8, 2}, | ||
413 | { 0x00203f04, 4}, | ||
414 | { 0x00203f20, 2}, | ||
415 | { 0x00240040, 41}, | ||
416 | { 0x002400ec, 1}, | ||
417 | { 0x002400f8, 7}, | ||
418 | { 0x00240120, 2}, | ||
419 | { 0x00240240, 41}, | ||
420 | { 0x002402ec, 1}, | ||
421 | { 0x002402f8, 7}, | ||
422 | { 0x00240320, 2}, | ||
423 | { 0x00240440, 41}, | ||
424 | { 0x002404ec, 1}, | ||
425 | { 0x002404f8, 7}, | ||
426 | { 0x00240520, 2}, | ||
427 | { 0x00240640, 41}, | ||
428 | { 0x002406ec, 1}, | ||
429 | { 0x002406f8, 7}, | ||
430 | { 0x00240720, 2}, | ||
431 | { 0x00240840, 41}, | ||
432 | { 0x002408ec, 1}, | ||
433 | { 0x002408f8, 7}, | ||
434 | { 0x00240920, 2}, | ||
435 | { 0x00240a40, 41}, | ||
436 | { 0x00240aec, 1}, | ||
437 | { 0x00240af8, 7}, | ||
438 | { 0x00240b20, 2}, | ||
439 | { 0x00240c40, 41}, | ||
440 | { 0x00240cec, 1}, | ||
441 | { 0x00240cf8, 2}, | ||
442 | { 0x00240d04, 4}, | ||
443 | { 0x00240d20, 2}, | ||
444 | { 0x00240e40, 41}, | ||
445 | { 0x00240eec, 1}, | ||
446 | { 0x00240ef8, 2}, | ||
447 | { 0x00240f04, 4}, | ||
448 | { 0x00240f20, 2}, | ||
449 | { 0x00241040, 41}, | ||
450 | { 0x002410ec, 1}, | ||
451 | { 0x002410f8, 2}, | ||
452 | { 0x00241104, 4}, | ||
453 | { 0x00241120, 2}, | ||
454 | { 0x00241240, 41}, | ||
455 | { 0x002412ec, 1}, | ||
456 | { 0x002412f8, 2}, | ||
457 | { 0x00241304, 4}, | ||
458 | { 0x00241320, 2}, | ||
459 | { 0x00241440, 41}, | ||
460 | { 0x002414ec, 1}, | ||
461 | { 0x002414f8, 2}, | ||
462 | { 0x00241504, 4}, | ||
463 | { 0x00241520, 2}, | ||
464 | { 0x00241640, 41}, | ||
465 | { 0x002416ec, 1}, | ||
466 | { 0x002416f8, 2}, | ||
467 | { 0x00241704, 4}, | ||
468 | { 0x00241720, 2}, | ||
469 | { 0x00241840, 41}, | ||
470 | { 0x002418ec, 1}, | ||
471 | { 0x002418f8, 2}, | ||
472 | { 0x00241904, 4}, | ||
473 | { 0x00241920, 2}, | ||
474 | { 0x00241a40, 41}, | ||
475 | { 0x00241aec, 1}, | ||
476 | { 0x00241af8, 2}, | ||
477 | { 0x00241b04, 4}, | ||
478 | { 0x00241b20, 2}, | ||
479 | { 0x00241c40, 41}, | ||
480 | { 0x00241cec, 1}, | ||
481 | { 0x00241cf8, 2}, | ||
482 | { 0x00241d04, 4}, | ||
483 | { 0x00241d20, 2}, | ||
484 | { 0x00241e40, 41}, | ||
485 | { 0x00241eec, 1}, | ||
486 | { 0x00241ef8, 2}, | ||
487 | { 0x00241f04, 4}, | ||
488 | { 0x00241f20, 2}, | ||
489 | { 0x00242040, 41}, | ||
490 | { 0x002420ec, 1}, | ||
491 | { 0x002420f8, 2}, | ||
492 | { 0x00242104, 4}, | ||
493 | { 0x00242120, 2}, | ||
494 | { 0x00242240, 41}, | ||
495 | { 0x002422ec, 1}, | ||
496 | { 0x002422f8, 2}, | ||
497 | { 0x00242304, 4}, | ||
498 | { 0x00242320, 2}, | ||
499 | { 0x00242440, 41}, | ||
500 | { 0x002424ec, 1}, | ||
501 | { 0x002424f8, 2}, | ||
502 | { 0x00242504, 4}, | ||
503 | { 0x00242520, 2}, | ||
504 | { 0x00242640, 41}, | ||
505 | { 0x002426ec, 1}, | ||
506 | { 0x002426f8, 2}, | ||
507 | { 0x00242704, 4}, | ||
508 | { 0x00242720, 2}, | ||
509 | { 0x00242840, 41}, | ||
510 | { 0x002428ec, 1}, | ||
511 | { 0x002428f8, 2}, | ||
512 | { 0x00242904, 4}, | ||
513 | { 0x00242920, 2}, | ||
514 | { 0x00242a40, 41}, | ||
515 | { 0x00242aec, 1}, | ||
516 | { 0x00242af8, 2}, | ||
517 | { 0x00242b04, 4}, | ||
518 | { 0x00242b20, 2}, | ||
519 | { 0x00242c40, 41}, | ||
520 | { 0x00242cec, 1}, | ||
521 | { 0x00242cf8, 2}, | ||
522 | { 0x00242d04, 4}, | ||
523 | { 0x00242d20, 2}, | ||
524 | { 0x00242e40, 41}, | ||
525 | { 0x00242eec, 1}, | ||
526 | { 0x00242ef8, 2}, | ||
527 | { 0x00242f04, 4}, | ||
528 | { 0x00242f20, 2}, | ||
529 | { 0x00243040, 41}, | ||
530 | { 0x002430ec, 1}, | ||
531 | { 0x002430f8, 2}, | ||
532 | { 0x00243104, 4}, | ||
533 | { 0x00243120, 2}, | ||
534 | { 0x00243240, 41}, | ||
535 | { 0x002432ec, 1}, | ||
536 | { 0x002432f8, 2}, | ||
537 | { 0x00243304, 4}, | ||
538 | { 0x00243320, 2}, | ||
539 | { 0x00243440, 41}, | ||
540 | { 0x002434ec, 1}, | ||
541 | { 0x002434f8, 2}, | ||
542 | { 0x00243504, 4}, | ||
543 | { 0x00243520, 2}, | ||
544 | { 0x00243640, 41}, | ||
545 | { 0x002436ec, 1}, | ||
546 | { 0x002436f8, 2}, | ||
547 | { 0x00243704, 4}, | ||
548 | { 0x00243720, 2}, | ||
549 | { 0x00243840, 41}, | ||
550 | { 0x002438ec, 1}, | ||
551 | { 0x002438f8, 2}, | ||
552 | { 0x00243904, 4}, | ||
553 | { 0x00243920, 2}, | ||
554 | { 0x00243a40, 41}, | ||
555 | { 0x00243aec, 1}, | ||
556 | { 0x00243af8, 2}, | ||
557 | { 0x00243b04, 4}, | ||
558 | { 0x00243b20, 2}, | ||
559 | { 0x00243c40, 41}, | ||
560 | { 0x00243cec, 1}, | ||
561 | { 0x00243cf8, 2}, | ||
562 | { 0x00243d04, 4}, | ||
563 | { 0x00243d20, 2}, | ||
564 | { 0x00243e40, 41}, | ||
565 | { 0x00243eec, 1}, | ||
566 | { 0x00243ef8, 2}, | ||
567 | { 0x00243f04, 4}, | ||
568 | { 0x00243f20, 2}, | ||
569 | { 0x00244000, 1}, | ||
570 | { 0x00244008, 1}, | ||
571 | { 0x00244010, 2}, | ||
572 | { 0x00246000, 1}, | ||
573 | { 0x00246008, 1}, | ||
574 | { 0x00246010, 2}, | ||
575 | { 0x00248000, 1}, | ||
576 | { 0x00248008, 1}, | ||
577 | { 0x00248010, 2}, | ||
578 | { 0x0024a000, 1}, | ||
579 | { 0x0024a008, 1}, | ||
580 | { 0x0024a010, 11}, | ||
581 | { 0x0024a040, 3}, | ||
582 | { 0x0024a050, 3}, | ||
583 | { 0x0024a060, 4}, | ||
584 | { 0x0024a074, 7}, | ||
585 | { 0x0024a094, 3}, | ||
586 | { 0x0024a0a4, 1}, | ||
587 | { 0x0024a100, 6}, | ||
588 | { 0x00250040, 25}, | ||
589 | { 0x002500c8, 7}, | ||
590 | { 0x002500ec, 1}, | ||
591 | { 0x002500f8, 2}, | ||
592 | { 0x00250104, 4}, | ||
593 | { 0x00250120, 2}, | ||
594 | { 0x00250240, 25}, | ||
595 | { 0x002502c8, 7}, | ||
596 | { 0x002502ec, 1}, | ||
597 | { 0x002502f8, 2}, | ||
598 | { 0x00250304, 4}, | ||
599 | { 0x00250320, 2}, | ||
600 | { 0x00250840, 25}, | ||
601 | { 0x002508c8, 7}, | ||
602 | { 0x002508ec, 1}, | ||
603 | { 0x002508f8, 2}, | ||
604 | { 0x00250904, 4}, | ||
605 | { 0x00250920, 2}, | ||
606 | { 0x00250a40, 25}, | ||
607 | { 0x00250ac8, 7}, | ||
608 | { 0x00250aec, 1}, | ||
609 | { 0x00250af8, 2}, | ||
610 | { 0x00250b04, 4}, | ||
611 | { 0x00250b20, 2}, | ||
612 | { 0x00251800, 3}, | ||
613 | { 0x00251810, 2}, | ||
614 | { 0x00251a00, 3}, | ||
615 | { 0x00251a10, 2}, | ||
616 | { 0x00278040, 25}, | ||
617 | { 0x002780c8, 7}, | ||
618 | { 0x002780ec, 1}, | ||
619 | { 0x002780f8, 2}, | ||
620 | { 0x00278104, 4}, | ||
621 | { 0x00278120, 2}, | ||
622 | { 0x00278240, 25}, | ||
623 | { 0x002782c8, 7}, | ||
624 | { 0x002782ec, 1}, | ||
625 | { 0x002782f8, 2}, | ||
626 | { 0x00278304, 4}, | ||
627 | { 0x00278320, 2}, | ||
628 | { 0x00278440, 25}, | ||
629 | { 0x002784c8, 7}, | ||
630 | { 0x002784ec, 1}, | ||
631 | { 0x002784f8, 2}, | ||
632 | { 0x00278504, 4}, | ||
633 | { 0x00278520, 2}, | ||
634 | { 0x00278640, 25}, | ||
635 | { 0x002786c8, 7}, | ||
636 | { 0x002786ec, 1}, | ||
637 | { 0x002786f8, 2}, | ||
638 | { 0x00278704, 4}, | ||
639 | { 0x00278720, 2}, | ||
640 | { 0x00278840, 25}, | ||
641 | { 0x002788c8, 7}, | ||
642 | { 0x002788ec, 1}, | ||
643 | { 0x002788f8, 2}, | ||
644 | { 0x00278904, 4}, | ||
645 | { 0x00278920, 2}, | ||
646 | { 0x00278a40, 25}, | ||
647 | { 0x00278ac8, 7}, | ||
648 | { 0x00278aec, 1}, | ||
649 | { 0x00278af8, 2}, | ||
650 | { 0x00278b04, 4}, | ||
651 | { 0x00278b20, 2}, | ||
652 | { 0x00278c40, 25}, | ||
653 | { 0x00278cc8, 7}, | ||
654 | { 0x00278cec, 1}, | ||
655 | { 0x00278cf8, 2}, | ||
656 | { 0x00278d04, 4}, | ||
657 | { 0x00278d20, 2}, | ||
658 | { 0x00278e40, 25}, | ||
659 | { 0x00278ec8, 7}, | ||
660 | { 0x00278eec, 1}, | ||
661 | { 0x00278ef8, 2}, | ||
662 | { 0x00278f04, 4}, | ||
663 | { 0x00278f20, 2}, | ||
664 | { 0x00279040, 25}, | ||
665 | { 0x002790c8, 7}, | ||
666 | { 0x002790ec, 1}, | ||
667 | { 0x002790f8, 2}, | ||
668 | { 0x00279104, 4}, | ||
669 | { 0x00279120, 2}, | ||
670 | { 0x00279240, 25}, | ||
671 | { 0x002792c8, 7}, | ||
672 | { 0x002792ec, 1}, | ||
673 | { 0x002792f8, 2}, | ||
674 | { 0x00279304, 4}, | ||
675 | { 0x00279320, 2}, | ||
676 | { 0x00279440, 25}, | ||
677 | { 0x002794c8, 7}, | ||
678 | { 0x002794ec, 1}, | ||
679 | { 0x002794f8, 2}, | ||
680 | { 0x00279504, 4}, | ||
681 | { 0x00279520, 2}, | ||
682 | { 0x00279640, 25}, | ||
683 | { 0x002796c8, 7}, | ||
684 | { 0x002796ec, 1}, | ||
685 | { 0x002796f8, 2}, | ||
686 | { 0x00279704, 4}, | ||
687 | { 0x00279720, 2}, | ||
688 | { 0x00279840, 25}, | ||
689 | { 0x002798c8, 7}, | ||
690 | { 0x002798ec, 1}, | ||
691 | { 0x002798f8, 2}, | ||
692 | { 0x00279904, 4}, | ||
693 | { 0x00279920, 2}, | ||
694 | { 0x00279a40, 25}, | ||
695 | { 0x00279ac8, 7}, | ||
696 | { 0x00279aec, 1}, | ||
697 | { 0x00279af8, 2}, | ||
698 | { 0x00279b04, 4}, | ||
699 | { 0x00279b20, 2}, | ||
700 | { 0x00279c40, 25}, | ||
701 | { 0x00279cc8, 7}, | ||
702 | { 0x00279cec, 1}, | ||
703 | { 0x00279cf8, 2}, | ||
704 | { 0x00279d04, 4}, | ||
705 | { 0x00279d20, 2}, | ||
706 | { 0x00279e40, 25}, | ||
707 | { 0x00279ec8, 7}, | ||
708 | { 0x00279eec, 1}, | ||
709 | { 0x00279ef8, 2}, | ||
710 | { 0x00279f04, 4}, | ||
711 | { 0x00279f20, 2}, | ||
712 | { 0x0027a040, 25}, | ||
713 | { 0x0027a0c8, 7}, | ||
714 | { 0x0027a0ec, 1}, | ||
715 | { 0x0027a0f8, 2}, | ||
716 | { 0x0027a104, 4}, | ||
717 | { 0x0027a120, 2}, | ||
718 | { 0x0027a240, 25}, | ||
719 | { 0x0027a2c8, 7}, | ||
720 | { 0x0027a2ec, 1}, | ||
721 | { 0x0027a2f8, 2}, | ||
722 | { 0x0027a304, 4}, | ||
723 | { 0x0027a320, 2}, | ||
724 | { 0x0027a440, 25}, | ||
725 | { 0x0027a4c8, 7}, | ||
726 | { 0x0027a4ec, 1}, | ||
727 | { 0x0027a4f8, 2}, | ||
728 | { 0x0027a504, 4}, | ||
729 | { 0x0027a520, 2}, | ||
730 | { 0x0027a640, 25}, | ||
731 | { 0x0027a6c8, 7}, | ||
732 | { 0x0027a6ec, 1}, | ||
733 | { 0x0027a6f8, 2}, | ||
734 | { 0x0027a704, 4}, | ||
735 | { 0x0027a720, 2}, | ||
736 | { 0x0027a840, 25}, | ||
737 | { 0x0027a8c8, 7}, | ||
738 | { 0x0027a8ec, 1}, | ||
739 | { 0x0027a8f8, 2}, | ||
740 | { 0x0027a904, 4}, | ||
741 | { 0x0027a920, 2}, | ||
742 | { 0x0027aa40, 25}, | ||
743 | { 0x0027aac8, 7}, | ||
744 | { 0x0027aaec, 1}, | ||
745 | { 0x0027aaf8, 2}, | ||
746 | { 0x0027ab04, 4}, | ||
747 | { 0x0027ab20, 2}, | ||
748 | { 0x0027ac40, 25}, | ||
749 | { 0x0027acc8, 7}, | ||
750 | { 0x0027acec, 1}, | ||
751 | { 0x0027acf8, 2}, | ||
752 | { 0x0027ad04, 4}, | ||
753 | { 0x0027ad20, 2}, | ||
754 | { 0x0027ae40, 25}, | ||
755 | { 0x0027aec8, 7}, | ||
756 | { 0x0027aeec, 1}, | ||
757 | { 0x0027aef8, 2}, | ||
758 | { 0x0027af04, 4}, | ||
759 | { 0x0027af20, 2}, | ||
760 | { 0x0027b040, 25}, | ||
761 | { 0x0027b0c8, 7}, | ||
762 | { 0x0027b0ec, 1}, | ||
763 | { 0x0027b0f8, 2}, | ||
764 | { 0x0027b104, 4}, | ||
765 | { 0x0027b120, 2}, | ||
766 | { 0x0027b240, 25}, | ||
767 | { 0x0027b2c8, 7}, | ||
768 | { 0x0027b2ec, 1}, | ||
769 | { 0x0027b2f8, 2}, | ||
770 | { 0x0027b304, 4}, | ||
771 | { 0x0027b320, 2}, | ||
772 | { 0x0027b440, 25}, | ||
773 | { 0x0027b4c8, 7}, | ||
774 | { 0x0027b4ec, 1}, | ||
775 | { 0x0027b4f8, 2}, | ||
776 | { 0x0027b504, 4}, | ||
777 | { 0x0027b520, 2}, | ||
778 | { 0x0027b640, 25}, | ||
779 | { 0x0027b6c8, 7}, | ||
780 | { 0x0027b6ec, 1}, | ||
781 | { 0x0027b6f8, 2}, | ||
782 | { 0x0027b704, 4}, | ||
783 | { 0x0027b720, 2}, | ||
784 | { 0x0027b840, 25}, | ||
785 | { 0x0027b8c8, 7}, | ||
786 | { 0x0027b8ec, 1}, | ||
787 | { 0x0027b8f8, 2}, | ||
788 | { 0x0027b904, 4}, | ||
789 | { 0x0027b920, 2}, | ||
790 | { 0x0027ba40, 25}, | ||
791 | { 0x0027bac8, 7}, | ||
792 | { 0x0027baec, 1}, | ||
793 | { 0x0027baf8, 2}, | ||
794 | { 0x0027bb04, 4}, | ||
795 | { 0x0027bb20, 2}, | ||
796 | { 0x0027bc40, 25}, | ||
797 | { 0x0027bcc8, 7}, | ||
798 | { 0x0027bcec, 1}, | ||
799 | { 0x0027bcf8, 2}, | ||
800 | { 0x0027bd04, 4}, | ||
801 | { 0x0027bd20, 2}, | ||
802 | { 0x0027be40, 25}, | ||
803 | { 0x0027bec8, 7}, | ||
804 | { 0x0027beec, 1}, | ||
805 | { 0x0027bef8, 2}, | ||
806 | { 0x0027bf04, 4}, | ||
807 | { 0x0027bf20, 2}, | ||
808 | { 0x0027c040, 25}, | ||
809 | { 0x0027c0c8, 7}, | ||
810 | { 0x0027c0ec, 1}, | ||
811 | { 0x0027c0f8, 2}, | ||
812 | { 0x0027c104, 4}, | ||
813 | { 0x0027c120, 2}, | ||
814 | { 0x0027c240, 25}, | ||
815 | { 0x0027c2c8, 7}, | ||
816 | { 0x0027c2ec, 1}, | ||
817 | { 0x0027c2f8, 2}, | ||
818 | { 0x0027c304, 4}, | ||
819 | { 0x0027c320, 2}, | ||
820 | { 0x0027c440, 25}, | ||
821 | { 0x0027c4c8, 7}, | ||
822 | { 0x0027c4ec, 1}, | ||
823 | { 0x0027c4f8, 2}, | ||
824 | { 0x0027c504, 4}, | ||
825 | { 0x0027c520, 2}, | ||
826 | { 0x0027c640, 25}, | ||
827 | { 0x0027c6c8, 7}, | ||
828 | { 0x0027c6ec, 1}, | ||
829 | { 0x0027c6f8, 2}, | ||
830 | { 0x0027c704, 4}, | ||
831 | { 0x0027c720, 2}, | ||
832 | { 0x0027c840, 25}, | ||
833 | { 0x0027c8c8, 7}, | ||
834 | { 0x0027c8ec, 1}, | ||
835 | { 0x0027c8f8, 2}, | ||
836 | { 0x0027c904, 4}, | ||
837 | { 0x0027c920, 2}, | ||
838 | { 0x0027ca40, 25}, | ||
839 | { 0x0027cac8, 7}, | ||
840 | { 0x0027caec, 1}, | ||
841 | { 0x0027caf8, 2}, | ||
842 | { 0x0027cb04, 4}, | ||
843 | { 0x0027cb20, 2}, | ||
844 | { 0x0027cc40, 25}, | ||
845 | { 0x0027ccc8, 7}, | ||
846 | { 0x0027ccec, 1}, | ||
847 | { 0x0027ccf8, 2}, | ||
848 | { 0x0027cd04, 4}, | ||
849 | { 0x0027cd20, 2}, | ||
850 | { 0x0027ce40, 25}, | ||
851 | { 0x0027cec8, 7}, | ||
852 | { 0x0027ceec, 1}, | ||
853 | { 0x0027cef8, 2}, | ||
854 | { 0x0027cf04, 4}, | ||
855 | { 0x0027cf20, 2}, | ||
856 | { 0x0027d040, 25}, | ||
857 | { 0x0027d0c8, 7}, | ||
858 | { 0x0027d0ec, 1}, | ||
859 | { 0x0027d0f8, 2}, | ||
860 | { 0x0027d104, 4}, | ||
861 | { 0x0027d120, 2}, | ||
862 | { 0x0027d240, 25}, | ||
863 | { 0x0027d2c8, 7}, | ||
864 | { 0x0027d2ec, 1}, | ||
865 | { 0x0027d2f8, 2}, | ||
866 | { 0x0027d304, 4}, | ||
867 | { 0x0027d320, 2}, | ||
868 | { 0x0027d440, 25}, | ||
869 | { 0x0027d4c8, 7}, | ||
870 | { 0x0027d4ec, 1}, | ||
871 | { 0x0027d4f8, 2}, | ||
872 | { 0x0027d504, 4}, | ||
873 | { 0x0027d520, 2}, | ||
874 | { 0x0027d640, 25}, | ||
875 | { 0x0027d6c8, 7}, | ||
876 | { 0x0027d6ec, 1}, | ||
877 | { 0x0027d6f8, 2}, | ||
878 | { 0x0027d704, 4}, | ||
879 | { 0x0027d720, 2}, | ||
880 | { 0x0027d840, 25}, | ||
881 | { 0x0027d8c8, 7}, | ||
882 | { 0x0027d8ec, 1}, | ||
883 | { 0x0027d8f8, 2}, | ||
884 | { 0x0027d904, 4}, | ||
885 | { 0x0027d920, 2}, | ||
886 | { 0x0027da40, 25}, | ||
887 | { 0x0027dac8, 7}, | ||
888 | { 0x0027daec, 1}, | ||
889 | { 0x0027daf8, 2}, | ||
890 | { 0x0027db04, 4}, | ||
891 | { 0x0027db20, 2}, | ||
892 | { 0x0027dc40, 25}, | ||
893 | { 0x0027dcc8, 7}, | ||
894 | { 0x0027dcec, 1}, | ||
895 | { 0x0027dcf8, 2}, | ||
896 | { 0x0027dd04, 4}, | ||
897 | { 0x0027dd20, 2}, | ||
898 | { 0x0027de40, 25}, | ||
899 | { 0x0027dec8, 7}, | ||
900 | { 0x0027deec, 1}, | ||
901 | { 0x0027def8, 2}, | ||
902 | { 0x0027df04, 4}, | ||
903 | { 0x0027df20, 2}, | ||
904 | { 0x0027e040, 25}, | ||
905 | { 0x0027e0c8, 7}, | ||
906 | { 0x0027e0ec, 1}, | ||
907 | { 0x0027e0f8, 2}, | ||
908 | { 0x0027e104, 4}, | ||
909 | { 0x0027e120, 2}, | ||
910 | { 0x0027e240, 25}, | ||
911 | { 0x0027e2c8, 7}, | ||
912 | { 0x0027e2ec, 1}, | ||
913 | { 0x0027e2f8, 2}, | ||
914 | { 0x0027e304, 4}, | ||
915 | { 0x0027e320, 2}, | ||
916 | { 0x0027e440, 25}, | ||
917 | { 0x0027e4c8, 7}, | ||
918 | { 0x0027e4ec, 1}, | ||
919 | { 0x0027e4f8, 2}, | ||
920 | { 0x0027e504, 4}, | ||
921 | { 0x0027e520, 2}, | ||
922 | { 0x0027e640, 25}, | ||
923 | { 0x0027e6c8, 7}, | ||
924 | { 0x0027e6ec, 1}, | ||
925 | { 0x0027e6f8, 2}, | ||
926 | { 0x0027e704, 4}, | ||
927 | { 0x0027e720, 2}, | ||
928 | { 0x0027e840, 25}, | ||
929 | { 0x0027e8c8, 7}, | ||
930 | { 0x0027e8ec, 1}, | ||
931 | { 0x0027e8f8, 2}, | ||
932 | { 0x0027e904, 4}, | ||
933 | { 0x0027e920, 2}, | ||
934 | { 0x0027ea40, 25}, | ||
935 | { 0x0027eac8, 7}, | ||
936 | { 0x0027eaec, 1}, | ||
937 | { 0x0027eaf8, 2}, | ||
938 | { 0x0027eb04, 4}, | ||
939 | { 0x0027eb20, 2}, | ||
940 | { 0x0027ec40, 25}, | ||
941 | { 0x0027ecc8, 7}, | ||
942 | { 0x0027ecec, 1}, | ||
943 | { 0x0027ecf8, 2}, | ||
944 | { 0x0027ed04, 4}, | ||
945 | { 0x0027ed20, 2}, | ||
946 | { 0x0027ee40, 25}, | ||
947 | { 0x0027eec8, 7}, | ||
948 | { 0x0027eeec, 1}, | ||
949 | { 0x0027eef8, 2}, | ||
950 | { 0x0027ef04, 4}, | ||
951 | { 0x0027ef20, 2}, | ||
952 | { 0x0027f040, 25}, | ||
953 | { 0x0027f0c8, 7}, | ||
954 | { 0x0027f0ec, 1}, | ||
955 | { 0x0027f0f8, 2}, | ||
956 | { 0x0027f104, 4}, | ||
957 | { 0x0027f120, 2}, | ||
958 | { 0x0027f240, 25}, | ||
959 | { 0x0027f2c8, 7}, | ||
960 | { 0x0027f2ec, 1}, | ||
961 | { 0x0027f2f8, 2}, | ||
962 | { 0x0027f304, 4}, | ||
963 | { 0x0027f320, 2}, | ||
964 | { 0x0027f440, 25}, | ||
965 | { 0x0027f4c8, 7}, | ||
966 | { 0x0027f4ec, 1}, | ||
967 | { 0x0027f4f8, 2}, | ||
968 | { 0x0027f504, 4}, | ||
969 | { 0x0027f520, 2}, | ||
970 | { 0x0027f640, 25}, | ||
971 | { 0x0027f6c8, 7}, | ||
972 | { 0x0027f6ec, 1}, | ||
973 | { 0x0027f6f8, 2}, | ||
974 | { 0x0027f704, 4}, | ||
975 | { 0x0027f720, 2}, | ||
976 | { 0x0027f840, 25}, | ||
977 | { 0x0027f8c8, 7}, | ||
978 | { 0x0027f8ec, 1}, | ||
979 | { 0x0027f8f8, 2}, | ||
980 | { 0x0027f904, 4}, | ||
981 | { 0x0027f920, 2}, | ||
982 | { 0x0027fa40, 25}, | ||
983 | { 0x0027fac8, 7}, | ||
984 | { 0x0027faec, 1}, | ||
985 | { 0x0027faf8, 2}, | ||
986 | { 0x0027fb04, 4}, | ||
987 | { 0x0027fb20, 2}, | ||
988 | { 0x0027fc40, 25}, | ||
989 | { 0x0027fcc8, 7}, | ||
990 | { 0x0027fcec, 1}, | ||
991 | { 0x0027fcf8, 2}, | ||
992 | { 0x0027fd04, 4}, | ||
993 | { 0x0027fd20, 2}, | ||
994 | { 0x0027fe40, 25}, | ||
995 | { 0x0027fec8, 7}, | ||
996 | { 0x0027feec, 1}, | ||
997 | { 0x0027fef8, 2}, | ||
998 | { 0x0027ff04, 4}, | ||
999 | { 0x0027ff20, 2}, | ||
1000 | { 0x00400500, 1}, | ||
1001 | { 0x0040415c, 1}, | ||
1002 | { 0x00404468, 1}, | ||
1003 | { 0x00404498, 1}, | ||
1004 | { 0x00405800, 1}, | ||
1005 | { 0x00405840, 2}, | ||
1006 | { 0x00405850, 1}, | ||
1007 | { 0x00405908, 1}, | ||
1008 | { 0x00405a00, 1}, | ||
1009 | { 0x00405b50, 1}, | ||
1010 | { 0x00406024, 5}, | ||
1011 | { 0x00407010, 1}, | ||
1012 | { 0x00407808, 1}, | ||
1013 | { 0x0040803c, 1}, | ||
1014 | { 0x00408804, 1}, | ||
1015 | { 0x0040880c, 1}, | ||
1016 | { 0x00408900, 2}, | ||
1017 | { 0x00408910, 1}, | ||
1018 | { 0x00408944, 1}, | ||
1019 | { 0x00408984, 1}, | ||
1020 | { 0x004090a8, 1}, | ||
1021 | { 0x004098a0, 1}, | ||
1022 | { 0x00409b00, 1}, | ||
1023 | { 0x0041000c, 1}, | ||
1024 | { 0x00410110, 1}, | ||
1025 | { 0x00410184, 1}, | ||
1026 | { 0x0041040c, 1}, | ||
1027 | { 0x00410510, 1}, | ||
1028 | { 0x00410584, 1}, | ||
1029 | { 0x00418000, 1}, | ||
1030 | { 0x00418008, 1}, | ||
1031 | { 0x00418380, 2}, | ||
1032 | { 0x00418400, 2}, | ||
1033 | { 0x004184a0, 1}, | ||
1034 | { 0x00418604, 1}, | ||
1035 | { 0x00418680, 1}, | ||
1036 | { 0x00418704, 1}, | ||
1037 | { 0x00418714, 1}, | ||
1038 | { 0x00418800, 1}, | ||
1039 | { 0x0041881c, 1}, | ||
1040 | { 0x00418830, 1}, | ||
1041 | { 0x00418884, 1}, | ||
1042 | { 0x004188b0, 1}, | ||
1043 | { 0x004188c8, 3}, | ||
1044 | { 0x004188fc, 1}, | ||
1045 | { 0x00418b04, 1}, | ||
1046 | { 0x00418c04, 1}, | ||
1047 | { 0x00418c10, 8}, | ||
1048 | { 0x00418c88, 1}, | ||
1049 | { 0x00418d00, 1}, | ||
1050 | { 0x00418e00, 1}, | ||
1051 | { 0x00418e08, 1}, | ||
1052 | { 0x00418e34, 1}, | ||
1053 | { 0x00418e40, 4}, | ||
1054 | { 0x00418e58, 16}, | ||
1055 | { 0x00418f08, 1}, | ||
1056 | { 0x00419000, 1}, | ||
1057 | { 0x0041900c, 1}, | ||
1058 | { 0x00419018, 1}, | ||
1059 | { 0x00419854, 1}, | ||
1060 | { 0x00419864, 1}, | ||
1061 | { 0x00419a04, 2}, | ||
1062 | { 0x00419ab0, 1}, | ||
1063 | { 0x00419b04, 1}, | ||
1064 | { 0x00419b3c, 1}, | ||
1065 | { 0x00419b48, 1}, | ||
1066 | { 0x00419b50, 1}, | ||
1067 | { 0x00419ba0, 2}, | ||
1068 | { 0x00419bb0, 1}, | ||
1069 | { 0x00419bdc, 1}, | ||
1070 | { 0x00419c0c, 1}, | ||
1071 | { 0x00419d00, 1}, | ||
1072 | { 0x00419d08, 2}, | ||
1073 | { 0x00419e08, 1}, | ||
1074 | { 0x00419e80, 8}, | ||
1075 | { 0x00419ea8, 5}, | ||
1076 | { 0x00419f00, 8}, | ||
1077 | { 0x00419f28, 5}, | ||
1078 | { 0x00419f80, 8}, | ||
1079 | { 0x00419fa8, 5}, | ||
1080 | { 0x0041a02c, 2}, | ||
1081 | { 0x0041a0a8, 1}, | ||
1082 | { 0x0041a8a0, 3}, | ||
1083 | { 0x0041b014, 1}, | ||
1084 | { 0x0041b0cc, 1}, | ||
1085 | { 0x0041b1dc, 1}, | ||
1086 | { 0x0041b214, 1}, | ||
1087 | { 0x0041b2cc, 1}, | ||
1088 | { 0x0041b3dc, 1}, | ||
1089 | { 0x0041be0c, 3}, | ||
1090 | { 0x0041becc, 1}, | ||
1091 | { 0x0041bfdc, 1}, | ||
1092 | { 0x0041c054, 1}, | ||
1093 | { 0x0041c2b0, 1}, | ||
1094 | { 0x0041c304, 1}, | ||
1095 | { 0x0041c33c, 1}, | ||
1096 | { 0x0041c348, 1}, | ||
1097 | { 0x0041c350, 1}, | ||
1098 | { 0x0041c3a0, 2}, | ||
1099 | { 0x0041c3b0, 1}, | ||
1100 | { 0x0041c3dc, 1}, | ||
1101 | { 0x0041c40c, 1}, | ||
1102 | { 0x0041c500, 1}, | ||
1103 | { 0x0041c508, 2}, | ||
1104 | { 0x0041c608, 1}, | ||
1105 | { 0x0041c680, 8}, | ||
1106 | { 0x0041c6a8, 5}, | ||
1107 | { 0x0041c700, 8}, | ||
1108 | { 0x0041c728, 5}, | ||
1109 | { 0x0041c780, 8}, | ||
1110 | { 0x0041c7a8, 5}, | ||
1111 | { 0x0041c854, 1}, | ||
1112 | { 0x0041cab0, 1}, | ||
1113 | { 0x0041cb04, 1}, | ||
1114 | { 0x0041cb3c, 1}, | ||
1115 | { 0x0041cb48, 1}, | ||
1116 | { 0x0041cb50, 1}, | ||
1117 | { 0x0041cba0, 2}, | ||
1118 | { 0x0041cbb0, 1}, | ||
1119 | { 0x0041cbdc, 1}, | ||
1120 | { 0x0041cc0c, 1}, | ||
1121 | { 0x0041cd00, 1}, | ||
1122 | { 0x0041cd08, 2}, | ||
1123 | { 0x0041ce08, 1}, | ||
1124 | { 0x0041ce80, 8}, | ||
1125 | { 0x0041cea8, 5}, | ||
1126 | { 0x0041cf00, 8}, | ||
1127 | { 0x0041cf28, 5}, | ||
1128 | { 0x0041cf80, 8}, | ||
1129 | { 0x0041cfa8, 5}, | ||
1130 | { 0x0041d054, 1}, | ||
1131 | { 0x0041d2b0, 1}, | ||
1132 | { 0x0041d304, 1}, | ||
1133 | { 0x0041d33c, 1}, | ||
1134 | { 0x0041d348, 1}, | ||
1135 | { 0x0041d350, 1}, | ||
1136 | { 0x0041d3a0, 2}, | ||
1137 | { 0x0041d3b0, 1}, | ||
1138 | { 0x0041d3dc, 1}, | ||
1139 | { 0x0041d40c, 1}, | ||
1140 | { 0x0041d500, 1}, | ||
1141 | { 0x0041d508, 2}, | ||
1142 | { 0x0041d608, 1}, | ||
1143 | { 0x0041d680, 8}, | ||
1144 | { 0x0041d6a8, 5}, | ||
1145 | { 0x0041d700, 8}, | ||
1146 | { 0x0041d728, 5}, | ||
1147 | { 0x0041d780, 8}, | ||
1148 | { 0x0041d7a8, 5}, | ||
1149 | { 0x0041d854, 1}, | ||
1150 | { 0x0041dab0, 1}, | ||
1151 | { 0x0041db04, 1}, | ||
1152 | { 0x0041db3c, 1}, | ||
1153 | { 0x0041db48, 1}, | ||
1154 | { 0x0041db50, 1}, | ||
1155 | { 0x0041dba0, 2}, | ||
1156 | { 0x0041dbb0, 1}, | ||
1157 | { 0x0041dbdc, 1}, | ||
1158 | { 0x0041dc0c, 1}, | ||
1159 | { 0x0041dd00, 1}, | ||
1160 | { 0x0041dd08, 2}, | ||
1161 | { 0x0041de08, 1}, | ||
1162 | { 0x0041de80, 8}, | ||
1163 | { 0x0041dea8, 5}, | ||
1164 | { 0x0041df00, 8}, | ||
1165 | { 0x0041df28, 5}, | ||
1166 | { 0x0041df80, 8}, | ||
1167 | { 0x0041dfa8, 5}, | ||
1168 | { 0x00481a00, 19}, | ||
1169 | { 0x00481b00, 50}, | ||
1170 | { 0x00481e00, 50}, | ||
1171 | { 0x00481f00, 50}, | ||
1172 | { 0x00484200, 19}, | ||
1173 | { 0x00484300, 50}, | ||
1174 | { 0x00484600, 50}, | ||
1175 | { 0x00484700, 50}, | ||
1176 | { 0x00484a00, 19}, | ||
1177 | { 0x00484b00, 50}, | ||
1178 | { 0x00484e00, 50}, | ||
1179 | { 0x00484f00, 50}, | ||
1180 | { 0x00485200, 19}, | ||
1181 | { 0x00485300, 50}, | ||
1182 | { 0x00485600, 50}, | ||
1183 | { 0x00485700, 50}, | ||
1184 | { 0x00485a00, 19}, | ||
1185 | { 0x00485b00, 50}, | ||
1186 | { 0x00485e00, 50}, | ||
1187 | { 0x00485f00, 50}, | ||
1188 | { 0x00500384, 1}, | ||
1189 | { 0x005004a0, 1}, | ||
1190 | { 0x00500604, 1}, | ||
1191 | { 0x00500680, 1}, | ||
1192 | { 0x00500714, 1}, | ||
1193 | { 0x0050081c, 1}, | ||
1194 | { 0x00500884, 1}, | ||
1195 | { 0x005008b0, 1}, | ||
1196 | { 0x005008c8, 3}, | ||
1197 | { 0x005008fc, 1}, | ||
1198 | { 0x00500b04, 1}, | ||
1199 | { 0x00500c04, 1}, | ||
1200 | { 0x00500c10, 8}, | ||
1201 | { 0x00500c88, 1}, | ||
1202 | { 0x00500d00, 1}, | ||
1203 | { 0x00500e08, 1}, | ||
1204 | { 0x00500f08, 1}, | ||
1205 | { 0x00501000, 1}, | ||
1206 | { 0x0050100c, 1}, | ||
1207 | { 0x00501018, 1}, | ||
1208 | { 0x00501854, 1}, | ||
1209 | { 0x00501ab0, 1}, | ||
1210 | { 0x00501b04, 1}, | ||
1211 | { 0x00501b3c, 1}, | ||
1212 | { 0x00501b48, 1}, | ||
1213 | { 0x00501b50, 1}, | ||
1214 | { 0x00501ba0, 2}, | ||
1215 | { 0x00501bb0, 1}, | ||
1216 | { 0x00501bdc, 1}, | ||
1217 | { 0x00501c0c, 1}, | ||
1218 | { 0x00501d00, 1}, | ||
1219 | { 0x00501d08, 2}, | ||
1220 | { 0x00501e08, 1}, | ||
1221 | { 0x00501e80, 8}, | ||
1222 | { 0x00501ea8, 5}, | ||
1223 | { 0x00501f00, 8}, | ||
1224 | { 0x00501f28, 5}, | ||
1225 | { 0x00501f80, 8}, | ||
1226 | { 0x00501fa8, 5}, | ||
1227 | { 0x0050202c, 2}, | ||
1228 | { 0x005020a8, 1}, | ||
1229 | { 0x005028a0, 3}, | ||
1230 | { 0x00503014, 1}, | ||
1231 | { 0x005030cc, 1}, | ||
1232 | { 0x005031dc, 1}, | ||
1233 | { 0x00503214, 1}, | ||
1234 | { 0x005032cc, 1}, | ||
1235 | { 0x005033dc, 1}, | ||
1236 | { 0x00503e14, 1}, | ||
1237 | { 0x00503ecc, 1}, | ||
1238 | { 0x00503fdc, 1}, | ||
1239 | { 0x00504054, 1}, | ||
1240 | { 0x005042b0, 1}, | ||
1241 | { 0x00504304, 1}, | ||
1242 | { 0x0050433c, 1}, | ||
1243 | { 0x00504348, 1}, | ||
1244 | { 0x00504350, 1}, | ||
1245 | { 0x005043a0, 2}, | ||
1246 | { 0x005043b0, 1}, | ||
1247 | { 0x005043dc, 1}, | ||
1248 | { 0x0050440c, 1}, | ||
1249 | { 0x00504500, 1}, | ||
1250 | { 0x00504508, 2}, | ||
1251 | { 0x00504608, 1}, | ||
1252 | { 0x00504680, 8}, | ||
1253 | { 0x005046a8, 5}, | ||
1254 | { 0x00504700, 8}, | ||
1255 | { 0x00504728, 5}, | ||
1256 | { 0x00504780, 8}, | ||
1257 | { 0x005047a8, 5}, | ||
1258 | { 0x00504854, 1}, | ||
1259 | { 0x00504ab0, 1}, | ||
1260 | { 0x00504b04, 1}, | ||
1261 | { 0x00504b3c, 1}, | ||
1262 | { 0x00504b48, 1}, | ||
1263 | { 0x00504b50, 1}, | ||
1264 | { 0x00504ba0, 2}, | ||
1265 | { 0x00504bb0, 1}, | ||
1266 | { 0x00504bdc, 1}, | ||
1267 | { 0x00504c0c, 1}, | ||
1268 | { 0x00504d00, 1}, | ||
1269 | { 0x00504d08, 2}, | ||
1270 | { 0x00504e08, 1}, | ||
1271 | { 0x00504e80, 8}, | ||
1272 | { 0x00504ea8, 5}, | ||
1273 | { 0x00504f00, 8}, | ||
1274 | { 0x00504f28, 5}, | ||
1275 | { 0x00504f80, 8}, | ||
1276 | { 0x00504fa8, 5}, | ||
1277 | { 0x00505054, 1}, | ||
1278 | { 0x005052b0, 1}, | ||
1279 | { 0x00505304, 1}, | ||
1280 | { 0x0050533c, 1}, | ||
1281 | { 0x00505348, 1}, | ||
1282 | { 0x00505350, 1}, | ||
1283 | { 0x005053a0, 2}, | ||
1284 | { 0x005053b0, 1}, | ||
1285 | { 0x005053dc, 1}, | ||
1286 | { 0x0050540c, 1}, | ||
1287 | { 0x00505500, 1}, | ||
1288 | { 0x00505508, 2}, | ||
1289 | { 0x00505608, 1}, | ||
1290 | { 0x00505680, 8}, | ||
1291 | { 0x005056a8, 5}, | ||
1292 | { 0x00505700, 8}, | ||
1293 | { 0x00505728, 5}, | ||
1294 | { 0x00505780, 8}, | ||
1295 | { 0x005057a8, 5}, | ||
1296 | { 0x00505854, 1}, | ||
1297 | { 0x00505ab0, 1}, | ||
1298 | { 0x00505b04, 1}, | ||
1299 | { 0x00505b3c, 1}, | ||
1300 | { 0x00505b48, 1}, | ||
1301 | { 0x00505b50, 1}, | ||
1302 | { 0x00505ba0, 2}, | ||
1303 | { 0x00505bb0, 1}, | ||
1304 | { 0x00505bdc, 1}, | ||
1305 | { 0x00505c0c, 1}, | ||
1306 | { 0x00505d00, 1}, | ||
1307 | { 0x00505d08, 2}, | ||
1308 | { 0x00505e08, 1}, | ||
1309 | { 0x00505e80, 8}, | ||
1310 | { 0x00505ea8, 5}, | ||
1311 | { 0x00505f00, 8}, | ||
1312 | { 0x00505f28, 5}, | ||
1313 | { 0x00505f80, 8}, | ||
1314 | { 0x00505fa8, 5}, | ||
1315 | { 0x00581a00, 19}, | ||
1316 | { 0x00581b00, 50}, | ||
1317 | { 0x00581e00, 50}, | ||
1318 | { 0x00581f00, 50}, | ||
1319 | { 0x00584200, 19}, | ||
1320 | { 0x00584300, 50}, | ||
1321 | { 0x00584600, 50}, | ||
1322 | { 0x00584700, 50}, | ||
1323 | { 0x00584a00, 19}, | ||
1324 | { 0x00584b00, 50}, | ||
1325 | { 0x00584e00, 50}, | ||
1326 | { 0x00584f00, 50}, | ||
1327 | { 0x00585200, 19}, | ||
1328 | { 0x00585300, 50}, | ||
1329 | { 0x00585600, 50}, | ||
1330 | { 0x00585700, 50}, | ||
1331 | { 0x00585a00, 19}, | ||
1332 | { 0x00585b00, 50}, | ||
1333 | { 0x00585e00, 50}, | ||
1334 | { 0x00585f00, 50}, | ||
1335 | { 0x00900100, 1}, | ||
1336 | { 0x009a0100, 1}, | ||
1337 | { 0x00a00160, 2}, | ||
1338 | { 0x00a007d0, 1}, | ||
1339 | { 0x00a04200, 1}, | ||
1340 | { 0x00a04470, 2}, | ||
1341 | { 0x00a08190, 1}, | ||
1342 | { 0x00a08198, 4}, | ||
1343 | { 0x00a0c820, 2}, | ||
1344 | { 0x00a0cc20, 2}, | ||
1345 | { 0x00a0e470, 2}, | ||
1346 | { 0x00a0e490, 9}, | ||
1347 | { 0x00a0e6a8, 7}, | ||
1348 | { 0x00a0e6c8, 2}, | ||
1349 | { 0x00a0e6d4, 7}, | ||
1350 | { 0x00a0e6f4, 2}, | ||
1351 | { 0x00a0ec70, 2}, | ||
1352 | { 0x00a0ec90, 9}, | ||
1353 | { 0x00a0eea8, 7}, | ||
1354 | { 0x00a0eec8, 2}, | ||
1355 | { 0x00a0eed4, 7}, | ||
1356 | { 0x00a0eef4, 2}, | ||
1357 | { 0x00a10190, 1}, | ||
1358 | { 0x00a10198, 4}, | ||
1359 | { 0x00a14820, 2}, | ||
1360 | { 0x00a14c20, 2}, | ||
1361 | { 0x00a16470, 2}, | ||
1362 | { 0x00a16490, 9}, | ||
1363 | { 0x00a166a8, 7}, | ||
1364 | { 0x00a166c8, 2}, | ||
1365 | { 0x00a166d4, 7}, | ||
1366 | { 0x00a166f4, 2}, | ||
1367 | { 0x00a16c70, 2}, | ||
1368 | { 0x00a16c90, 9}, | ||
1369 | { 0x00a16ea8, 7}, | ||
1370 | { 0x00a16ec8, 2}, | ||
1371 | { 0x00a16ed4, 7}, | ||
1372 | { 0x00a16ef4, 2}, | ||
1373 | { 0x00a18190, 1}, | ||
1374 | { 0x00a18198, 4}, | ||
1375 | { 0x00a1c820, 2}, | ||
1376 | { 0x00a1cc20, 2}, | ||
1377 | { 0x00a1e470, 2}, | ||
1378 | { 0x00a1e490, 9}, | ||
1379 | { 0x00a1e6a8, 7}, | ||
1380 | { 0x00a1e6c8, 2}, | ||
1381 | { 0x00a1e6d4, 7}, | ||
1382 | { 0x00a1e6f4, 2}, | ||
1383 | { 0x00a1ec70, 2}, | ||
1384 | { 0x00a1ec90, 9}, | ||
1385 | { 0x00a1eea8, 7}, | ||
1386 | { 0x00a1eec8, 2}, | ||
1387 | { 0x00a1eed4, 7}, | ||
1388 | { 0x00a1eef4, 2}, | ||
1389 | { 0x00a20190, 1}, | ||
1390 | { 0x00a20198, 4}, | ||
1391 | { 0x00a24820, 2}, | ||
1392 | { 0x00a24c20, 2}, | ||
1393 | { 0x00a26470, 2}, | ||
1394 | { 0x00a26490, 9}, | ||
1395 | { 0x00a266a8, 7}, | ||
1396 | { 0x00a266c8, 2}, | ||
1397 | { 0x00a266d4, 7}, | ||
1398 | { 0x00a266f4, 2}, | ||
1399 | { 0x00a26c70, 2}, | ||
1400 | { 0x00a26c90, 9}, | ||
1401 | { 0x00a26ea8, 7}, | ||
1402 | { 0x00a26ec8, 2}, | ||
1403 | { 0x00a26ed4, 7}, | ||
1404 | { 0x00a26ef4, 2}, | ||
1405 | { 0x00a28190, 1}, | ||
1406 | { 0x00a28198, 4}, | ||
1407 | { 0x00a2c820, 2}, | ||
1408 | { 0x00a2cc20, 2}, | ||
1409 | { 0x00a2e470, 2}, | ||
1410 | { 0x00a2e490, 9}, | ||
1411 | { 0x00a2e6a8, 7}, | ||
1412 | { 0x00a2e6c8, 2}, | ||
1413 | { 0x00a2e6d4, 7}, | ||
1414 | { 0x00a2e6f4, 2}, | ||
1415 | { 0x00a2ec70, 2}, | ||
1416 | { 0x00a2ec90, 9}, | ||
1417 | { 0x00a2eea8, 7}, | ||
1418 | { 0x00a2eec8, 2}, | ||
1419 | { 0x00a2eed4, 7}, | ||
1420 | { 0x00a2eef4, 2}, | ||
1421 | { 0x00a30190, 1}, | ||
1422 | { 0x00a30198, 4}, | ||
1423 | { 0x00a34820, 2}, | ||
1424 | { 0x00a34c20, 2}, | ||
1425 | { 0x00a36470, 2}, | ||
1426 | { 0x00a36490, 9}, | ||
1427 | { 0x00a366a8, 7}, | ||
1428 | { 0x00a366c8, 2}, | ||
1429 | { 0x00a366d4, 7}, | ||
1430 | { 0x00a366f4, 2}, | ||
1431 | { 0x00a36c70, 2}, | ||
1432 | { 0x00a36c90, 9}, | ||
1433 | { 0x00a36ea8, 7}, | ||
1434 | { 0x00a36ec8, 2}, | ||
1435 | { 0x00a36ed4, 7}, | ||
1436 | { 0x00a36ef4, 2}, | ||
1437 | { 0x00a38190, 1}, | ||
1438 | { 0x00a38198, 4}, | ||
1439 | { 0x00a3c820, 2}, | ||
1440 | { 0x00a3cc20, 2}, | ||
1441 | { 0x00a3e470, 2}, | ||
1442 | { 0x00a3e490, 9}, | ||
1443 | { 0x00a3e6a8, 7}, | ||
1444 | { 0x00a3e6c8, 2}, | ||
1445 | { 0x00a3e6d4, 7}, | ||
1446 | { 0x00a3e6f4, 2}, | ||
1447 | { 0x00a3ec70, 2}, | ||
1448 | { 0x00a3ec90, 9}, | ||
1449 | { 0x00a3eea8, 7}, | ||
1450 | { 0x00a3eec8, 2}, | ||
1451 | { 0x00a3eed4, 7}, | ||
1452 | { 0x00a3eef4, 2}, | ||
1453 | }; | ||
1454 | |||
1455 | |||
1456 | static const u32 gv11b_global_whitelist_ranges_count = | ||
1457 | ARRAY_SIZE(gv11b_global_whitelist_ranges); | ||
1458 | |||
1459 | /* context */ | ||
1460 | |||
1461 | /* runcontrol */ | ||
1462 | static const u32 gv11b_runcontrol_whitelist[] = { | ||
1463 | }; | ||
1464 | static const u32 gv11b_runcontrol_whitelist_count = | ||
1465 | ARRAY_SIZE(gv11b_runcontrol_whitelist); | ||
1466 | |||
1467 | static const struct regop_offset_range gv11b_runcontrol_whitelist_ranges[] = { | ||
1468 | }; | ||
1469 | static const u32 gv11b_runcontrol_whitelist_ranges_count = | ||
1470 | ARRAY_SIZE(gv11b_runcontrol_whitelist_ranges); | ||
1471 | |||
1472 | |||
1473 | /* quad ctl */ | ||
1474 | static const u32 gv11b_qctl_whitelist[] = { | ||
1475 | }; | ||
1476 | static const u32 gv11b_qctl_whitelist_count = | ||
1477 | ARRAY_SIZE(gv11b_qctl_whitelist); | ||
1478 | |||
1479 | static const struct regop_offset_range gv11b_qctl_whitelist_ranges[] = { | ||
1480 | }; | ||
1481 | static const u32 gv11b_qctl_whitelist_ranges_count = | ||
1482 | ARRAY_SIZE(gv11b_qctl_whitelist_ranges); | ||
1483 | |||
1484 | const struct regop_offset_range *gv11b_get_global_whitelist_ranges(void) | ||
1485 | { | ||
1486 | return gv11b_global_whitelist_ranges; | ||
1487 | } | ||
1488 | |||
1489 | int gv11b_get_global_whitelist_ranges_count(void) | ||
1490 | { | ||
1491 | return gv11b_global_whitelist_ranges_count; | ||
1492 | } | ||
1493 | |||
1494 | const struct regop_offset_range *gv11b_get_context_whitelist_ranges(void) | ||
1495 | { | ||
1496 | return gv11b_global_whitelist_ranges; | ||
1497 | } | ||
1498 | |||
1499 | int gv11b_get_context_whitelist_ranges_count(void) | ||
1500 | { | ||
1501 | return gv11b_global_whitelist_ranges_count; | ||
1502 | } | ||
1503 | |||
1504 | const u32 *gv11b_get_runcontrol_whitelist(void) | ||
1505 | { | ||
1506 | return gv11b_runcontrol_whitelist; | ||
1507 | } | ||
1508 | |||
1509 | int gv11b_get_runcontrol_whitelist_count(void) | ||
1510 | { | ||
1511 | return gv11b_runcontrol_whitelist_count; | ||
1512 | } | ||
1513 | |||
1514 | const struct regop_offset_range *gv11b_get_runcontrol_whitelist_ranges(void) | ||
1515 | { | ||
1516 | return gv11b_runcontrol_whitelist_ranges; | ||
1517 | } | ||
1518 | |||
1519 | int gv11b_get_runcontrol_whitelist_ranges_count(void) | ||
1520 | { | ||
1521 | return gv11b_runcontrol_whitelist_ranges_count; | ||
1522 | } | ||
1523 | |||
1524 | const u32 *gv11b_get_qctl_whitelist(void) | ||
1525 | { | ||
1526 | return gv11b_qctl_whitelist; | ||
1527 | } | ||
1528 | |||
1529 | int gv11b_get_qctl_whitelist_count(void) | ||
1530 | { | ||
1531 | return gv11b_qctl_whitelist_count; | ||
1532 | } | ||
1533 | |||
1534 | const struct regop_offset_range *gv11b_get_qctl_whitelist_ranges(void) | ||
1535 | { | ||
1536 | return gv11b_qctl_whitelist_ranges; | ||
1537 | } | ||
1538 | |||
1539 | int gv11b_get_qctl_whitelist_ranges_count(void) | ||
1540 | { | ||
1541 | return gv11b_qctl_whitelist_ranges_count; | ||
1542 | } | ||
1543 | |||
1544 | int gv11b_apply_smpc_war(struct dbg_session_gk20a *dbg_s) | ||
1545 | { | ||
1546 | /* Not needed on gv11b */ | ||
1547 | return 0; | ||
1548 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/regops_gv11b.h b/drivers/gpu/nvgpu/gv11b/regops_gv11b.h new file mode 100644 index 00000000..0ee2edfe --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/regops_gv11b.h | |||
@@ -0,0 +1,42 @@ | |||
1 | /* | ||
2 | * | ||
3 | * Tegra GV11B GPU Driver Register Ops | ||
4 | * | ||
5 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
6 | * | ||
7 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
8 | * copy of this software and associated documentation files (the "Software"), | ||
9 | * to deal in the Software without restriction, including without limitation | ||
10 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
11 | * and/or sell copies of the Software, and to permit persons to whom the | ||
12 | * Software is furnished to do so, subject to the following conditions: | ||
13 | * | ||
14 | * The above copyright notice and this permission notice shall be included in | ||
15 | * all copies or substantial portions of the Software. | ||
16 | * | ||
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
23 | * DEALINGS IN THE SOFTWARE. | ||
24 | */ | ||
25 | #ifndef __REGOPS_GV11B_H_ | ||
26 | #define __REGOPS_GV11B_H_ | ||
27 | |||
28 | const struct regop_offset_range *gv11b_get_global_whitelist_ranges(void); | ||
29 | int gv11b_get_global_whitelist_ranges_count(void); | ||
30 | const struct regop_offset_range *gv11b_get_context_whitelist_ranges(void); | ||
31 | int gv11b_get_context_whitelist_ranges_count(void); | ||
32 | const u32 *gv11b_get_runcontrol_whitelist(void); | ||
33 | int gv11b_get_runcontrol_whitelist_count(void); | ||
34 | const struct regop_offset_range *gv11b_get_runcontrol_whitelist_ranges(void); | ||
35 | int gv11b_get_runcontrol_whitelist_ranges_count(void); | ||
36 | const u32 *gv11b_get_qctl_whitelist(void); | ||
37 | int gv11b_get_qctl_whitelist_count(void); | ||
38 | const struct regop_offset_range *gv11b_get_qctl_whitelist_ranges(void); | ||
39 | int gv11b_get_qctl_whitelist_ranges_count(void); | ||
40 | int gv11b_apply_smpc_war(struct dbg_session_gk20a *dbg_s); | ||
41 | |||
42 | #endif /* __REGOPS_GV11B_H_ */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c new file mode 100644 index 00000000..fe1aa8a5 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.c | |||
@@ -0,0 +1,185 @@ | |||
1 | /* | ||
2 | * Volta GPU series Subcontext | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include "gk20a/gk20a.h" | ||
26 | |||
27 | #include "gv11b/subctx_gv11b.h" | ||
28 | |||
29 | #include <nvgpu/dma.h> | ||
30 | #include <nvgpu/log.h> | ||
31 | #include <nvgpu/gmmu.h> | ||
32 | |||
33 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
34 | #include <nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h> | ||
35 | |||
36 | static void gv11b_init_subcontext_pdb(struct channel_gk20a *c, | ||
37 | struct nvgpu_mem *inst_block); | ||
38 | |||
39 | static void gv11b_subctx_commit_valid_mask(struct channel_gk20a *c, | ||
40 | struct nvgpu_mem *inst_block); | ||
41 | static void gv11b_subctx_commit_pdb(struct channel_gk20a *c, | ||
42 | struct nvgpu_mem *inst_block); | ||
43 | |||
44 | void gv11b_free_subctx_header(struct channel_gk20a *c) | ||
45 | { | ||
46 | struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header; | ||
47 | struct gk20a *g = c->g; | ||
48 | |||
49 | nvgpu_log(g, gpu_dbg_fn, "gv11b_free_subctx_header"); | ||
50 | |||
51 | if (ctx->mem.gpu_va) { | ||
52 | nvgpu_gmmu_unmap(c->vm, &ctx->mem, ctx->mem.gpu_va); | ||
53 | |||
54 | nvgpu_dma_free(g, &ctx->mem); | ||
55 | } | ||
56 | } | ||
57 | |||
58 | int gv11b_alloc_subctx_header(struct channel_gk20a *c) | ||
59 | { | ||
60 | struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header; | ||
61 | struct gk20a *g = c->g; | ||
62 | int ret = 0; | ||
63 | |||
64 | nvgpu_log(g, gpu_dbg_fn, "gv11b_alloc_subctx_header"); | ||
65 | |||
66 | if (ctx->mem.gpu_va == 0) { | ||
67 | ret = nvgpu_dma_alloc_flags_sys(g, | ||
68 | 0, /* No Special flags */ | ||
69 | ctxsw_prog_fecs_header_v(), | ||
70 | &ctx->mem); | ||
71 | if (ret) { | ||
72 | nvgpu_err(g, "failed to allocate sub ctx header"); | ||
73 | return ret; | ||
74 | } | ||
75 | ctx->mem.gpu_va = nvgpu_gmmu_map(c->vm, | ||
76 | &ctx->mem, | ||
77 | ctx->mem.size, | ||
78 | 0, /* not GPU-cacheable */ | ||
79 | gk20a_mem_flag_none, true, | ||
80 | ctx->mem.aperture); | ||
81 | if (!ctx->mem.gpu_va) { | ||
82 | nvgpu_err(g, "failed to map ctx header"); | ||
83 | nvgpu_dma_free(g, &ctx->mem); | ||
84 | return -ENOMEM; | ||
85 | } | ||
86 | /* Now clear the buffer */ | ||
87 | if (nvgpu_mem_begin(g, &ctx->mem)) | ||
88 | return -ENOMEM; | ||
89 | |||
90 | nvgpu_memset(g, &ctx->mem, 0, 0, ctx->mem.size); | ||
91 | nvgpu_mem_end(g, &ctx->mem); | ||
92 | |||
93 | gv11b_init_subcontext_pdb(c, &c->inst_block); | ||
94 | } | ||
95 | return ret; | ||
96 | } | ||
97 | |||
98 | static void gv11b_init_subcontext_pdb(struct channel_gk20a *c, | ||
99 | struct nvgpu_mem *inst_block) | ||
100 | { | ||
101 | struct gk20a *g = c->g; | ||
102 | |||
103 | gv11b_subctx_commit_pdb(c, inst_block); | ||
104 | gv11b_subctx_commit_valid_mask(c, inst_block); | ||
105 | |||
106 | nvgpu_log(g, gpu_dbg_info, " subctx %d instblk set", c->t19x.subctx_id); | ||
107 | nvgpu_mem_wr32(g, inst_block, ram_in_engine_wfi_veid_w(), | ||
108 | ram_in_engine_wfi_veid_f(c->t19x.subctx_id)); | ||
109 | |||
110 | } | ||
111 | |||
112 | int gv11b_update_subctx_header(struct channel_gk20a *c, u64 gpu_va) | ||
113 | { | ||
114 | struct ctx_header_desc *ctx = &c->ch_ctx.ctx_header; | ||
115 | struct nvgpu_mem *gr_mem; | ||
116 | struct gk20a *g = c->g; | ||
117 | int ret = 0; | ||
118 | u32 addr_lo, addr_hi; | ||
119 | |||
120 | addr_lo = u64_lo32(gpu_va); | ||
121 | addr_hi = u64_hi32(gpu_va); | ||
122 | |||
123 | gr_mem = &ctx->mem; | ||
124 | g->ops.mm.l2_flush(g, true); | ||
125 | if (nvgpu_mem_begin(g, gr_mem)) | ||
126 | return -ENOMEM; | ||
127 | |||
128 | nvgpu_mem_wr(g, gr_mem, | ||
129 | ctxsw_prog_main_image_context_buffer_ptr_hi_o(), addr_hi); | ||
130 | nvgpu_mem_wr(g, gr_mem, | ||
131 | ctxsw_prog_main_image_context_buffer_ptr_o(), addr_lo); | ||
132 | |||
133 | nvgpu_mem_wr(g, gr_mem, | ||
134 | ctxsw_prog_main_image_ctl_o(), | ||
135 | ctxsw_prog_main_image_ctl_type_per_veid_header_v()); | ||
136 | nvgpu_mem_end(g, gr_mem); | ||
137 | return ret; | ||
138 | } | ||
139 | |||
140 | void gv11b_subctx_commit_valid_mask(struct channel_gk20a *c, | ||
141 | struct nvgpu_mem *inst_block) | ||
142 | { | ||
143 | struct gk20a *g = c->g; | ||
144 | |||
145 | /* Make all subctx pdbs valid */ | ||
146 | nvgpu_mem_wr32(g, inst_block, 166, 0xffffffff); | ||
147 | nvgpu_mem_wr32(g, inst_block, 167, 0xffffffff); | ||
148 | } | ||
149 | |||
150 | void gv11b_subctx_commit_pdb(struct channel_gk20a *c, | ||
151 | struct nvgpu_mem *inst_block) | ||
152 | { | ||
153 | struct gk20a *g = c->g; | ||
154 | struct fifo_gk20a *f = &g->fifo; | ||
155 | struct vm_gk20a *vm = c->vm; | ||
156 | u32 lo, hi; | ||
157 | u32 subctx_id = 0; | ||
158 | u32 format_word; | ||
159 | u32 pdb_addr_lo, pdb_addr_hi; | ||
160 | u64 pdb_addr; | ||
161 | u32 aperture = nvgpu_aperture_mask(g, vm->pdb.mem, | ||
162 | ram_in_sc_page_dir_base_target_sys_mem_ncoh_v(), | ||
163 | ram_in_sc_page_dir_base_target_vid_mem_v()); | ||
164 | |||
165 | pdb_addr = nvgpu_mem_get_addr(g, vm->pdb.mem); | ||
166 | pdb_addr_lo = u64_lo32(pdb_addr >> ram_in_base_shift_v()); | ||
167 | pdb_addr_hi = u64_hi32(pdb_addr); | ||
168 | format_word = ram_in_sc_page_dir_base_target_f( | ||
169 | aperture, 0) | | ||
170 | ram_in_sc_page_dir_base_vol_f( | ||
171 | ram_in_sc_page_dir_base_vol_true_v(), 0) | | ||
172 | ram_in_sc_page_dir_base_fault_replay_tex_f(1, 0) | | ||
173 | ram_in_sc_page_dir_base_fault_replay_gcc_f(1, 0) | | ||
174 | ram_in_sc_use_ver2_pt_format_f(1, 0) | | ||
175 | ram_in_sc_big_page_size_f(1, 0) | | ||
176 | ram_in_sc_page_dir_base_lo_0_f(pdb_addr_lo); | ||
177 | nvgpu_log(g, gpu_dbg_info, " pdb info lo %x hi %x", | ||
178 | format_word, pdb_addr_hi); | ||
179 | for (subctx_id = 0; subctx_id < f->t19x.max_subctx_count; subctx_id++) { | ||
180 | lo = ram_in_sc_page_dir_base_vol_0_w() + (4 * subctx_id); | ||
181 | hi = ram_in_sc_page_dir_base_hi_0_w() + (4 * subctx_id); | ||
182 | nvgpu_mem_wr32(g, inst_block, lo, format_word); | ||
183 | nvgpu_mem_wr32(g, inst_block, hi, pdb_addr_hi); | ||
184 | } | ||
185 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h new file mode 100644 index 00000000..10dc0ba5 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/subctx_gv11b.h | |||
@@ -0,0 +1,34 @@ | |||
1 | /* | ||
2 | * | ||
3 | * Volta GPU series Subcontext | ||
4 | * | ||
5 | * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved. | ||
6 | * | ||
7 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
8 | * copy of this software and associated documentation files (the "Software"), | ||
9 | * to deal in the Software without restriction, including without limitation | ||
10 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
11 | * and/or sell copies of the Software, and to permit persons to whom the | ||
12 | * Software is furnished to do so, subject to the following conditions: | ||
13 | * | ||
14 | * The above copyright notice and this permission notice shall be included in | ||
15 | * all copies or substantial portions of the Software. | ||
16 | * | ||
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
23 | * DEALINGS IN THE SOFTWARE. | ||
24 | */ | ||
25 | #ifndef __SUBCONTEXT_GV11B_H__ | ||
26 | #define __SUBCONTEXT_GV11B_H__ | ||
27 | |||
28 | int gv11b_alloc_subctx_header(struct channel_gk20a *c); | ||
29 | |||
30 | void gv11b_free_subctx_header(struct channel_gk20a *c); | ||
31 | |||
32 | int gv11b_update_subctx_header(struct channel_gk20a *c, u64 gpu_va); | ||
33 | |||
34 | #endif /* __SUBCONTEXT_GV11B_H__ */ | ||
diff --git a/drivers/gpu/nvgpu/gv11b/therm_gv11b.c b/drivers/gpu/nvgpu/gv11b/therm_gv11b.c new file mode 100644 index 00000000..18987119 --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/therm_gv11b.c | |||
@@ -0,0 +1,75 @@ | |||
1 | /* | ||
2 | * GV11B Therm | ||
3 | * | ||
4 | * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include "gk20a/gk20a.h" | ||
26 | |||
27 | #include <nvgpu/soc.h> | ||
28 | |||
29 | #include <nvgpu/hw/gv11b/hw_therm_gv11b.h> | ||
30 | |||
31 | int gv11b_elcg_init_idle_filters(struct gk20a *g) | ||
32 | { | ||
33 | u32 gate_ctrl, idle_filter; | ||
34 | u32 engine_id; | ||
35 | u32 active_engine_id = 0; | ||
36 | struct fifo_gk20a *f = &g->fifo; | ||
37 | |||
38 | if (nvgpu_platform_is_simulation(g)) | ||
39 | return 0; | ||
40 | |||
41 | gk20a_dbg_info("init clock/power gate reg"); | ||
42 | |||
43 | for (engine_id = 0; engine_id < f->num_engines; engine_id++) { | ||
44 | active_engine_id = f->active_engines_list[engine_id]; | ||
45 | |||
46 | gate_ctrl = gk20a_readl(g, therm_gate_ctrl_r(active_engine_id)); | ||
47 | gate_ctrl = set_field(gate_ctrl, | ||
48 | therm_gate_ctrl_eng_idle_filt_exp_m(), | ||
49 | therm_gate_ctrl_eng_idle_filt_exp__prod_f()); | ||
50 | gate_ctrl = set_field(gate_ctrl, | ||
51 | therm_gate_ctrl_eng_idle_filt_mant_m(), | ||
52 | therm_gate_ctrl_eng_idle_filt_mant__prod_f()); | ||
53 | gate_ctrl = set_field(gate_ctrl, | ||
54 | therm_gate_ctrl_eng_delay_before_m(), | ||
55 | therm_gate_ctrl_eng_delay_before__prod_f()); | ||
56 | gate_ctrl = set_field(gate_ctrl, | ||
57 | therm_gate_ctrl_eng_delay_after_m(), | ||
58 | therm_gate_ctrl_eng_delay_after__prod_f()); | ||
59 | gk20a_writel(g, therm_gate_ctrl_r(active_engine_id), gate_ctrl); | ||
60 | } | ||
61 | |||
62 | idle_filter = gk20a_readl(g, therm_fecs_idle_filter_r()); | ||
63 | idle_filter = set_field(idle_filter, | ||
64 | therm_fecs_idle_filter_value_m(), | ||
65 | therm_fecs_idle_filter_value__prod_f()); | ||
66 | gk20a_writel(g, therm_fecs_idle_filter_r(), idle_filter); | ||
67 | |||
68 | idle_filter = gk20a_readl(g, therm_hubmmu_idle_filter_r()); | ||
69 | idle_filter = set_field(idle_filter, | ||
70 | therm_hubmmu_idle_filter_value_m(), | ||
71 | therm_hubmmu_idle_filter_value__prod_f()); | ||
72 | gk20a_writel(g, therm_hubmmu_idle_filter_r(), idle_filter); | ||
73 | |||
74 | return 0; | ||
75 | } | ||
diff --git a/drivers/gpu/nvgpu/gv11b/therm_gv11b.h b/drivers/gpu/nvgpu/gv11b/therm_gv11b.h new file mode 100644 index 00000000..1d89597b --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/therm_gv11b.h | |||
@@ -0,0 +1,28 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | #ifndef THERM_GV11B_H | ||
23 | #define THERM_GV11B_H | ||
24 | |||
25 | struct gk20a; | ||
26 | int gv11b_elcg_init_idle_filters(struct gk20a *g); | ||
27 | |||
28 | #endif /* THERM_GV11B_H */ | ||