summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/gv11b/fb_gv11b.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/gv11b/fb_gv11b.c')
-rw-r--r--drivers/gpu/nvgpu/gv11b/fb_gv11b.c1555
1 files changed, 1555 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c
new file mode 100644
index 00000000..ec487bdf
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c
@@ -0,0 +1,1555 @@
1/*
2 * GV11B FB
3 *
4 * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26
27#include <nvgpu/dma.h>
28#include <nvgpu/log.h>
29#include <nvgpu/enabled.h>
30#include <nvgpu/gmmu.h>
31#include <nvgpu/barrier.h>
32#include <nvgpu/soc.h>
33
34#include "gk20a/gk20a.h"
35#include "gk20a/mm_gk20a.h"
36
37#include "gp10b/fb_gp10b.h"
38
39#include "gv11b/fifo_gv11b.h"
40#include "gv11b/fb_gv11b.h"
41#include "gv11b/ce_gv11b.h"
42
43#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
44#include <nvgpu/hw/gv11b/hw_mc_gv11b.h>
45#include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
46#include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h>
47
48static int gv11b_fb_fix_page_fault(struct gk20a *g,
49 struct mmu_fault_info *mmfault);
50
51static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
52 u32 invalidate_replay_val);
53
54static void gv11b_init_nvlink_soc_credits(struct gk20a *g)
55{
56 if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) {
57 nvgpu_info(g, "nvlink soc credits init done by bpmp");
58 } else {
59 /* MSS_NVLINK_1_BASE */
60 void __iomem *soc1 = ioremap(0x01f20010, 4096);
61 /* MSS_NVLINK_2_BASE */
62 void __iomem *soc2 = ioremap(0x01f40010, 4096);
63 /* MSS_NVLINK_3_BASE */
64 void __iomem *soc3 = ioremap(0x01f60010, 4096);
65 /* MSS_NVLINK_4_BASE */
66 void __iomem *soc4 = ioremap(0x01f80010, 4096);
67 u32 val;
68
69 nvgpu_info(g, "init nvlink soc credits");
70
71 val = readl_relaxed(soc1);
72 writel_relaxed(val, soc1);
73 val = readl_relaxed(soc1 + 4);
74 writel_relaxed(val, soc1 + 4);
75
76 val = readl_relaxed(soc2);
77 writel_relaxed(val, soc2);
78 val = readl_relaxed(soc2 + 4);
79 writel_relaxed(val, soc2 + 4);
80
81 val = readl_relaxed(soc3);
82 writel_relaxed(val, soc3);
83 val = readl_relaxed(soc3 + 4);
84 writel_relaxed(val, soc3 + 4);
85
86 val = readl_relaxed(soc4);
87 writel_relaxed(val, soc4);
88 val = readl_relaxed(soc4 + 4);
89 writel_relaxed(val, soc4 + 4);
90 }
91}
92
93void gv11b_fb_init_fs_state(struct gk20a *g)
94{
95 nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb");
96
97 nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x",
98 gk20a_readl(g, fb_fbhub_num_active_ltcs_r()));
99
100 nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u",
101 fb_mmu_num_active_ltcs_count_v(
102 gk20a_readl(g, fb_mmu_num_active_ltcs_r())));
103}
104
105void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr)
106{
107 u32 max_size = gr->max_comptag_mem;
108 /* one tag line covers 64KB */
109 u32 max_comptag_lines = max_size << 4;
110 u32 compbit_base_post_divide;
111 u64 compbit_base_post_multiply64;
112 u64 compbit_store_iova;
113 u64 compbit_base_post_divide64;
114
115 if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL))
116 compbit_store_iova = nvgpu_mem_get_phys_addr(g,
117 &gr->compbit_store.mem);
118 else
119 compbit_store_iova = nvgpu_mem_get_addr(g,
120 &gr->compbit_store.mem);
121
122 compbit_base_post_divide64 = compbit_store_iova >>
123 fb_mmu_cbc_base_address_alignment_shift_v();
124
125 do_div(compbit_base_post_divide64, g->ltc_count);
126 compbit_base_post_divide = u64_lo32(compbit_base_post_divide64);
127
128 compbit_base_post_multiply64 = ((u64)compbit_base_post_divide *
129 g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v();
130
131 if (compbit_base_post_multiply64 < compbit_store_iova)
132 compbit_base_post_divide++;
133
134 if (g->ops.ltc.cbc_fix_config)
135 compbit_base_post_divide =
136 g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide);
137
138 gk20a_writel(g, fb_mmu_cbc_base_r(),
139 fb_mmu_cbc_base_address_f(compbit_base_post_divide));
140
141 nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte,
142 "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n",
143 (u32)(compbit_store_iova >> 32),
144 (u32)(compbit_store_iova & 0xffffffff),
145 compbit_base_post_divide);
146 nvgpu_log(g, gpu_dbg_fn, "cbc base %x",
147 gk20a_readl(g, fb_mmu_cbc_base_r()));
148
149 gr->compbit_store.base_hw = compbit_base_post_divide;
150
151 g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate,
152 0, max_comptag_lines - 1);
153
154}
155
156void gv11b_fb_reset(struct gk20a *g)
157{
158 if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) {
159 nvgpu_log(g, gpu_dbg_info, "mc_elpg_enable set by bpmp");
160 } else {
161 u32 mc_elpg_enable_val;
162
163 nvgpu_log(g, gpu_dbg_info, "enable xbar, pfb and hub");
164 mc_elpg_enable_val = mc_elpg_enable_xbar_enabled_f() |
165 mc_elpg_enable_pfb_enabled_f() |
166 mc_elpg_enable_hub_enabled_f();
167 mc_elpg_enable_val |= gk20a_readl(g, mc_elpg_enable_r());
168 gk20a_writel(g, mc_elpg_enable_r(), mc_elpg_enable_val);
169
170 }
171 /* fs hub should be out of reset by now */
172 gv11b_init_nvlink_soc_credits(g);
173}
174
175static const char * const invalid_str = "invalid";
176
177static const char *const fault_type_descs_gv11b[] = {
178 "invalid pde",
179 "invalid pde size",
180 "invalid pte",
181 "limit violation",
182 "unbound inst block",
183 "priv violation",
184 "write",
185 "read",
186 "pitch mask violation",
187 "work creation",
188 "unsupported aperture",
189 "compression failure",
190 "unsupported kind",
191 "region violation",
192 "poison",
193 "atomic"
194};
195
196static const char *const fault_client_type_descs_gv11b[] = {
197 "gpc",
198 "hub",
199};
200
201static const char *const fault_access_type_descs_gv11b[] = {
202 "virt read",
203 "virt write",
204 "virt atomic strong",
205 "virt prefetch",
206 "virt atomic weak",
207 "xxx",
208 "xxx",
209 "xxx",
210 "phys read",
211 "phys write",
212 "phys atomic",
213 "phys prefetch",
214};
215
216static const char *const hub_client_descs_gv11b[] = {
217 "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu",
218 "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2",
219 "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc",
220 "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb",
221 "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1",
222 "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8",
223 "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3",
224 "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0",
225 "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp",
226 "dont care"
227};
228
229static const char *const gpc_client_descs_gv11b[] = {
230 "t1 0", "t1 1", "t1 2", "t1 3",
231 "t1 4", "t1 5", "t1 6", "t1 7",
232 "pe 0", "pe 1", "pe 2", "pe 3",
233 "pe 4", "pe 5", "pe 6", "pe 7",
234 "rast", "gcc", "gpccs",
235 "prop 0", "prop 1", "prop 2", "prop 3",
236 "gpm",
237 "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3",
238 "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7",
239 "utlb",
240 "t1 8", "t1 9", "t1 10", "t1 11",
241 "t1 12", "t1 13", "t1 14", "t1 15",
242 "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3",
243 "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7",
244 "pe 8", "pe 9", "tpccs 8", "tpccs 9",
245 "t1 16", "t1 17", "t1 18", "t1 19",
246 "pe 10", "pe 11", "tpccs 10", "tpccs 11",
247 "t1 20", "t1 21", "t1 22", "t1 23",
248 "pe 12", "pe 13", "tpccs 12", "tpccs 13",
249 "t1 24", "t1 25", "t1 26", "t1 27",
250 "pe 14", "pe 15", "tpccs 14", "tpccs 15",
251 "t1 28", "t1 29", "t1 30", "t1 31",
252 "pe 16", "pe 17", "tpccs 16", "tpccs 17",
253 "t1 32", "t1 33", "t1 34", "t1 35",
254 "pe 18", "pe 19", "tpccs 18", "tpccs 19",
255 "t1 36", "t1 37", "t1 38", "t1 39",
256};
257
258u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g,
259 unsigned int index)
260{
261 u32 reg_val;
262
263 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
264 return fb_mmu_fault_buffer_size_enable_v(reg_val);
265}
266
267static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g,
268 unsigned int index, u32 next)
269{
270 u32 reg_val;
271
272 nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next);
273
274 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
275 reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(),
276 fb_mmu_fault_buffer_get_ptr_f(next));
277
278 /* while the fault is being handled it is possible for overflow
279 * to happen,
280 */
281 if (reg_val & fb_mmu_fault_buffer_get_overflow_m())
282 reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f();
283
284 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
285
286 /* make sure get ptr update is visible to everyone to avoid
287 * reading already read entry
288 */
289 nvgpu_mb();
290}
291
292static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g,
293 unsigned int index)
294{
295 u32 reg_val;
296
297 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
298 return fb_mmu_fault_buffer_get_ptr_v(reg_val);
299}
300
301static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g,
302 unsigned int index)
303{
304 u32 reg_val;
305
306 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_put_r(index));
307 return fb_mmu_fault_buffer_put_ptr_v(reg_val);
308}
309
310static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g,
311 unsigned int index)
312{
313 u32 reg_val;
314
315 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
316 return fb_mmu_fault_buffer_size_val_v(reg_val);
317}
318
319static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g,
320 unsigned int index, u32 *get_idx)
321{
322 u32 put_idx;
323
324 *get_idx = gv11b_fb_fault_buffer_get_index(g, index);
325 put_idx = gv11b_fb_fault_buffer_put_index(g, index);
326
327 return *get_idx == put_idx;
328}
329
330static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g,
331 unsigned int index)
332{
333 u32 get_idx, put_idx, entries;
334
335
336 get_idx = gv11b_fb_fault_buffer_get_index(g, index);
337
338 put_idx = gv11b_fb_fault_buffer_put_index(g, index);
339
340 entries = gv11b_fb_fault_buffer_size_val(g, index);
341
342 return get_idx == ((put_idx + 1) % entries);
343}
344
345void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g,
346 unsigned int index, unsigned int state)
347{
348 u32 fault_status;
349 u32 reg_val;
350
351 nvgpu_log_fn(g, " ");
352
353 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index));
354 if (state) {
355 if (gv11b_fb_is_fault_buf_enabled(g, index)) {
356 nvgpu_log_info(g, "fault buffer is already enabled");
357 } else {
358 reg_val |= fb_mmu_fault_buffer_size_enable_true_f();
359 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index),
360 reg_val);
361 }
362
363 } else {
364 struct nvgpu_timeout timeout;
365 u32 delay = GR_IDLE_CHECK_DEFAULT;
366
367 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
368 NVGPU_TIMER_CPU_TIMER);
369
370 reg_val &= (~(fb_mmu_fault_buffer_size_enable_m()));
371 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val);
372
373 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
374
375 do {
376 if (!(fault_status & fb_mmu_fault_status_busy_true_f()))
377 break;
378 /*
379 * Make sure fault buffer is disabled.
380 * This is to avoid accessing fault buffer by hw
381 * during the window BAR2 is being unmapped by s/w
382 */
383 nvgpu_log_info(g, "fault status busy set, check again");
384 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
385
386 nvgpu_usleep_range(delay, delay * 2);
387 delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX);
388 } while (!nvgpu_timeout_expired_msg(&timeout,
389 "fault status busy set"));
390 }
391}
392
393void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index)
394{
395 u32 addr_lo;
396 u32 addr_hi;
397
398 nvgpu_log_fn(g, " ");
399
400 gv11b_fb_fault_buf_set_state_hw(g, index,
401 FAULT_BUF_DISABLED);
402 addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >>
403 ram_in_base_shift_v());
404 addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va);
405
406 gk20a_writel(g, fb_mmu_fault_buffer_lo_r(index),
407 fb_mmu_fault_buffer_lo_addr_f(addr_lo));
408
409 gk20a_writel(g, fb_mmu_fault_buffer_hi_r(index),
410 fb_mmu_fault_buffer_hi_addr_f(addr_hi));
411
412 gk20a_writel(g, fb_mmu_fault_buffer_size_r(index),
413 fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) |
414 fb_mmu_fault_buffer_size_overflow_intr_enable_f());
415
416 gv11b_fb_fault_buf_set_state_hw(g, index, FAULT_BUF_ENABLED);
417}
418
419static void gv11b_fb_intr_en_set(struct gk20a *g,
420 unsigned int index, u32 mask)
421{
422 u32 reg_val;
423
424 reg_val = gk20a_readl(g, fb_niso_intr_en_set_r(index));
425 reg_val |= mask;
426 gk20a_writel(g, fb_niso_intr_en_set_r(index), reg_val);
427}
428
429static void gv11b_fb_intr_en_clr(struct gk20a *g,
430 unsigned int index, u32 mask)
431{
432 u32 reg_val;
433
434 reg_val = gk20a_readl(g, fb_niso_intr_en_clr_r(index));
435 reg_val |= mask;
436 gk20a_writel(g, fb_niso_intr_en_clr_r(index), reg_val);
437}
438
439static u32 gv11b_fb_get_hub_intr_clr_mask(struct gk20a *g,
440 unsigned int intr_type)
441{
442 u32 mask = 0;
443
444 if (intr_type & HUB_INTR_TYPE_OTHER) {
445 mask |=
446 fb_niso_intr_en_clr_mmu_other_fault_notify_m();
447 }
448
449 if (intr_type & HUB_INTR_TYPE_NONREPLAY) {
450 mask |=
451 fb_niso_intr_en_clr_mmu_nonreplayable_fault_notify_m() |
452 fb_niso_intr_en_clr_mmu_nonreplayable_fault_overflow_m();
453 }
454
455 if (intr_type & HUB_INTR_TYPE_REPLAY) {
456 mask |=
457 fb_niso_intr_en_clr_mmu_replayable_fault_notify_m() |
458 fb_niso_intr_en_clr_mmu_replayable_fault_overflow_m();
459 }
460
461 if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) {
462 mask |=
463 fb_niso_intr_en_clr_mmu_ecc_uncorrected_error_notify_m();
464 }
465
466 if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) {
467 mask |=
468 fb_niso_intr_en_clr_hub_access_counter_notify_m() |
469 fb_niso_intr_en_clr_hub_access_counter_error_m();
470 }
471
472 return mask;
473}
474
475static u32 gv11b_fb_get_hub_intr_en_mask(struct gk20a *g,
476 unsigned int intr_type)
477{
478 u32 mask = 0;
479
480 if (intr_type & HUB_INTR_TYPE_OTHER) {
481 mask |=
482 fb_niso_intr_en_set_mmu_other_fault_notify_m();
483 }
484
485 if (intr_type & HUB_INTR_TYPE_NONREPLAY) {
486 mask |=
487 fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
488 fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m();
489 }
490
491 if (intr_type & HUB_INTR_TYPE_REPLAY) {
492 mask |=
493 fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
494 fb_niso_intr_en_set_mmu_replayable_fault_overflow_m();
495 }
496
497 if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) {
498 mask |=
499 fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m();
500 }
501
502 if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) {
503 mask |=
504 fb_niso_intr_en_set_hub_access_counter_notify_m() |
505 fb_niso_intr_en_set_hub_access_counter_error_m();
506 }
507
508 return mask;
509}
510
511void gv11b_fb_enable_hub_intr(struct gk20a *g,
512 unsigned int index, unsigned int intr_type)
513{
514 u32 mask = 0;
515
516 mask = gv11b_fb_get_hub_intr_en_mask(g, intr_type);
517
518 if (mask)
519 gv11b_fb_intr_en_set(g, index, mask);
520}
521
522void gv11b_fb_disable_hub_intr(struct gk20a *g,
523 unsigned int index, unsigned int intr_type)
524{
525 u32 mask = 0;
526
527 mask = gv11b_fb_get_hub_intr_clr_mask(g, intr_type);
528
529 if (mask)
530 gv11b_fb_intr_en_clr(g, index, mask);
531}
532
533static void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status)
534{
535 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
536 u32 corrected_delta, uncorrected_delta;
537 u32 corrected_overflow, uncorrected_overflow;
538
539 ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r());
540 corrected_cnt = gk20a_readl(g,
541 fb_mmu_l2tlb_ecc_corrected_err_count_r());
542 uncorrected_cnt = gk20a_readl(g,
543 fb_mmu_l2tlb_ecc_uncorrected_err_count_r());
544
545 corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v(
546 corrected_cnt);
547 uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v(
548 uncorrected_cnt);
549 corrected_overflow = ecc_status &
550 fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m();
551
552 uncorrected_overflow = ecc_status &
553 fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m();
554
555 /* clear the interrupt */
556 if ((corrected_delta > 0) || corrected_overflow)
557 gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0);
558 if ((uncorrected_delta > 0) || uncorrected_overflow)
559 gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0);
560
561 gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(),
562 fb_mmu_l2tlb_ecc_status_reset_clear_f());
563
564 /* Handle overflow */
565 if (corrected_overflow)
566 corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s());
567 if (uncorrected_overflow)
568 uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s());
569
570
571 g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0] +=
572 corrected_delta;
573 g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0] +=
574 uncorrected_delta;
575
576 if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m())
577 nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
578 if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m())
579 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
580 if (corrected_overflow || uncorrected_overflow)
581 nvgpu_info(g, "mmu l2tlb ecc counter overflow!");
582
583 nvgpu_log(g, gpu_dbg_intr,
584 "ecc error address: 0x%x", ecc_addr);
585 nvgpu_log(g, gpu_dbg_intr,
586 "ecc error count corrected: %d, uncorrected %d",
587 g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0],
588 g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0]);
589}
590
591static void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status)
592{
593 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
594 u32 corrected_delta, uncorrected_delta;
595 u32 corrected_overflow, uncorrected_overflow;
596
597 ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r());
598 corrected_cnt = gk20a_readl(g,
599 fb_mmu_hubtlb_ecc_corrected_err_count_r());
600 uncorrected_cnt = gk20a_readl(g,
601 fb_mmu_hubtlb_ecc_uncorrected_err_count_r());
602
603 corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v(
604 corrected_cnt);
605 uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v(
606 uncorrected_cnt);
607 corrected_overflow = ecc_status &
608 fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m();
609
610 uncorrected_overflow = ecc_status &
611 fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m();
612
613 /* clear the interrupt */
614 if ((corrected_delta > 0) || corrected_overflow)
615 gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0);
616 if ((uncorrected_delta > 0) || uncorrected_overflow)
617 gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0);
618
619 gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(),
620 fb_mmu_hubtlb_ecc_status_reset_clear_f());
621
622 /* Handle overflow */
623 if (corrected_overflow)
624 corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s());
625 if (uncorrected_overflow)
626 uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s());
627
628
629 g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0] +=
630 corrected_delta;
631 g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0] +=
632 uncorrected_delta;
633
634 if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m())
635 nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
636 if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m())
637 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
638 if (corrected_overflow || uncorrected_overflow)
639 nvgpu_info(g, "mmu hubtlb ecc counter overflow!");
640
641 nvgpu_log(g, gpu_dbg_intr,
642 "ecc error address: 0x%x", ecc_addr);
643 nvgpu_log(g, gpu_dbg_intr,
644 "ecc error count corrected: %d, uncorrected %d",
645 g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0],
646 g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0]);
647}
648
649static void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status)
650{
651 u32 ecc_addr, corrected_cnt, uncorrected_cnt;
652 u32 corrected_delta, uncorrected_delta;
653 u32 corrected_overflow, uncorrected_overflow;
654
655 ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r());
656 corrected_cnt = gk20a_readl(g,
657 fb_mmu_fillunit_ecc_corrected_err_count_r());
658 uncorrected_cnt = gk20a_readl(g,
659 fb_mmu_fillunit_ecc_uncorrected_err_count_r());
660
661 corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v(
662 corrected_cnt);
663 uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v(
664 uncorrected_cnt);
665 corrected_overflow = ecc_status &
666 fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m();
667
668 uncorrected_overflow = ecc_status &
669 fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m();
670
671 /* clear the interrupt */
672 if ((corrected_delta > 0) || corrected_overflow)
673 gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0);
674 if ((uncorrected_delta > 0) || uncorrected_overflow)
675 gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0);
676
677 gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(),
678 fb_mmu_fillunit_ecc_status_reset_clear_f());
679
680 /* Handle overflow */
681 if (corrected_overflow)
682 corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s());
683 if (uncorrected_overflow)
684 uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s());
685
686
687 g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0] +=
688 corrected_delta;
689 g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0] +=
690 uncorrected_delta;
691
692 if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m())
693 nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error");
694 if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m())
695 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error");
696 if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m())
697 nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error");
698 if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m())
699 nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error");
700
701 if (corrected_overflow || uncorrected_overflow)
702 nvgpu_info(g, "mmu fillunit ecc counter overflow!");
703
704 nvgpu_log(g, gpu_dbg_intr,
705 "ecc error address: 0x%x", ecc_addr);
706 nvgpu_log(g, gpu_dbg_intr,
707 "ecc error count corrected: %d, uncorrected %d",
708 g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0],
709 g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0]);
710}
711
712static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault)
713{
714 if (WARN_ON(mmfault->fault_type >=
715 ARRAY_SIZE(fault_type_descs_gv11b)))
716 mmfault->fault_type_desc = invalid_str;
717 else
718 mmfault->fault_type_desc =
719 fault_type_descs_gv11b[mmfault->fault_type];
720
721 if (WARN_ON(mmfault->client_type >=
722 ARRAY_SIZE(fault_client_type_descs_gv11b)))
723 mmfault->client_type_desc = invalid_str;
724 else
725 mmfault->client_type_desc =
726 fault_client_type_descs_gv11b[mmfault->client_type];
727
728 mmfault->client_id_desc = invalid_str;
729 if (mmfault->client_type ==
730 gmmu_fault_client_type_hub_v()) {
731
732 if (!(WARN_ON(mmfault->client_id >=
733 ARRAY_SIZE(hub_client_descs_gv11b))))
734 mmfault->client_id_desc =
735 hub_client_descs_gv11b[mmfault->client_id];
736 } else if (mmfault->client_type ==
737 gmmu_fault_client_type_gpc_v()) {
738 if (!(WARN_ON(mmfault->client_id >=
739 ARRAY_SIZE(gpc_client_descs_gv11b))))
740 mmfault->client_id_desc =
741 gpc_client_descs_gv11b[mmfault->client_id];
742 }
743
744}
745
746static void gv11b_fb_print_fault_info(struct gk20a *g,
747 struct mmu_fault_info *mmfault)
748{
749 if (mmfault && mmfault->valid) {
750 nvgpu_err(g, "[MMU FAULT] "
751 "mmu engine id: %d, "
752 "ch id: %d, "
753 "fault addr: 0x%llx, "
754 "fault addr aperture: %d, "
755 "fault type: %s, "
756 "access type: %s, ",
757 mmfault->mmu_engine_id,
758 mmfault->chid,
759 mmfault->fault_addr,
760 mmfault->fault_addr_aperture,
761 mmfault->fault_type_desc,
762 fault_access_type_descs_gv11b[mmfault->access_type]);
763 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
764 "mmu engine id: %d, "
765 "faulted act eng id if any: 0x%x, "
766 "faulted veid if any: 0x%x, "
767 "faulted pbdma id if any: 0x%x, "
768 "fault addr: 0x%llx, ",
769 mmfault->mmu_engine_id,
770 mmfault->faulted_engine,
771 mmfault->faulted_subid,
772 mmfault->faulted_pbdma,
773 mmfault->fault_addr);
774 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
775 "fault addr aperture: %d, "
776 "fault type: %s, "
777 "access type: %s, "
778 "inst ptr: 0x%llx, "
779 "inst ptr aperture: %d, ",
780 mmfault->fault_addr_aperture,
781 mmfault->fault_type_desc,
782 fault_access_type_descs_gv11b[mmfault->access_type],
783 mmfault->inst_ptr,
784 mmfault->inst_aperture);
785 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
786 "ch id: %d, "
787 "timestamp hi:lo 0x%08x:0x%08x, "
788 "client type: %s, "
789 "client id: %s, "
790 "gpc id if client type is gpc: %d, ",
791 mmfault->chid,
792 mmfault->timestamp_hi, mmfault->timestamp_lo,
793 mmfault->client_type_desc,
794 mmfault->client_id_desc,
795 mmfault->gpc_id);
796 nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
797 "protected mode: %d, "
798 "replayable fault: %d, "
799 "replayable fault en: %d ",
800 mmfault->protected_mode,
801 mmfault->replayable_fault,
802 mmfault->replay_fault_en);
803 }
804}
805
806/*
807 *Fault buffer format
808 *
809 * 31 28 24 23 16 15 8 7 4 0
810 *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-.
811 *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0|
812 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
813 *| inst_hi |
814 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
815 *| addr_31_12 | |AP |
816 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
817 *| addr_63_32 |
818 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
819 *| timestamp_lo |
820 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
821 *| timestamp_hi |
822 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
823 *| (reserved) | engine_id |
824 *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-'
825 *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype|
826 */
827
828static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g,
829 struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault)
830{
831 u32 rd32_val;
832 u32 addr_lo, addr_hi;
833 u64 inst_ptr;
834 u32 chid = FIFO_INVAL_CHANNEL_ID;
835 struct channel_gk20a *refch;
836
837 memset(mmfault, 0, sizeof(*mmfault));
838
839 rd32_val = nvgpu_mem_rd32(g, mem, offset +
840 gmmu_fault_buf_entry_inst_lo_w());
841 addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val);
842 addr_lo = addr_lo << ram_in_base_shift_v();
843
844 addr_hi = nvgpu_mem_rd32(g, mem, offset +
845 gmmu_fault_buf_entry_inst_hi_w());
846 addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi);
847
848 inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo);
849
850 /* refch will be put back after fault is handled */
851 refch = gk20a_refch_from_inst_ptr(g, inst_ptr);
852 if (refch)
853 chid = refch->chid;
854
855 /* it is ok to continue even if refch is NULL */
856 mmfault->refch = refch;
857 mmfault->chid = chid;
858 mmfault->inst_ptr = inst_ptr;
859 mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val);
860
861 rd32_val = nvgpu_mem_rd32(g, mem, offset +
862 gmmu_fault_buf_entry_addr_lo_w());
863
864 mmfault->fault_addr_aperture =
865 gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val);
866 addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val);
867 addr_lo = addr_lo << ram_in_base_shift_v();
868
869 rd32_val = nvgpu_mem_rd32(g, mem, offset +
870 gmmu_fault_buf_entry_addr_hi_w());
871 addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val);
872 mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo);
873
874 rd32_val = nvgpu_mem_rd32(g, mem, offset +
875 gmmu_fault_buf_entry_timestamp_lo_w());
876 mmfault->timestamp_lo =
877 gmmu_fault_buf_entry_timestamp_lo_v(rd32_val);
878
879 rd32_val = nvgpu_mem_rd32(g, mem, offset +
880 gmmu_fault_buf_entry_timestamp_hi_w());
881 mmfault->timestamp_hi =
882 gmmu_fault_buf_entry_timestamp_hi_v(rd32_val);
883
884 rd32_val = nvgpu_mem_rd32(g, mem, offset +
885 gmmu_fault_buf_entry_engine_id_w());
886
887 mmfault->mmu_engine_id =
888 gmmu_fault_buf_entry_engine_id_v(rd32_val);
889 gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id,
890 &mmfault->faulted_engine, &mmfault->faulted_subid,
891 &mmfault->faulted_pbdma);
892
893 rd32_val = nvgpu_mem_rd32(g, mem, offset +
894 gmmu_fault_buf_entry_fault_type_w());
895 mmfault->client_id =
896 gmmu_fault_buf_entry_client_v(rd32_val);
897 mmfault->replayable_fault =
898 gmmu_fault_buf_entry_replayable_fault_v(rd32_val);
899
900 mmfault->fault_type =
901 gmmu_fault_buf_entry_fault_type_v(rd32_val);
902 mmfault->access_type =
903 gmmu_fault_buf_entry_access_type_v(rd32_val);
904
905 mmfault->client_type =
906 gmmu_fault_buf_entry_mmu_client_type_v(rd32_val);
907
908 mmfault->gpc_id =
909 gmmu_fault_buf_entry_gpc_id_v(rd32_val);
910 mmfault->protected_mode =
911 gmmu_fault_buf_entry_protected_mode_v(rd32_val);
912
913 mmfault->replay_fault_en =
914 gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val);
915
916 mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val);
917
918 rd32_val = nvgpu_mem_rd32(g, mem, offset +
919 gmmu_fault_buf_entry_fault_type_w());
920 rd32_val &= ~(gmmu_fault_buf_entry_valid_m());
921 nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(),
922 rd32_val);
923
924 gv11b_fb_parse_mmfault(mmfault);
925}
926
927static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g,
928 struct mmu_fault_info *mmfault, u32 *invalidate_replay_val)
929{
930 unsigned int id_type;
931 u32 num_lce, act_eng_bitmask = 0;
932 int err = 0;
933 u32 id = ((u32)~0);
934
935 if (!mmfault->valid)
936 return;
937
938 gv11b_fb_print_fault_info(g, mmfault);
939
940 num_lce = gv11b_ce_get_num_lce(g);
941 if ((mmfault->mmu_engine_id >=
942 gmmu_fault_mmu_eng_id_ce0_v()) &&
943 (mmfault->mmu_engine_id <
944 gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) {
945 /* CE page faults are not reported as replayable */
946 nvgpu_log(g, gpu_dbg_intr, "CE Faulted");
947 err = gv11b_fb_fix_page_fault(g, mmfault);
948 gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch,
949 mmfault->faulted_pbdma, mmfault->faulted_engine);
950 if (!err) {
951 nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed");
952 *invalidate_replay_val = 0;
953 /* refch in mmfault is assigned at the time of copying
954 * fault info from snap reg or bar2 fault buf
955 */
956 gk20a_channel_put(mmfault->refch);
957 return;
958 }
959 /* Do recovery. Channel recovery needs refch */
960 nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed");
961 }
962
963 if (!mmfault->replayable_fault) {
964 if (mmfault->fault_type ==
965 gmmu_fault_type_unbound_inst_block_v()) {
966 /*
967 * Bug 1847172: When an engine faults due to an unbound
968 * instance block, the fault cannot be isolated to a
969 * single context so we need to reset the entire runlist
970 */
971 id_type = ID_TYPE_UNKNOWN;
972 nvgpu_log(g, gpu_dbg_intr, "UNBOUND INST BLOCK MMU FAULT");
973
974 } else if (mmfault->refch) {
975 if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) {
976 id = mmfault->refch->tsgid;
977 id_type = ID_TYPE_TSG;
978 } else {
979 id = mmfault->chid;
980 id_type = ID_TYPE_CHANNEL;
981 }
982 } else {
983 id_type = ID_TYPE_UNKNOWN;
984 }
985 if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID)
986 act_eng_bitmask = BIT(mmfault->faulted_engine);
987
988 g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask,
989 id, id_type, RC_TYPE_MMU_FAULT, mmfault);
990 } else {
991 if (mmfault->fault_type == gmmu_fault_type_pte_v()) {
992 nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix");
993 err = gv11b_fb_fix_page_fault(g, mmfault);
994 if (err)
995 *invalidate_replay_val |=
996 fb_mmu_invalidate_replay_cancel_global_f();
997 else
998 *invalidate_replay_val |=
999 fb_mmu_invalidate_replay_start_ack_all_f();
1000 } else {
1001 /* cancel faults other than invalid pte */
1002 *invalidate_replay_val |=
1003 fb_mmu_invalidate_replay_cancel_global_f();
1004 }
1005 /* refch in mmfault is assigned at the time of copying
1006 * fault info from snap reg or bar2 fault buf
1007 */
1008 gk20a_channel_put(mmfault->refch);
1009 }
1010}
1011
1012static void gv11b_fb_replay_or_cancel_faults(struct gk20a *g,
1013 u32 invalidate_replay_val)
1014{
1015 int err = 0;
1016
1017 nvgpu_log_fn(g, " ");
1018
1019 if (invalidate_replay_val &
1020 fb_mmu_invalidate_replay_cancel_global_f()) {
1021 /*
1022 * cancel faults so that next time it faults as
1023 * replayable faults and channel recovery can be done
1024 */
1025 err = gv11b_fb_mmu_invalidate_replay(g,
1026 fb_mmu_invalidate_replay_cancel_global_f());
1027 } else if (invalidate_replay_val &
1028 fb_mmu_invalidate_replay_start_ack_all_f()) {
1029 /* pte valid is fixed. replay faulting request */
1030 err = gv11b_fb_mmu_invalidate_replay(g,
1031 fb_mmu_invalidate_replay_start_ack_all_f());
1032 }
1033}
1034
1035static void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g,
1036 u32 fault_status, unsigned int index)
1037{
1038 u32 get_indx, offset, rd32_val, entries;
1039 struct nvgpu_mem *mem;
1040 struct mmu_fault_info *mmfault;
1041 u32 invalidate_replay_val = 0;
1042 u64 prev_fault_addr = 0ULL;
1043 u64 next_fault_addr = 0ULL;
1044
1045 if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) {
1046 nvgpu_log(g, gpu_dbg_intr,
1047 "SPURIOUS mmu fault: reg index:%d", index);
1048 return;
1049 }
1050 nvgpu_info(g, "%s MMU FAULT" ,
1051 index == REPLAY_REG_INDEX ? "REPLAY" : "NON-REPLAY");
1052
1053 nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx);
1054
1055 mem = &g->mm.hw_fault_buf[index];
1056 mmfault = g->mm.fault_info[index];
1057
1058 entries = gv11b_fb_fault_buffer_size_val(g, index);
1059 nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries);
1060
1061 offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32);
1062 nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset);
1063
1064 rd32_val = nvgpu_mem_rd32(g, mem,
1065 offset + gmmu_fault_buf_entry_valid_w());
1066 nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val);
1067
1068 while ((rd32_val & gmmu_fault_buf_entry_valid_m())) {
1069
1070 nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val);
1071
1072 gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault);
1073
1074 get_indx = (get_indx + 1) % entries;
1075 nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx);
1076
1077 gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx);
1078
1079 offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32);
1080 nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset);
1081
1082 rd32_val = nvgpu_mem_rd32(g, mem,
1083 offset + gmmu_fault_buf_entry_valid_w());
1084
1085 if (index == REPLAY_REG_INDEX && mmfault->fault_addr != 0ULL) {
1086 /* fault_addr "0" is not supposed to be fixed ever.
1087 * For the first time when prev = 0, next = 0 and
1088 * fault addr is also 0 then handle_mmu_fault_common will
1089 * not be called. Fix by checking fault_addr not equal to 0
1090 */
1091 prev_fault_addr = next_fault_addr;
1092 next_fault_addr = mmfault->fault_addr;
1093 if (prev_fault_addr == next_fault_addr) {
1094 nvgpu_log(g, gpu_dbg_intr, "pte is fixed");
1095 if (mmfault->refch)
1096 gk20a_channel_put(mmfault->refch);
1097 /* pte already fixed for this addr */
1098 continue;
1099 }
1100 }
1101
1102 gv11b_fb_handle_mmu_fault_common(g, mmfault,
1103 &invalidate_replay_val);
1104
1105 }
1106 if (index == REPLAY_REG_INDEX && invalidate_replay_val)
1107 gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val);
1108}
1109
1110static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g,
1111 u32 fault_status, struct mmu_fault_info *mmfault)
1112{
1113 u32 reg_val;
1114 u32 addr_lo, addr_hi;
1115 u64 inst_ptr;
1116 int chid = FIFO_INVAL_CHANNEL_ID;
1117 struct channel_gk20a *refch;
1118
1119 memset(mmfault, 0, sizeof(*mmfault));
1120
1121 if (!(fault_status & fb_mmu_fault_status_valid_set_f())) {
1122
1123 nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set");
1124 return;
1125 }
1126
1127 reg_val = gk20a_readl(g, fb_mmu_fault_inst_lo_r());
1128 addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val);
1129 addr_lo = addr_lo << ram_in_base_shift_v();
1130
1131 addr_hi = gk20a_readl(g, fb_mmu_fault_inst_hi_r());
1132 addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi);
1133 inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo);
1134
1135 /* refch will be put back after fault is handled */
1136 refch = gk20a_refch_from_inst_ptr(g, inst_ptr);
1137 if (refch)
1138 chid = refch->chid;
1139
1140 /* It is still ok to continue if refch is NULL */
1141 mmfault->refch = refch;
1142 mmfault->chid = chid;
1143 mmfault->inst_ptr = inst_ptr;
1144 mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val);
1145 mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val);
1146
1147 gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id,
1148 &mmfault->faulted_engine, &mmfault->faulted_subid,
1149 &mmfault->faulted_pbdma);
1150
1151 reg_val = gk20a_readl(g, fb_mmu_fault_addr_lo_r());
1152 addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val);
1153 addr_lo = addr_lo << ram_in_base_shift_v();
1154
1155 mmfault->fault_addr_aperture =
1156 fb_mmu_fault_addr_lo_phys_aperture_v(reg_val);
1157
1158 addr_hi = gk20a_readl(g, fb_mmu_fault_addr_hi_r());
1159 addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi);
1160 mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo);
1161
1162 reg_val = gk20a_readl(g, fb_mmu_fault_info_r());
1163 mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val);
1164 mmfault->replayable_fault =
1165 fb_mmu_fault_info_replayable_fault_v(reg_val);
1166 mmfault->client_id = fb_mmu_fault_info_client_v(reg_val);
1167 mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val);
1168 mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val);
1169 mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val);
1170 mmfault->protected_mode =
1171 fb_mmu_fault_info_protected_mode_v(reg_val);
1172 mmfault->replay_fault_en =
1173 fb_mmu_fault_info_replayable_fault_en_v(reg_val);
1174
1175 mmfault->valid = fb_mmu_fault_info_valid_v(reg_val);
1176
1177 fault_status &= ~(fb_mmu_fault_status_valid_m());
1178 gk20a_writel(g, fb_mmu_fault_status_r(), fault_status);
1179
1180 gv11b_fb_parse_mmfault(mmfault);
1181
1182}
1183
1184static void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g,
1185 u32 fault_status)
1186{
1187 u32 reg_val;
1188 unsigned int index = REPLAY_REG_INDEX;
1189
1190 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
1191
1192 if (fault_status &
1193 fb_mmu_fault_status_replayable_getptr_corrupted_m()) {
1194
1195 nvgpu_err(g, "replayable getptr corrupted set");
1196
1197 gv11b_fb_fault_buf_configure_hw(g, index);
1198
1199 reg_val = set_field(reg_val,
1200 fb_mmu_fault_buffer_get_getptr_corrupted_m(),
1201 fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
1202 }
1203
1204 if (fault_status &
1205 fb_mmu_fault_status_replayable_overflow_m()) {
1206 bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
1207
1208 nvgpu_err(g, "replayable overflow: buffer full:%s",
1209 buffer_full?"true":"false");
1210
1211 reg_val = set_field(reg_val,
1212 fb_mmu_fault_buffer_get_overflow_m(),
1213 fb_mmu_fault_buffer_get_overflow_clear_f());
1214 }
1215
1216 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
1217}
1218
1219static void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g,
1220 u32 fault_status)
1221{
1222 u32 reg_val;
1223 unsigned int index = NONREPLAY_REG_INDEX;
1224
1225 reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index));
1226
1227 if (fault_status &
1228 fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) {
1229
1230 nvgpu_err(g, "non replayable getptr corrupted set");
1231
1232 gv11b_fb_fault_buf_configure_hw(g, index);
1233
1234 reg_val = set_field(reg_val,
1235 fb_mmu_fault_buffer_get_getptr_corrupted_m(),
1236 fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
1237 }
1238
1239 if (fault_status &
1240 fb_mmu_fault_status_non_replayable_overflow_m()) {
1241
1242 bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
1243
1244 nvgpu_err(g, "non replayable overflow: buffer full:%s",
1245 buffer_full?"true":"false");
1246
1247 reg_val = set_field(reg_val,
1248 fb_mmu_fault_buffer_get_overflow_m(),
1249 fb_mmu_fault_buffer_get_overflow_clear_f());
1250 }
1251
1252 gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
1253}
1254
1255static void gv11b_fb_handle_bar2_fault(struct gk20a *g,
1256 struct mmu_fault_info *mmfault, u32 fault_status)
1257{
1258 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX,
1259 HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY);
1260
1261
1262 if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) {
1263 if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX))
1264 gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX);
1265 }
1266
1267 if (fault_status & fb_mmu_fault_status_replayable_error_m()) {
1268 if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX))
1269 gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX);
1270 }
1271 gv11b_ce_mthd_buffer_fault_in_bar2_fault(g);
1272
1273 g->ops.mm.init_bar2_mm_hw_setup(g);
1274
1275 if (mmfault->refch) {
1276 gk20a_channel_put(mmfault->refch);
1277 mmfault->refch = NULL;
1278 }
1279 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX,
1280 HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY);
1281}
1282
1283static void gv11b_fb_handle_other_fault_notify(struct gk20a *g,
1284 u32 fault_status)
1285{
1286 struct mmu_fault_info *mmfault;
1287 u32 invalidate_replay_val = 0;
1288
1289 mmfault = g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY];
1290
1291 gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault);
1292
1293 /* BAR2/Physical faults will not be snapped in hw fault buf */
1294 if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) {
1295 nvgpu_err(g, "BAR2 MMU FAULT");
1296 gv11b_fb_handle_bar2_fault(g, mmfault, fault_status);
1297
1298 } else if (mmfault->mmu_engine_id ==
1299 gmmu_fault_mmu_eng_id_physical_v()) {
1300 /* usually means VPR or out of bounds physical accesses */
1301 nvgpu_err(g, "PHYSICAL MMU FAULT");
1302
1303 } else {
1304 gv11b_fb_handle_mmu_fault_common(g, mmfault,
1305 &invalidate_replay_val);
1306
1307 if (invalidate_replay_val)
1308 gv11b_fb_replay_or_cancel_faults(g,
1309 invalidate_replay_val);
1310 }
1311}
1312
1313static void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status)
1314{
1315 u32 dropped_faults = 0;
1316
1317 dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() |
1318 fb_mmu_fault_status_dropped_bar1_virt_set_f() |
1319 fb_mmu_fault_status_dropped_bar2_phys_set_f() |
1320 fb_mmu_fault_status_dropped_bar2_virt_set_f() |
1321 fb_mmu_fault_status_dropped_ifb_phys_set_f() |
1322 fb_mmu_fault_status_dropped_ifb_virt_set_f() |
1323 fb_mmu_fault_status_dropped_other_phys_set_f()|
1324 fb_mmu_fault_status_dropped_other_virt_set_f();
1325
1326 if (fault_status & dropped_faults) {
1327 nvgpu_err(g, "dropped mmu fault (0x%08x)",
1328 fault_status & dropped_faults);
1329 gk20a_writel(g, fb_mmu_fault_status_r(), dropped_faults);
1330 }
1331}
1332
1333
1334static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr)
1335{
1336 u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r());
1337
1338 nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status);
1339
1340 if (niso_intr &
1341 fb_niso_intr_mmu_other_fault_notify_m()) {
1342
1343 gv11b_fb_handle_dropped_mmu_fault(g, fault_status);
1344
1345 gv11b_fb_handle_other_fault_notify(g, fault_status);
1346 }
1347
1348 if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) {
1349
1350 if (niso_intr &
1351 fb_niso_intr_mmu_nonreplayable_fault_notify_m()) {
1352
1353 gv11b_fb_handle_mmu_nonreplay_replay_fault(g,
1354 fault_status, NONREPLAY_REG_INDEX);
1355
1356 /*
1357 * When all the faults are processed,
1358 * GET and PUT will have same value and mmu fault status
1359 * bit will be reset by HW
1360 */
1361 }
1362 if (niso_intr &
1363 fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) {
1364
1365 gv11b_fb_handle_nonreplay_fault_overflow(g,
1366 fault_status);
1367 }
1368
1369 }
1370
1371 if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) {
1372
1373 if (niso_intr &
1374 fb_niso_intr_mmu_replayable_fault_notify_m()) {
1375
1376 gv11b_fb_handle_mmu_nonreplay_replay_fault(g,
1377 fault_status, REPLAY_REG_INDEX);
1378 }
1379 if (niso_intr &
1380 fb_niso_intr_mmu_replayable_fault_overflow_m()) {
1381
1382 gv11b_fb_handle_replay_fault_overflow(g,
1383 fault_status);
1384 }
1385
1386 }
1387
1388 nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status");
1389 gk20a_writel(g, fb_mmu_fault_status_r(),
1390 fb_mmu_fault_status_valid_clear_f());
1391}
1392
1393void gv11b_fb_hub_isr(struct gk20a *g)
1394{
1395 u32 status, niso_intr;
1396
1397 nvgpu_mutex_acquire(&g->mm.hub_isr_mutex);
1398
1399 niso_intr = gk20a_readl(g, fb_niso_intr_r());
1400
1401 nvgpu_info(g, "enter hub isr, niso_intr = 0x%08x", niso_intr);
1402
1403 if (niso_intr &
1404 (fb_niso_intr_hub_access_counter_notify_m() |
1405 fb_niso_intr_hub_access_counter_error_m())) {
1406
1407 nvgpu_info(g, "hub access counter notify/error");
1408 }
1409 if (niso_intr &
1410 fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) {
1411
1412 nvgpu_info(g, "ecc uncorrected error notify");
1413
1414 /* disable interrupts during handling */
1415 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX,
1416 HUB_INTR_TYPE_ECC_UNCORRECTED);
1417
1418 status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r());
1419 if (status)
1420 gv11b_handle_l2tlb_ecc_isr(g, status);
1421
1422 status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r());
1423 if (status)
1424 gv11b_handle_hubtlb_ecc_isr(g, status);
1425
1426 status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r());
1427 if (status)
1428 gv11b_handle_fillunit_ecc_isr(g, status);
1429
1430 /* re-enable interrupts after handling */
1431 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX,
1432 HUB_INTR_TYPE_ECC_UNCORRECTED);
1433
1434 }
1435 if (niso_intr &
1436 (fb_niso_intr_mmu_other_fault_notify_m() |
1437 fb_niso_intr_mmu_replayable_fault_notify_m() |
1438 fb_niso_intr_mmu_replayable_fault_overflow_m() |
1439 fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
1440 fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) {
1441
1442 nvgpu_info(g, "MMU Fault");
1443 gv11b_fb_handle_mmu_fault(g, niso_intr);
1444 }
1445
1446 nvgpu_mutex_release(&g->mm.hub_isr_mutex);
1447}
1448
1449bool gv11b_fb_mmu_fault_pending(struct gk20a *g)
1450{
1451 if (gk20a_readl(g, fb_niso_intr_r()) &
1452 (fb_niso_intr_mmu_other_fault_notify_m() |
1453 fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() |
1454 fb_niso_intr_mmu_replayable_fault_notify_m() |
1455 fb_niso_intr_mmu_replayable_fault_overflow_m() |
1456 fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
1457 fb_niso_intr_mmu_nonreplayable_fault_overflow_m()))
1458 return true;
1459
1460 return false;
1461}
1462
1463static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
1464 u32 invalidate_replay_val)
1465{
1466 int err = -ETIMEDOUT;
1467 u32 reg_val;
1468 struct nvgpu_timeout timeout;
1469
1470 gk20a_dbg_fn("");
1471
1472 nvgpu_mutex_acquire(&g->mm.tlb_lock);
1473
1474 reg_val = gk20a_readl(g, fb_mmu_invalidate_r());
1475
1476 reg_val |= fb_mmu_invalidate_all_va_true_f() |
1477 fb_mmu_invalidate_all_pdb_true_f() |
1478 invalidate_replay_val |
1479 fb_mmu_invalidate_trigger_true_f();
1480
1481 gk20a_writel(g, fb_mmu_invalidate_r(), reg_val);
1482
1483 /* retry 200 times */
1484 nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER);
1485 do {
1486 reg_val = gk20a_readl(g, fb_mmu_ctrl_r());
1487 if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) !=
1488 fb_mmu_ctrl_pri_fifo_empty_false_f()) {
1489 err = 0;
1490 break;
1491 }
1492 nvgpu_udelay(5);
1493 } while (!nvgpu_timeout_expired_msg(&timeout,
1494 "invalidate replay failed on 0x%llx"));
1495 if (err)
1496 nvgpu_err(g, "invalidate replay timedout");
1497
1498 nvgpu_mutex_release(&g->mm.tlb_lock);
1499
1500 return err;
1501}
1502
1503static int gv11b_fb_fix_page_fault(struct gk20a *g,
1504 struct mmu_fault_info *mmfault)
1505{
1506 int err = 0;
1507 u32 pte[2];
1508
1509 if (mmfault->refch == NULL) {
1510 nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL");
1511 return -EINVAL;
1512 }
1513
1514 err = __nvgpu_get_pte(g,
1515 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1516 if (err) {
1517 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found");
1518 return err;
1519 }
1520 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1521 "pte: %#08x %#08x", pte[1], pte[0]);
1522
1523 if (pte[0] == 0x0 && pte[1] == 0x0) {
1524 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1525 "pte all zeros, do not set valid");
1526 return -1;
1527 }
1528 if (pte[0] & gmmu_new_pte_valid_true_f()) {
1529 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1530 "pte valid already set");
1531 return -1;
1532 }
1533
1534 pte[0] |= gmmu_new_pte_valid_true_f();
1535 if (pte[0] & gmmu_new_pte_read_only_true_f())
1536 pte[0] &= ~(gmmu_new_pte_read_only_true_f());
1537 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1538 "new pte: %#08x %#08x", pte[1], pte[0]);
1539
1540 err = __nvgpu_set_pte(g,
1541 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1542 if (err) {
1543 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed");
1544 return err;
1545 }
1546 /* invalidate tlb so that GMMU does not use old cached translation */
1547 g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem);
1548
1549 err = __nvgpu_get_pte(g,
1550 mmfault->refch->vm, mmfault->fault_addr, &pte[0]);
1551 nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte,
1552 "pte after tlb invalidate: %#08x %#08x",
1553 pte[1], pte[0]);
1554 return err;
1555}
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-15 23:31:50 -0400