diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/fb/fb_gv11b.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/fb/fb_gv11b.c | 1516 |
1 files changed, 1516 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/fb/fb_gv11b.c b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c new file mode 100644 index 00000000..69a71575 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fb/fb_gv11b.c | |||
@@ -0,0 +1,1516 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <nvgpu/dma.h> | ||
26 | #include <nvgpu/log.h> | ||
27 | #include <nvgpu/enabled.h> | ||
28 | #include <nvgpu/gmmu.h> | ||
29 | #include <nvgpu/barrier.h> | ||
30 | #include <nvgpu/bug.h> | ||
31 | #include <nvgpu/soc.h> | ||
32 | |||
33 | #include "gk20a/gk20a.h" | ||
34 | #include "gk20a/mm_gk20a.h" | ||
35 | |||
36 | #include "gv11b/fifo_gv11b.h" | ||
37 | #include "gv11b/ce_gv11b.h" | ||
38 | |||
39 | #include "fb_gk20a.h" | ||
40 | #include "fb_gp10b.h" | ||
41 | #include "fb_gv11b.h" | ||
42 | |||
43 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
44 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
45 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
46 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
47 | |||
48 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
49 | struct mmu_fault_info *mmfault); | ||
50 | |||
51 | static void gv11b_init_nvlink_soc_credits(struct gk20a *g) | ||
52 | { | ||
53 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
54 | nvgpu_log(g, gpu_dbg_info, "nvlink soc credits init done by bpmp"); | ||
55 | } else { | ||
56 | #ifndef __NVGPU_POSIX__ | ||
57 | nvgpu_mss_nvlink_init_credits(g); | ||
58 | #endif | ||
59 | } | ||
60 | } | ||
61 | |||
62 | void gv11b_fb_init_hw(struct gk20a *g) | ||
63 | { | ||
64 | gk20a_fb_init_hw(g); | ||
65 | |||
66 | g->ops.fb.enable_hub_intr(g); | ||
67 | } | ||
68 | |||
69 | void gv11b_fb_init_fs_state(struct gk20a *g) | ||
70 | { | ||
71 | nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); | ||
72 | |||
73 | nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", | ||
74 | gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); | ||
75 | |||
76 | nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", | ||
77 | fb_mmu_num_active_ltcs_count_v( | ||
78 | gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); | ||
79 | |||
80 | if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) { | ||
81 | /* Bypass MMU check for non-secure boot. For | ||
82 | * secure-boot,this register write has no-effect */ | ||
83 | gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU); | ||
84 | } | ||
85 | } | ||
86 | |||
87 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) | ||
88 | { | ||
89 | u32 max_size = gr->max_comptag_mem; | ||
90 | /* one tag line covers 64KB */ | ||
91 | u32 max_comptag_lines = max_size << 4; | ||
92 | u32 compbit_base_post_divide; | ||
93 | u64 compbit_base_post_multiply64; | ||
94 | u64 compbit_store_iova; | ||
95 | u64 compbit_base_post_divide64; | ||
96 | |||
97 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) | ||
98 | compbit_store_iova = nvgpu_mem_get_phys_addr(g, | ||
99 | &gr->compbit_store.mem); | ||
100 | else | ||
101 | compbit_store_iova = nvgpu_mem_get_addr(g, | ||
102 | &gr->compbit_store.mem); | ||
103 | |||
104 | compbit_base_post_divide64 = compbit_store_iova >> | ||
105 | fb_mmu_cbc_base_address_alignment_shift_v(); | ||
106 | |||
107 | do_div(compbit_base_post_divide64, g->ltc_count); | ||
108 | compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); | ||
109 | |||
110 | compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * | ||
111 | g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); | ||
112 | |||
113 | if (compbit_base_post_multiply64 < compbit_store_iova) | ||
114 | compbit_base_post_divide++; | ||
115 | |||
116 | if (g->ops.ltc.cbc_fix_config) | ||
117 | compbit_base_post_divide = | ||
118 | g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); | ||
119 | |||
120 | gk20a_writel(g, fb_mmu_cbc_base_r(), | ||
121 | fb_mmu_cbc_base_address_f(compbit_base_post_divide)); | ||
122 | |||
123 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, | ||
124 | "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", | ||
125 | (u32)(compbit_store_iova >> 32), | ||
126 | (u32)(compbit_store_iova & 0xffffffff), | ||
127 | compbit_base_post_divide); | ||
128 | nvgpu_log(g, gpu_dbg_fn, "cbc base %x", | ||
129 | gk20a_readl(g, fb_mmu_cbc_base_r())); | ||
130 | |||
131 | gr->compbit_store.base_hw = compbit_base_post_divide; | ||
132 | |||
133 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, | ||
134 | 0, max_comptag_lines - 1); | ||
135 | |||
136 | } | ||
137 | |||
138 | void gv11b_fb_reset(struct gk20a *g) | ||
139 | { | ||
140 | gv11b_init_nvlink_soc_credits(g); | ||
141 | } | ||
142 | |||
143 | static const char * const invalid_str = "invalid"; | ||
144 | |||
145 | static const char *const fault_type_descs_gv11b[] = { | ||
146 | "invalid pde", | ||
147 | "invalid pde size", | ||
148 | "invalid pte", | ||
149 | "limit violation", | ||
150 | "unbound inst block", | ||
151 | "priv violation", | ||
152 | "write", | ||
153 | "read", | ||
154 | "pitch mask violation", | ||
155 | "work creation", | ||
156 | "unsupported aperture", | ||
157 | "compression failure", | ||
158 | "unsupported kind", | ||
159 | "region violation", | ||
160 | "poison", | ||
161 | "atomic" | ||
162 | }; | ||
163 | |||
164 | static const char *const fault_client_type_descs_gv11b[] = { | ||
165 | "gpc", | ||
166 | "hub", | ||
167 | }; | ||
168 | |||
169 | static const char *const fault_access_type_descs_gv11b[] = { | ||
170 | "virt read", | ||
171 | "virt write", | ||
172 | "virt atomic strong", | ||
173 | "virt prefetch", | ||
174 | "virt atomic weak", | ||
175 | "xxx", | ||
176 | "xxx", | ||
177 | "xxx", | ||
178 | "phys read", | ||
179 | "phys write", | ||
180 | "phys atomic", | ||
181 | "phys prefetch", | ||
182 | }; | ||
183 | |||
184 | static const char *const hub_client_descs_gv11b[] = { | ||
185 | "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", | ||
186 | "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", | ||
187 | "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", | ||
188 | "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", | ||
189 | "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", | ||
190 | "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", | ||
191 | "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", | ||
192 | "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", | ||
193 | "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", | ||
194 | "dont care" | ||
195 | }; | ||
196 | |||
197 | static const char *const gpc_client_descs_gv11b[] = { | ||
198 | "t1 0", "t1 1", "t1 2", "t1 3", | ||
199 | "t1 4", "t1 5", "t1 6", "t1 7", | ||
200 | "pe 0", "pe 1", "pe 2", "pe 3", | ||
201 | "pe 4", "pe 5", "pe 6", "pe 7", | ||
202 | "rast", "gcc", "gpccs", | ||
203 | "prop 0", "prop 1", "prop 2", "prop 3", | ||
204 | "gpm", | ||
205 | "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", | ||
206 | "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", | ||
207 | "utlb", | ||
208 | "t1 8", "t1 9", "t1 10", "t1 11", | ||
209 | "t1 12", "t1 13", "t1 14", "t1 15", | ||
210 | "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", | ||
211 | "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", | ||
212 | "pe 8", "pe 9", "tpccs 8", "tpccs 9", | ||
213 | "t1 16", "t1 17", "t1 18", "t1 19", | ||
214 | "pe 10", "pe 11", "tpccs 10", "tpccs 11", | ||
215 | "t1 20", "t1 21", "t1 22", "t1 23", | ||
216 | "pe 12", "pe 13", "tpccs 12", "tpccs 13", | ||
217 | "t1 24", "t1 25", "t1 26", "t1 27", | ||
218 | "pe 14", "pe 15", "tpccs 14", "tpccs 15", | ||
219 | "t1 28", "t1 29", "t1 30", "t1 31", | ||
220 | "pe 16", "pe 17", "tpccs 16", "tpccs 17", | ||
221 | "t1 32", "t1 33", "t1 34", "t1 35", | ||
222 | "pe 18", "pe 19", "tpccs 18", "tpccs 19", | ||
223 | "t1 36", "t1 37", "t1 38", "t1 39", | ||
224 | }; | ||
225 | |||
226 | bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index) | ||
227 | { | ||
228 | u32 reg_val; | ||
229 | |||
230 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
231 | return fb_mmu_fault_buffer_size_enable_v(reg_val) != 0U; | ||
232 | } | ||
233 | |||
234 | static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, | ||
235 | u32 index, u32 next) | ||
236 | { | ||
237 | u32 reg_val; | ||
238 | |||
239 | nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); | ||
240 | |||
241 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
242 | reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), | ||
243 | fb_mmu_fault_buffer_get_ptr_f(next)); | ||
244 | |||
245 | /* while the fault is being handled it is possible for overflow | ||
246 | * to happen, | ||
247 | */ | ||
248 | if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) | ||
249 | reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); | ||
250 | |||
251 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
252 | |||
253 | /* make sure get ptr update is visible to everyone to avoid | ||
254 | * reading already read entry | ||
255 | */ | ||
256 | nvgpu_mb(); | ||
257 | } | ||
258 | |||
259 | static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, u32 index) | ||
260 | { | ||
261 | u32 reg_val; | ||
262 | |||
263 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
264 | return fb_mmu_fault_buffer_get_ptr_v(reg_val); | ||
265 | } | ||
266 | |||
267 | static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, u32 index) | ||
268 | { | ||
269 | u32 reg_val; | ||
270 | |||
271 | reg_val = g->ops.fb.read_mmu_fault_buffer_put(g, index); | ||
272 | return fb_mmu_fault_buffer_put_ptr_v(reg_val); | ||
273 | } | ||
274 | |||
275 | static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index) | ||
276 | { | ||
277 | u32 reg_val; | ||
278 | |||
279 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
280 | return fb_mmu_fault_buffer_size_val_v(reg_val); | ||
281 | } | ||
282 | |||
283 | static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, | ||
284 | u32 index, u32 *get_idx) | ||
285 | { | ||
286 | u32 put_idx; | ||
287 | |||
288 | *get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
289 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
290 | |||
291 | return *get_idx == put_idx; | ||
292 | } | ||
293 | |||
294 | static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, u32 index) | ||
295 | { | ||
296 | u32 get_idx, put_idx, entries; | ||
297 | |||
298 | |||
299 | get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
300 | |||
301 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
302 | |||
303 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
304 | |||
305 | return get_idx == ((put_idx + 1) % entries); | ||
306 | } | ||
307 | |||
308 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
309 | u32 index, u32 state) | ||
310 | { | ||
311 | u32 fault_status; | ||
312 | u32 reg_val; | ||
313 | |||
314 | nvgpu_log_fn(g, " "); | ||
315 | |||
316 | reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index); | ||
317 | if (state == NVGPU_FB_MMU_FAULT_BUF_ENABLED) { | ||
318 | if (gv11b_fb_is_fault_buf_enabled(g, index)) { | ||
319 | nvgpu_log_info(g, "fault buffer is already enabled"); | ||
320 | } else { | ||
321 | reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); | ||
322 | g->ops.fb.write_mmu_fault_buffer_size(g, index, | ||
323 | reg_val); | ||
324 | } | ||
325 | |||
326 | } else { | ||
327 | struct nvgpu_timeout timeout; | ||
328 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
329 | |||
330 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
331 | NVGPU_TIMER_CPU_TIMER); | ||
332 | |||
333 | reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); | ||
334 | g->ops.fb.write_mmu_fault_buffer_size(g, index, reg_val); | ||
335 | |||
336 | fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
337 | |||
338 | do { | ||
339 | if (!(fault_status & fb_mmu_fault_status_busy_true_f())) | ||
340 | break; | ||
341 | /* | ||
342 | * Make sure fault buffer is disabled. | ||
343 | * This is to avoid accessing fault buffer by hw | ||
344 | * during the window BAR2 is being unmapped by s/w | ||
345 | */ | ||
346 | nvgpu_log_info(g, "fault status busy set, check again"); | ||
347 | fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
348 | |||
349 | nvgpu_usleep_range(delay, delay * 2); | ||
350 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
351 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
352 | "fault status busy set")); | ||
353 | } | ||
354 | } | ||
355 | |||
356 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index) | ||
357 | { | ||
358 | u32 addr_lo; | ||
359 | u32 addr_hi; | ||
360 | |||
361 | nvgpu_log_fn(g, " "); | ||
362 | |||
363 | gv11b_fb_fault_buf_set_state_hw(g, index, | ||
364 | NVGPU_FB_MMU_FAULT_BUF_DISABLED); | ||
365 | addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> | ||
366 | ram_in_base_shift_v()); | ||
367 | addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); | ||
368 | |||
369 | g->ops.fb.write_mmu_fault_buffer_lo_hi(g, index, | ||
370 | fb_mmu_fault_buffer_lo_addr_f(addr_lo), | ||
371 | fb_mmu_fault_buffer_hi_addr_f(addr_hi)); | ||
372 | |||
373 | g->ops.fb.write_mmu_fault_buffer_size(g, index, | ||
374 | fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | | ||
375 | fb_mmu_fault_buffer_size_overflow_intr_enable_f()); | ||
376 | |||
377 | gv11b_fb_fault_buf_set_state_hw(g, index, NVGPU_FB_MMU_FAULT_BUF_ENABLED); | ||
378 | } | ||
379 | |||
380 | void gv11b_fb_enable_hub_intr(struct gk20a *g) | ||
381 | { | ||
382 | u32 mask = 0; | ||
383 | |||
384 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
385 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
386 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
387 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
388 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | | ||
389 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
390 | |||
391 | gk20a_writel(g, fb_niso_intr_en_set_r(0), | ||
392 | mask); | ||
393 | } | ||
394 | |||
395 | void gv11b_fb_disable_hub_intr(struct gk20a *g) | ||
396 | { | ||
397 | u32 mask = 0; | ||
398 | |||
399 | mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() | | ||
400 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
401 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() | | ||
402 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
403 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() | | ||
404 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
405 | |||
406 | gk20a_writel(g, fb_niso_intr_en_clr_r(0), | ||
407 | mask); | ||
408 | } | ||
409 | |||
410 | void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
411 | { | ||
412 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
413 | u32 corrected_delta, uncorrected_delta; | ||
414 | u32 corrected_overflow, uncorrected_overflow; | ||
415 | |||
416 | ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); | ||
417 | corrected_cnt = gk20a_readl(g, | ||
418 | fb_mmu_l2tlb_ecc_corrected_err_count_r()); | ||
419 | uncorrected_cnt = gk20a_readl(g, | ||
420 | fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); | ||
421 | |||
422 | corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( | ||
423 | corrected_cnt); | ||
424 | uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( | ||
425 | uncorrected_cnt); | ||
426 | corrected_overflow = ecc_status & | ||
427 | fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
428 | |||
429 | uncorrected_overflow = ecc_status & | ||
430 | fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
431 | |||
432 | /* clear the interrupt */ | ||
433 | if ((corrected_delta > 0) || corrected_overflow) | ||
434 | gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); | ||
435 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
436 | gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); | ||
437 | |||
438 | gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), | ||
439 | fb_mmu_l2tlb_ecc_status_reset_clear_f()); | ||
440 | |||
441 | /* Handle overflow */ | ||
442 | if (corrected_overflow) | ||
443 | corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); | ||
444 | if (uncorrected_overflow) | ||
445 | uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); | ||
446 | |||
447 | |||
448 | g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0] += | ||
449 | corrected_delta; | ||
450 | g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0] += | ||
451 | uncorrected_delta; | ||
452 | |||
453 | if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) | ||
454 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
455 | if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) | ||
456 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
457 | if (corrected_overflow || uncorrected_overflow) | ||
458 | nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); | ||
459 | |||
460 | nvgpu_log(g, gpu_dbg_intr, | ||
461 | "ecc error address: 0x%x", ecc_addr); | ||
462 | nvgpu_log(g, gpu_dbg_intr, | ||
463 | "ecc error count corrected: %d, uncorrected %d", | ||
464 | g->ecc.fb.mmu_l2tlb_corrected_err_count.counters[0], | ||
465 | g->ecc.fb.mmu_l2tlb_uncorrected_err_count.counters[0]); | ||
466 | } | ||
467 | |||
468 | void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
469 | { | ||
470 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
471 | u32 corrected_delta, uncorrected_delta; | ||
472 | u32 corrected_overflow, uncorrected_overflow; | ||
473 | |||
474 | ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); | ||
475 | corrected_cnt = gk20a_readl(g, | ||
476 | fb_mmu_hubtlb_ecc_corrected_err_count_r()); | ||
477 | uncorrected_cnt = gk20a_readl(g, | ||
478 | fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); | ||
479 | |||
480 | corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( | ||
481 | corrected_cnt); | ||
482 | uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( | ||
483 | uncorrected_cnt); | ||
484 | corrected_overflow = ecc_status & | ||
485 | fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
486 | |||
487 | uncorrected_overflow = ecc_status & | ||
488 | fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
489 | |||
490 | /* clear the interrupt */ | ||
491 | if ((corrected_delta > 0) || corrected_overflow) | ||
492 | gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); | ||
493 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
494 | gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); | ||
495 | |||
496 | gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), | ||
497 | fb_mmu_hubtlb_ecc_status_reset_clear_f()); | ||
498 | |||
499 | /* Handle overflow */ | ||
500 | if (corrected_overflow) | ||
501 | corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); | ||
502 | if (uncorrected_overflow) | ||
503 | uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); | ||
504 | |||
505 | |||
506 | g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0] += | ||
507 | corrected_delta; | ||
508 | g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0] += | ||
509 | uncorrected_delta; | ||
510 | |||
511 | if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) | ||
512 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
513 | if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) | ||
514 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
515 | if (corrected_overflow || uncorrected_overflow) | ||
516 | nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); | ||
517 | |||
518 | nvgpu_log(g, gpu_dbg_intr, | ||
519 | "ecc error address: 0x%x", ecc_addr); | ||
520 | nvgpu_log(g, gpu_dbg_intr, | ||
521 | "ecc error count corrected: %d, uncorrected %d", | ||
522 | g->ecc.fb.mmu_hubtlb_corrected_err_count.counters[0], | ||
523 | g->ecc.fb.mmu_hubtlb_uncorrected_err_count.counters[0]); | ||
524 | } | ||
525 | |||
526 | void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
527 | { | ||
528 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
529 | u32 corrected_delta, uncorrected_delta; | ||
530 | u32 corrected_overflow, uncorrected_overflow; | ||
531 | |||
532 | ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); | ||
533 | corrected_cnt = gk20a_readl(g, | ||
534 | fb_mmu_fillunit_ecc_corrected_err_count_r()); | ||
535 | uncorrected_cnt = gk20a_readl(g, | ||
536 | fb_mmu_fillunit_ecc_uncorrected_err_count_r()); | ||
537 | |||
538 | corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( | ||
539 | corrected_cnt); | ||
540 | uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( | ||
541 | uncorrected_cnt); | ||
542 | corrected_overflow = ecc_status & | ||
543 | fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); | ||
544 | |||
545 | uncorrected_overflow = ecc_status & | ||
546 | fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
547 | |||
548 | /* clear the interrupt */ | ||
549 | if ((corrected_delta > 0) || corrected_overflow) | ||
550 | gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); | ||
551 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
552 | gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); | ||
553 | |||
554 | gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), | ||
555 | fb_mmu_fillunit_ecc_status_reset_clear_f()); | ||
556 | |||
557 | /* Handle overflow */ | ||
558 | if (corrected_overflow) | ||
559 | corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); | ||
560 | if (uncorrected_overflow) | ||
561 | uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); | ||
562 | |||
563 | |||
564 | g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0] += | ||
565 | corrected_delta; | ||
566 | g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0] += | ||
567 | uncorrected_delta; | ||
568 | |||
569 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) | ||
570 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); | ||
571 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) | ||
572 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); | ||
573 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) | ||
574 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); | ||
575 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) | ||
576 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); | ||
577 | |||
578 | if (corrected_overflow || uncorrected_overflow) | ||
579 | nvgpu_info(g, "mmu fillunit ecc counter overflow!"); | ||
580 | |||
581 | nvgpu_log(g, gpu_dbg_intr, | ||
582 | "ecc error address: 0x%x", ecc_addr); | ||
583 | nvgpu_log(g, gpu_dbg_intr, | ||
584 | "ecc error count corrected: %d, uncorrected %d", | ||
585 | g->ecc.fb.mmu_fillunit_corrected_err_count.counters[0], | ||
586 | g->ecc.fb.mmu_fillunit_uncorrected_err_count.counters[0]); | ||
587 | } | ||
588 | |||
589 | static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) | ||
590 | { | ||
591 | if (WARN_ON(mmfault->fault_type >= | ||
592 | ARRAY_SIZE(fault_type_descs_gv11b))) | ||
593 | mmfault->fault_type_desc = invalid_str; | ||
594 | else | ||
595 | mmfault->fault_type_desc = | ||
596 | fault_type_descs_gv11b[mmfault->fault_type]; | ||
597 | |||
598 | if (WARN_ON(mmfault->client_type >= | ||
599 | ARRAY_SIZE(fault_client_type_descs_gv11b))) | ||
600 | mmfault->client_type_desc = invalid_str; | ||
601 | else | ||
602 | mmfault->client_type_desc = | ||
603 | fault_client_type_descs_gv11b[mmfault->client_type]; | ||
604 | |||
605 | mmfault->client_id_desc = invalid_str; | ||
606 | if (mmfault->client_type == | ||
607 | gmmu_fault_client_type_hub_v()) { | ||
608 | |||
609 | if (!(WARN_ON(mmfault->client_id >= | ||
610 | ARRAY_SIZE(hub_client_descs_gv11b)))) | ||
611 | mmfault->client_id_desc = | ||
612 | hub_client_descs_gv11b[mmfault->client_id]; | ||
613 | } else if (mmfault->client_type == | ||
614 | gmmu_fault_client_type_gpc_v()) { | ||
615 | if (!(WARN_ON(mmfault->client_id >= | ||
616 | ARRAY_SIZE(gpc_client_descs_gv11b)))) | ||
617 | mmfault->client_id_desc = | ||
618 | gpc_client_descs_gv11b[mmfault->client_id]; | ||
619 | } | ||
620 | |||
621 | } | ||
622 | |||
623 | static void gv11b_fb_print_fault_info(struct gk20a *g, | ||
624 | struct mmu_fault_info *mmfault) | ||
625 | { | ||
626 | if (mmfault && mmfault->valid) { | ||
627 | nvgpu_err(g, "[MMU FAULT] " | ||
628 | "mmu engine id: %d, " | ||
629 | "ch id: %d, " | ||
630 | "fault addr: 0x%llx, " | ||
631 | "fault addr aperture: %d, " | ||
632 | "fault type: %s, " | ||
633 | "access type: %s, ", | ||
634 | mmfault->mmu_engine_id, | ||
635 | mmfault->chid, | ||
636 | mmfault->fault_addr, | ||
637 | mmfault->fault_addr_aperture, | ||
638 | mmfault->fault_type_desc, | ||
639 | fault_access_type_descs_gv11b[mmfault->access_type]); | ||
640 | nvgpu_err(g, "[MMU FAULT] " | ||
641 | "protected mode: %d, " | ||
642 | "client type: %s, " | ||
643 | "client id: %s, " | ||
644 | "gpc id if client type is gpc: %d, ", | ||
645 | mmfault->protected_mode, | ||
646 | mmfault->client_type_desc, | ||
647 | mmfault->client_id_desc, | ||
648 | mmfault->gpc_id); | ||
649 | |||
650 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
651 | "faulted act eng id if any: 0x%x, " | ||
652 | "faulted veid if any: 0x%x, " | ||
653 | "faulted pbdma id if any: 0x%x, ", | ||
654 | mmfault->faulted_engine, | ||
655 | mmfault->faulted_subid, | ||
656 | mmfault->faulted_pbdma); | ||
657 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
658 | "inst ptr: 0x%llx, " | ||
659 | "inst ptr aperture: %d, " | ||
660 | "replayable fault: %d, " | ||
661 | "replayable fault en: %d " | ||
662 | "timestamp hi:lo 0x%08x:0x%08x, ", | ||
663 | mmfault->inst_ptr, | ||
664 | mmfault->inst_aperture, | ||
665 | mmfault->replayable_fault, | ||
666 | mmfault->replay_fault_en, | ||
667 | mmfault->timestamp_hi, mmfault->timestamp_lo); | ||
668 | } | ||
669 | } | ||
670 | |||
671 | /* | ||
672 | *Fault buffer format | ||
673 | * | ||
674 | * 31 28 24 23 16 15 8 7 4 0 | ||
675 | *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. | ||
676 | *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| | ||
677 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
678 | *| inst_hi | | ||
679 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
680 | *| addr_31_12 | |AP | | ||
681 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
682 | *| addr_63_32 | | ||
683 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
684 | *| timestamp_lo | | ||
685 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
686 | *| timestamp_hi | | ||
687 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
688 | *| (reserved) | engine_id | | ||
689 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
690 | *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| | ||
691 | */ | ||
692 | |||
693 | static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, | ||
694 | struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) | ||
695 | { | ||
696 | u32 rd32_val; | ||
697 | u32 addr_lo, addr_hi; | ||
698 | u64 inst_ptr; | ||
699 | u32 chid = FIFO_INVAL_CHANNEL_ID; | ||
700 | struct channel_gk20a *refch; | ||
701 | |||
702 | memset(mmfault, 0, sizeof(*mmfault)); | ||
703 | |||
704 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
705 | gmmu_fault_buf_entry_inst_lo_w()); | ||
706 | addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); | ||
707 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
708 | |||
709 | addr_hi = nvgpu_mem_rd32(g, mem, offset + | ||
710 | gmmu_fault_buf_entry_inst_hi_w()); | ||
711 | addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); | ||
712 | |||
713 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
714 | |||
715 | /* refch will be put back after fault is handled */ | ||
716 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
717 | if (refch) | ||
718 | chid = refch->chid; | ||
719 | |||
720 | /* it is ok to continue even if refch is NULL */ | ||
721 | mmfault->refch = refch; | ||
722 | mmfault->chid = chid; | ||
723 | mmfault->inst_ptr = inst_ptr; | ||
724 | mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); | ||
725 | |||
726 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
727 | gmmu_fault_buf_entry_addr_lo_w()); | ||
728 | |||
729 | mmfault->fault_addr_aperture = | ||
730 | gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); | ||
731 | addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); | ||
732 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
733 | |||
734 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
735 | gmmu_fault_buf_entry_addr_hi_w()); | ||
736 | addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); | ||
737 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
738 | |||
739 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
740 | gmmu_fault_buf_entry_timestamp_lo_w()); | ||
741 | mmfault->timestamp_lo = | ||
742 | gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); | ||
743 | |||
744 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
745 | gmmu_fault_buf_entry_timestamp_hi_w()); | ||
746 | mmfault->timestamp_hi = | ||
747 | gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); | ||
748 | |||
749 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
750 | gmmu_fault_buf_entry_engine_id_w()); | ||
751 | |||
752 | mmfault->mmu_engine_id = | ||
753 | gmmu_fault_buf_entry_engine_id_v(rd32_val); | ||
754 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
755 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
756 | &mmfault->faulted_pbdma); | ||
757 | |||
758 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
759 | gmmu_fault_buf_entry_fault_type_w()); | ||
760 | mmfault->client_id = | ||
761 | gmmu_fault_buf_entry_client_v(rd32_val); | ||
762 | mmfault->replayable_fault = | ||
763 | gmmu_fault_buf_entry_replayable_fault_v(rd32_val); | ||
764 | |||
765 | mmfault->fault_type = | ||
766 | gmmu_fault_buf_entry_fault_type_v(rd32_val); | ||
767 | mmfault->access_type = | ||
768 | gmmu_fault_buf_entry_access_type_v(rd32_val); | ||
769 | |||
770 | mmfault->client_type = | ||
771 | gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); | ||
772 | |||
773 | mmfault->gpc_id = | ||
774 | gmmu_fault_buf_entry_gpc_id_v(rd32_val); | ||
775 | mmfault->protected_mode = | ||
776 | gmmu_fault_buf_entry_protected_mode_v(rd32_val); | ||
777 | |||
778 | mmfault->replay_fault_en = | ||
779 | gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); | ||
780 | |||
781 | mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); | ||
782 | |||
783 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
784 | gmmu_fault_buf_entry_fault_type_w()); | ||
785 | rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); | ||
786 | nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), | ||
787 | rd32_val); | ||
788 | |||
789 | gv11b_fb_parse_mmfault(mmfault); | ||
790 | } | ||
791 | |||
792 | static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, | ||
793 | struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) | ||
794 | { | ||
795 | unsigned int id_type; | ||
796 | u32 num_lce, act_eng_bitmask = 0; | ||
797 | int err = 0; | ||
798 | u32 id = ((u32)~0); | ||
799 | |||
800 | if (!mmfault->valid) | ||
801 | return; | ||
802 | |||
803 | gv11b_fb_print_fault_info(g, mmfault); | ||
804 | |||
805 | num_lce = gv11b_ce_get_num_lce(g); | ||
806 | if ((mmfault->mmu_engine_id >= | ||
807 | gmmu_fault_mmu_eng_id_ce0_v()) && | ||
808 | (mmfault->mmu_engine_id < | ||
809 | gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { | ||
810 | /* CE page faults are not reported as replayable */ | ||
811 | nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); | ||
812 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
813 | gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, | ||
814 | mmfault->faulted_pbdma, mmfault->faulted_engine); | ||
815 | if (!err) { | ||
816 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); | ||
817 | *invalidate_replay_val = 0; | ||
818 | /* refch in mmfault is assigned at the time of copying | ||
819 | * fault info from snap reg or bar2 fault buf | ||
820 | */ | ||
821 | gk20a_channel_put(mmfault->refch); | ||
822 | return; | ||
823 | } | ||
824 | /* Do recovery. Channel recovery needs refch */ | ||
825 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); | ||
826 | } | ||
827 | |||
828 | if (!mmfault->replayable_fault) { | ||
829 | if (mmfault->fault_type == | ||
830 | gmmu_fault_type_unbound_inst_block_v()) { | ||
831 | /* | ||
832 | * Bug 1847172: When an engine faults due to an unbound | ||
833 | * instance block, the fault cannot be isolated to a | ||
834 | * single context so we need to reset the entire runlist | ||
835 | */ | ||
836 | id_type = ID_TYPE_UNKNOWN; | ||
837 | |||
838 | } else if (mmfault->refch) { | ||
839 | if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { | ||
840 | id = mmfault->refch->tsgid; | ||
841 | id_type = ID_TYPE_TSG; | ||
842 | } else { | ||
843 | id = mmfault->chid; | ||
844 | id_type = ID_TYPE_CHANNEL; | ||
845 | } | ||
846 | if (mmfault->refch->mmu_nack_handled) { | ||
847 | /* We have already recovered for the same | ||
848 | * context, skip doing another recovery. | ||
849 | */ | ||
850 | mmfault->refch->mmu_nack_handled = false; | ||
851 | /* | ||
852 | * Recovery path can be entered twice for the | ||
853 | * same error in case of mmu nack. If mmu | ||
854 | * nack interrupt is handled before mmu fault | ||
855 | * then channel reference is increased to avoid | ||
856 | * closing the channel by userspace. Decrement | ||
857 | * channel reference. | ||
858 | */ | ||
859 | gk20a_channel_put(mmfault->refch); | ||
860 | /* refch in mmfault is assigned at the time | ||
861 | * of copying fault info from snap reg or bar2 | ||
862 | * fault buf. | ||
863 | */ | ||
864 | gk20a_channel_put(mmfault->refch); | ||
865 | return; | ||
866 | } | ||
867 | } else { | ||
868 | id_type = ID_TYPE_UNKNOWN; | ||
869 | } | ||
870 | if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
871 | act_eng_bitmask = BIT(mmfault->faulted_engine); | ||
872 | |||
873 | /* Indicate recovery is handled if mmu fault is a result of | ||
874 | * mmu nack. | ||
875 | */ | ||
876 | mmfault->refch->mmu_nack_handled = true; | ||
877 | g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, | ||
878 | id, id_type, RC_TYPE_MMU_FAULT, mmfault); | ||
879 | } else { | ||
880 | if (mmfault->fault_type == gmmu_fault_type_pte_v()) { | ||
881 | nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); | ||
882 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
883 | if (err) | ||
884 | *invalidate_replay_val |= | ||
885 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
886 | else | ||
887 | *invalidate_replay_val |= | ||
888 | fb_mmu_invalidate_replay_start_ack_all_f(); | ||
889 | } else { | ||
890 | /* cancel faults other than invalid pte */ | ||
891 | *invalidate_replay_val |= | ||
892 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
893 | } | ||
894 | /* refch in mmfault is assigned at the time of copying | ||
895 | * fault info from snap reg or bar2 fault buf | ||
896 | */ | ||
897 | gk20a_channel_put(mmfault->refch); | ||
898 | } | ||
899 | } | ||
900 | |||
901 | static int gv11b_fb_replay_or_cancel_faults(struct gk20a *g, | ||
902 | u32 invalidate_replay_val) | ||
903 | { | ||
904 | int err = 0; | ||
905 | |||
906 | nvgpu_log_fn(g, " "); | ||
907 | |||
908 | if (invalidate_replay_val & | ||
909 | fb_mmu_invalidate_replay_cancel_global_f()) { | ||
910 | /* | ||
911 | * cancel faults so that next time it faults as | ||
912 | * replayable faults and channel recovery can be done | ||
913 | */ | ||
914 | err = g->ops.fb.mmu_invalidate_replay(g, | ||
915 | fb_mmu_invalidate_replay_cancel_global_f()); | ||
916 | } else if (invalidate_replay_val & | ||
917 | fb_mmu_invalidate_replay_start_ack_all_f()) { | ||
918 | /* pte valid is fixed. replay faulting request */ | ||
919 | err = g->ops.fb.mmu_invalidate_replay(g, | ||
920 | fb_mmu_invalidate_replay_start_ack_all_f()); | ||
921 | } | ||
922 | |||
923 | return err; | ||
924 | } | ||
925 | |||
926 | void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, | ||
927 | u32 fault_status, u32 index) | ||
928 | { | ||
929 | u32 get_indx, offset, rd32_val, entries; | ||
930 | struct nvgpu_mem *mem; | ||
931 | struct mmu_fault_info *mmfault; | ||
932 | u32 invalidate_replay_val = 0; | ||
933 | u64 prev_fault_addr = 0ULL; | ||
934 | u64 next_fault_addr = 0ULL; | ||
935 | |||
936 | if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { | ||
937 | nvgpu_log(g, gpu_dbg_intr, | ||
938 | "SPURIOUS mmu fault: reg index:%d", index); | ||
939 | return; | ||
940 | } | ||
941 | nvgpu_log(g, gpu_dbg_intr, "%s MMU FAULT" , | ||
942 | index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX ? | ||
943 | "REPLAY" : "NON-REPLAY"); | ||
944 | |||
945 | nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); | ||
946 | |||
947 | mem = &g->mm.hw_fault_buf[index]; | ||
948 | mmfault = &g->mm.fault_info[index]; | ||
949 | |||
950 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
951 | nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); | ||
952 | |||
953 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
954 | nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); | ||
955 | |||
956 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
957 | offset + gmmu_fault_buf_entry_valid_w()); | ||
958 | nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); | ||
959 | |||
960 | while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { | ||
961 | |||
962 | nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); | ||
963 | |||
964 | gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); | ||
965 | |||
966 | get_indx = (get_indx + 1) % entries; | ||
967 | nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); | ||
968 | |||
969 | gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); | ||
970 | |||
971 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
972 | nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); | ||
973 | |||
974 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
975 | offset + gmmu_fault_buf_entry_valid_w()); | ||
976 | |||
977 | if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && | ||
978 | mmfault->fault_addr != 0ULL) { | ||
979 | /* fault_addr "0" is not supposed to be fixed ever. | ||
980 | * For the first time when prev = 0, next = 0 and | ||
981 | * fault addr is also 0 then handle_mmu_fault_common will | ||
982 | * not be called. Fix by checking fault_addr not equal to 0 | ||
983 | */ | ||
984 | prev_fault_addr = next_fault_addr; | ||
985 | next_fault_addr = mmfault->fault_addr; | ||
986 | if (prev_fault_addr == next_fault_addr) { | ||
987 | nvgpu_log(g, gpu_dbg_intr, "pte already scanned"); | ||
988 | if (mmfault->refch) | ||
989 | gk20a_channel_put(mmfault->refch); | ||
990 | continue; | ||
991 | } | ||
992 | } | ||
993 | |||
994 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
995 | &invalidate_replay_val); | ||
996 | |||
997 | } | ||
998 | if (index == NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX && | ||
999 | invalidate_replay_val != 0U) | ||
1000 | gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); | ||
1001 | } | ||
1002 | |||
1003 | static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, | ||
1004 | u32 fault_status, struct mmu_fault_info *mmfault) | ||
1005 | { | ||
1006 | u32 reg_val; | ||
1007 | u32 addr_lo, addr_hi; | ||
1008 | u64 inst_ptr; | ||
1009 | int chid = FIFO_INVAL_CHANNEL_ID; | ||
1010 | struct channel_gk20a *refch; | ||
1011 | |||
1012 | memset(mmfault, 0, sizeof(*mmfault)); | ||
1013 | |||
1014 | if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { | ||
1015 | |||
1016 | nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); | ||
1017 | return; | ||
1018 | } | ||
1019 | |||
1020 | g->ops.fb.read_mmu_fault_inst_lo_hi(g, ®_val, &addr_hi); | ||
1021 | |||
1022 | addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); | ||
1023 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1024 | |||
1025 | addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); | ||
1026 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1027 | |||
1028 | /* refch will be put back after fault is handled */ | ||
1029 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
1030 | if (refch) | ||
1031 | chid = refch->chid; | ||
1032 | |||
1033 | /* It is still ok to continue if refch is NULL */ | ||
1034 | mmfault->refch = refch; | ||
1035 | mmfault->chid = chid; | ||
1036 | mmfault->inst_ptr = inst_ptr; | ||
1037 | mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); | ||
1038 | mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); | ||
1039 | |||
1040 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
1041 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
1042 | &mmfault->faulted_pbdma); | ||
1043 | |||
1044 | g->ops.fb.read_mmu_fault_addr_lo_hi(g, ®_val, &addr_hi); | ||
1045 | |||
1046 | addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); | ||
1047 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1048 | |||
1049 | mmfault->fault_addr_aperture = | ||
1050 | fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); | ||
1051 | |||
1052 | addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); | ||
1053 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1054 | |||
1055 | reg_val = g->ops.fb.read_mmu_fault_info(g); | ||
1056 | mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); | ||
1057 | mmfault->replayable_fault = | ||
1058 | fb_mmu_fault_info_replayable_fault_v(reg_val); | ||
1059 | mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); | ||
1060 | mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); | ||
1061 | mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); | ||
1062 | mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); | ||
1063 | mmfault->protected_mode = | ||
1064 | fb_mmu_fault_info_protected_mode_v(reg_val); | ||
1065 | mmfault->replay_fault_en = | ||
1066 | fb_mmu_fault_info_replayable_fault_en_v(reg_val); | ||
1067 | |||
1068 | mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); | ||
1069 | |||
1070 | fault_status &= ~(fb_mmu_fault_status_valid_m()); | ||
1071 | g->ops.fb.write_mmu_fault_status(g, fault_status); | ||
1072 | |||
1073 | gv11b_fb_parse_mmfault(mmfault); | ||
1074 | |||
1075 | } | ||
1076 | |||
1077 | void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, | ||
1078 | u32 fault_status) | ||
1079 | { | ||
1080 | u32 reg_val; | ||
1081 | u32 index = NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX; | ||
1082 | |||
1083 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
1084 | |||
1085 | if (fault_status & | ||
1086 | fb_mmu_fault_status_replayable_getptr_corrupted_m()) { | ||
1087 | |||
1088 | nvgpu_err(g, "replayable getptr corrupted set"); | ||
1089 | |||
1090 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1091 | |||
1092 | reg_val = set_field(reg_val, | ||
1093 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1094 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1095 | } | ||
1096 | |||
1097 | if (fault_status & | ||
1098 | fb_mmu_fault_status_replayable_overflow_m()) { | ||
1099 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1100 | |||
1101 | nvgpu_err(g, "replayable overflow: buffer full:%s", | ||
1102 | buffer_full?"true":"false"); | ||
1103 | |||
1104 | reg_val = set_field(reg_val, | ||
1105 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1106 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1107 | } | ||
1108 | |||
1109 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
1110 | } | ||
1111 | |||
1112 | void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, | ||
1113 | u32 fault_status) | ||
1114 | { | ||
1115 | u32 reg_val; | ||
1116 | u32 index = NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX; | ||
1117 | |||
1118 | reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index); | ||
1119 | |||
1120 | if (fault_status & | ||
1121 | fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { | ||
1122 | |||
1123 | nvgpu_err(g, "non replayable getptr corrupted set"); | ||
1124 | |||
1125 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1126 | |||
1127 | reg_val = set_field(reg_val, | ||
1128 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1129 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1130 | } | ||
1131 | |||
1132 | if (fault_status & | ||
1133 | fb_mmu_fault_status_non_replayable_overflow_m()) { | ||
1134 | |||
1135 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1136 | |||
1137 | nvgpu_err(g, "non replayable overflow: buffer full:%s", | ||
1138 | buffer_full?"true":"false"); | ||
1139 | |||
1140 | reg_val = set_field(reg_val, | ||
1141 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1142 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1143 | } | ||
1144 | |||
1145 | g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val); | ||
1146 | } | ||
1147 | |||
1148 | static void gv11b_fb_handle_bar2_fault(struct gk20a *g, | ||
1149 | struct mmu_fault_info *mmfault, u32 fault_status) | ||
1150 | { | ||
1151 | if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { | ||
1152 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1153 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) | ||
1154 | gv11b_fb_fault_buf_configure_hw(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); | ||
1155 | } | ||
1156 | |||
1157 | if (fault_status & fb_mmu_fault_status_replayable_error_m()) { | ||
1158 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1159 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) | ||
1160 | gv11b_fb_fault_buf_configure_hw(g, | ||
1161 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1162 | } | ||
1163 | gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); | ||
1164 | |||
1165 | g->ops.bus.bar2_bind(g, &g->mm.bar2.inst_block); | ||
1166 | |||
1167 | if (mmfault->refch) { | ||
1168 | gk20a_channel_put(mmfault->refch); | ||
1169 | mmfault->refch = NULL; | ||
1170 | } | ||
1171 | } | ||
1172 | |||
1173 | void gv11b_fb_handle_other_fault_notify(struct gk20a *g, | ||
1174 | u32 fault_status) | ||
1175 | { | ||
1176 | struct mmu_fault_info *mmfault; | ||
1177 | u32 invalidate_replay_val = 0; | ||
1178 | |||
1179 | mmfault = &g->mm.fault_info[NVGPU_MM_MMU_FAULT_TYPE_OTHER_AND_NONREPLAY]; | ||
1180 | |||
1181 | gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); | ||
1182 | |||
1183 | /* BAR2/Physical faults will not be snapped in hw fault buf */ | ||
1184 | if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { | ||
1185 | nvgpu_err(g, "BAR2 MMU FAULT"); | ||
1186 | gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); | ||
1187 | |||
1188 | } else if (mmfault->mmu_engine_id == | ||
1189 | gmmu_fault_mmu_eng_id_physical_v()) { | ||
1190 | /* usually means VPR or out of bounds physical accesses */ | ||
1191 | nvgpu_err(g, "PHYSICAL MMU FAULT"); | ||
1192 | |||
1193 | } else { | ||
1194 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1195 | &invalidate_replay_val); | ||
1196 | |||
1197 | if (invalidate_replay_val) | ||
1198 | gv11b_fb_replay_or_cancel_faults(g, | ||
1199 | invalidate_replay_val); | ||
1200 | } | ||
1201 | } | ||
1202 | |||
1203 | void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) | ||
1204 | { | ||
1205 | u32 dropped_faults = 0; | ||
1206 | |||
1207 | dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | | ||
1208 | fb_mmu_fault_status_dropped_bar1_virt_set_f() | | ||
1209 | fb_mmu_fault_status_dropped_bar2_phys_set_f() | | ||
1210 | fb_mmu_fault_status_dropped_bar2_virt_set_f() | | ||
1211 | fb_mmu_fault_status_dropped_ifb_phys_set_f() | | ||
1212 | fb_mmu_fault_status_dropped_ifb_virt_set_f() | | ||
1213 | fb_mmu_fault_status_dropped_other_phys_set_f()| | ||
1214 | fb_mmu_fault_status_dropped_other_virt_set_f(); | ||
1215 | |||
1216 | if (fault_status & dropped_faults) { | ||
1217 | nvgpu_err(g, "dropped mmu fault (0x%08x)", | ||
1218 | fault_status & dropped_faults); | ||
1219 | g->ops.fb.write_mmu_fault_status(g, dropped_faults); | ||
1220 | } | ||
1221 | } | ||
1222 | |||
1223 | void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g) | ||
1224 | { | ||
1225 | u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
1226 | |||
1227 | if (!(fault_status & fb_mmu_fault_status_replayable_m())) | ||
1228 | return; | ||
1229 | |||
1230 | if (gv11b_fb_is_fault_buf_enabled(g, | ||
1231 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { | ||
1232 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1233 | fault_status, | ||
1234 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1235 | } | ||
1236 | } | ||
1237 | |||
1238 | static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) | ||
1239 | { | ||
1240 | u32 fault_status = g->ops.fb.read_mmu_fault_status(g); | ||
1241 | |||
1242 | nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); | ||
1243 | |||
1244 | if (niso_intr & | ||
1245 | fb_niso_intr_mmu_other_fault_notify_m()) { | ||
1246 | |||
1247 | gv11b_fb_handle_dropped_mmu_fault(g, fault_status); | ||
1248 | |||
1249 | gv11b_fb_handle_other_fault_notify(g, fault_status); | ||
1250 | } | ||
1251 | |||
1252 | if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX)) { | ||
1253 | |||
1254 | if (niso_intr & | ||
1255 | fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { | ||
1256 | |||
1257 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1258 | fault_status, | ||
1259 | NVGPU_FB_MMU_FAULT_NONREPLAY_REG_INDEX); | ||
1260 | |||
1261 | /* | ||
1262 | * When all the faults are processed, | ||
1263 | * GET and PUT will have same value and mmu fault status | ||
1264 | * bit will be reset by HW | ||
1265 | */ | ||
1266 | } | ||
1267 | if (niso_intr & | ||
1268 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { | ||
1269 | |||
1270 | gv11b_fb_handle_nonreplay_fault_overflow(g, | ||
1271 | fault_status); | ||
1272 | } | ||
1273 | |||
1274 | } | ||
1275 | |||
1276 | if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX)) { | ||
1277 | |||
1278 | if (niso_intr & | ||
1279 | fb_niso_intr_mmu_replayable_fault_notify_m()) { | ||
1280 | |||
1281 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1282 | fault_status, | ||
1283 | NVGPU_FB_MMU_FAULT_REPLAY_REG_INDEX); | ||
1284 | } | ||
1285 | if (niso_intr & | ||
1286 | fb_niso_intr_mmu_replayable_fault_overflow_m()) { | ||
1287 | |||
1288 | gv11b_fb_handle_replay_fault_overflow(g, | ||
1289 | fault_status); | ||
1290 | } | ||
1291 | |||
1292 | } | ||
1293 | |||
1294 | nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); | ||
1295 | g->ops.fb.write_mmu_fault_status(g, | ||
1296 | fb_mmu_fault_status_valid_clear_f()); | ||
1297 | } | ||
1298 | |||
1299 | void gv11b_fb_hub_isr(struct gk20a *g) | ||
1300 | { | ||
1301 | u32 status, niso_intr; | ||
1302 | |||
1303 | nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); | ||
1304 | |||
1305 | niso_intr = gk20a_readl(g, fb_niso_intr_r()); | ||
1306 | |||
1307 | nvgpu_log(g, gpu_dbg_intr, "enter hub isr, niso_intr = 0x%08x", | ||
1308 | niso_intr); | ||
1309 | |||
1310 | if (niso_intr & | ||
1311 | (fb_niso_intr_hub_access_counter_notify_m() | | ||
1312 | fb_niso_intr_hub_access_counter_error_m())) { | ||
1313 | |||
1314 | nvgpu_info(g, "hub access counter notify/error"); | ||
1315 | } | ||
1316 | if (niso_intr & | ||
1317 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { | ||
1318 | |||
1319 | nvgpu_info(g, "ecc uncorrected error notify"); | ||
1320 | |||
1321 | status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); | ||
1322 | if (status) | ||
1323 | gv11b_handle_l2tlb_ecc_isr(g, status); | ||
1324 | |||
1325 | status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); | ||
1326 | if (status) | ||
1327 | gv11b_handle_hubtlb_ecc_isr(g, status); | ||
1328 | |||
1329 | status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); | ||
1330 | if (status) | ||
1331 | gv11b_handle_fillunit_ecc_isr(g, status); | ||
1332 | } | ||
1333 | if (niso_intr & | ||
1334 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1335 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1336 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1337 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1338 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { | ||
1339 | |||
1340 | nvgpu_log(g, gpu_dbg_intr, "MMU Fault"); | ||
1341 | gv11b_fb_handle_mmu_fault(g, niso_intr); | ||
1342 | } | ||
1343 | |||
1344 | nvgpu_mutex_release(&g->mm.hub_isr_mutex); | ||
1345 | } | ||
1346 | |||
1347 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g) | ||
1348 | { | ||
1349 | if (gk20a_readl(g, fb_niso_intr_r()) & | ||
1350 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1351 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | | ||
1352 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1353 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1354 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1355 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) | ||
1356 | return true; | ||
1357 | |||
1358 | return false; | ||
1359 | } | ||
1360 | |||
1361 | int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
1362 | u32 invalidate_replay_val) | ||
1363 | { | ||
1364 | int err = -ETIMEDOUT; | ||
1365 | u32 reg_val; | ||
1366 | struct nvgpu_timeout timeout; | ||
1367 | |||
1368 | nvgpu_log_fn(g, " "); | ||
1369 | |||
1370 | nvgpu_mutex_acquire(&g->mm.tlb_lock); | ||
1371 | |||
1372 | reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); | ||
1373 | |||
1374 | reg_val |= fb_mmu_invalidate_all_va_true_f() | | ||
1375 | fb_mmu_invalidate_all_pdb_true_f() | | ||
1376 | invalidate_replay_val | | ||
1377 | fb_mmu_invalidate_trigger_true_f(); | ||
1378 | |||
1379 | gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); | ||
1380 | |||
1381 | /* retry 200 times */ | ||
1382 | nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); | ||
1383 | do { | ||
1384 | reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
1385 | if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != | ||
1386 | fb_mmu_ctrl_pri_fifo_empty_false_f()) { | ||
1387 | err = 0; | ||
1388 | break; | ||
1389 | } | ||
1390 | nvgpu_udelay(5); | ||
1391 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
1392 | "invalidate replay failed on 0x%llx")); | ||
1393 | if (err) | ||
1394 | nvgpu_err(g, "invalidate replay timedout"); | ||
1395 | |||
1396 | nvgpu_mutex_release(&g->mm.tlb_lock); | ||
1397 | |||
1398 | return err; | ||
1399 | } | ||
1400 | |||
1401 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
1402 | struct mmu_fault_info *mmfault) | ||
1403 | { | ||
1404 | int err = 0; | ||
1405 | u32 pte[2]; | ||
1406 | |||
1407 | if (mmfault->refch == NULL) { | ||
1408 | nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); | ||
1409 | return -EINVAL; | ||
1410 | } | ||
1411 | |||
1412 | err = __nvgpu_get_pte(g, | ||
1413 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1414 | if (err) { | ||
1415 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); | ||
1416 | return err; | ||
1417 | } | ||
1418 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1419 | "pte: %#08x %#08x", pte[1], pte[0]); | ||
1420 | |||
1421 | if (pte[0] == 0x0 && pte[1] == 0x0) { | ||
1422 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1423 | "pte all zeros, do not set valid"); | ||
1424 | return -1; | ||
1425 | } | ||
1426 | if (pte[0] & gmmu_new_pte_valid_true_f()) { | ||
1427 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1428 | "pte valid already set"); | ||
1429 | return -1; | ||
1430 | } | ||
1431 | |||
1432 | pte[0] |= gmmu_new_pte_valid_true_f(); | ||
1433 | if (pte[0] & gmmu_new_pte_read_only_true_f()) | ||
1434 | pte[0] &= ~(gmmu_new_pte_read_only_true_f()); | ||
1435 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1436 | "new pte: %#08x %#08x", pte[1], pte[0]); | ||
1437 | |||
1438 | err = __nvgpu_set_pte(g, | ||
1439 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1440 | if (err) { | ||
1441 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); | ||
1442 | return err; | ||
1443 | } | ||
1444 | /* invalidate tlb so that GMMU does not use old cached translation */ | ||
1445 | g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); | ||
1446 | |||
1447 | err = __nvgpu_get_pte(g, | ||
1448 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1449 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1450 | "pte after tlb invalidate: %#08x %#08x", | ||
1451 | pte[1], pte[0]); | ||
1452 | return err; | ||
1453 | } | ||
1454 | |||
1455 | void fb_gv11b_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index, | ||
1456 | u32 addr_lo, u32 addr_hi) | ||
1457 | { | ||
1458 | nvgpu_writel(g, fb_mmu_fault_buffer_lo_r(index), addr_lo); | ||
1459 | nvgpu_writel(g, fb_mmu_fault_buffer_hi_r(index), addr_hi); | ||
1460 | } | ||
1461 | |||
1462 | u32 fb_gv11b_read_mmu_fault_buffer_get(struct gk20a *g, u32 index) | ||
1463 | { | ||
1464 | return nvgpu_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1465 | } | ||
1466 | |||
1467 | void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index, | ||
1468 | u32 reg_val) | ||
1469 | { | ||
1470 | nvgpu_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1471 | } | ||
1472 | |||
1473 | u32 fb_gv11b_read_mmu_fault_buffer_put(struct gk20a *g, u32 index) | ||
1474 | { | ||
1475 | return nvgpu_readl(g, fb_mmu_fault_buffer_put_r(index)); | ||
1476 | } | ||
1477 | |||
1478 | u32 fb_gv11b_read_mmu_fault_buffer_size(struct gk20a *g, u32 index) | ||
1479 | { | ||
1480 | return nvgpu_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
1481 | } | ||
1482 | |||
1483 | void fb_gv11b_write_mmu_fault_buffer_size(struct gk20a *g, u32 index, | ||
1484 | u32 reg_val) | ||
1485 | { | ||
1486 | nvgpu_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); | ||
1487 | } | ||
1488 | |||
1489 | void fb_gv11b_read_mmu_fault_addr_lo_hi(struct gk20a *g, | ||
1490 | u32 *addr_lo, u32 *addr_hi) | ||
1491 | { | ||
1492 | *addr_lo = nvgpu_readl(g, fb_mmu_fault_addr_lo_r()); | ||
1493 | *addr_hi = nvgpu_readl(g, fb_mmu_fault_addr_hi_r()); | ||
1494 | } | ||
1495 | |||
1496 | void fb_gv11b_read_mmu_fault_inst_lo_hi(struct gk20a *g, | ||
1497 | u32 *inst_lo, u32 *inst_hi) | ||
1498 | { | ||
1499 | *inst_lo = nvgpu_readl(g, fb_mmu_fault_inst_lo_r()); | ||
1500 | *inst_hi = nvgpu_readl(g, fb_mmu_fault_inst_hi_r()); | ||
1501 | } | ||
1502 | |||
1503 | u32 fb_gv11b_read_mmu_fault_info(struct gk20a *g) | ||
1504 | { | ||
1505 | return nvgpu_readl(g, fb_mmu_fault_info_r()); | ||
1506 | } | ||
1507 | |||
1508 | u32 fb_gv11b_read_mmu_fault_status(struct gk20a *g) | ||
1509 | { | ||
1510 | return nvgpu_readl(g, fb_mmu_fault_status_r()); | ||
1511 | } | ||
1512 | |||
1513 | void fb_gv11b_write_mmu_fault_status(struct gk20a *g, u32 reg_val) | ||
1514 | { | ||
1515 | nvgpu_writel(g, fb_mmu_fault_status_r(), reg_val); | ||
1516 | } | ||