diff options
author | Deepak Nibade <dnibade@nvidia.com> | 2017-11-16 02:21:19 -0500 |
---|---|---|
committer | Deepak Nibade <dnibade@nvidia.com> | 2017-11-16 02:21:35 -0500 |
commit | ba8dc318595f597308902ad16ffed89bdbe7000f (patch) | |
tree | e882886e0cbc05ac39473b95ead16ee50bd69c15 /drivers/gpu/nvgpu/gv11b/fb_gv11b.c | |
parent | 69e032653df5aae335764f6346703a1e55c96a2d (diff) | |
parent | 77a90d0b8d2eb1bbb207ae5f46b357f2d7cd07ab (diff) |
Merge remote-tracking branch 'remotes/origin/dev/linux-nvgpu-t19x' into linux-nvgpu
Bug 200363166
Change-Id: Ic662d7b44b673db28dc0aeba338ae67cf2a43d64
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Diffstat (limited to 'drivers/gpu/nvgpu/gv11b/fb_gv11b.c')
-rw-r--r-- | drivers/gpu/nvgpu/gv11b/fb_gv11b.c | 1555 |
1 files changed, 1555 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/gv11b/fb_gv11b.c b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c new file mode 100644 index 00000000..ec487bdf --- /dev/null +++ b/drivers/gpu/nvgpu/gv11b/fb_gv11b.c | |||
@@ -0,0 +1,1555 @@ | |||
1 | /* | ||
2 | * GV11B FB | ||
3 | * | ||
4 | * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | |||
27 | #include <nvgpu/dma.h> | ||
28 | #include <nvgpu/log.h> | ||
29 | #include <nvgpu/enabled.h> | ||
30 | #include <nvgpu/gmmu.h> | ||
31 | #include <nvgpu/barrier.h> | ||
32 | #include <nvgpu/soc.h> | ||
33 | |||
34 | #include "gk20a/gk20a.h" | ||
35 | #include "gk20a/mm_gk20a.h" | ||
36 | |||
37 | #include "gp10b/fb_gp10b.h" | ||
38 | |||
39 | #include "gv11b/fifo_gv11b.h" | ||
40 | #include "gv11b/fb_gv11b.h" | ||
41 | #include "gv11b/ce_gv11b.h" | ||
42 | |||
43 | #include <nvgpu/hw/gv11b/hw_fb_gv11b.h> | ||
44 | #include <nvgpu/hw/gv11b/hw_mc_gv11b.h> | ||
45 | #include <nvgpu/hw/gv11b/hw_ram_gv11b.h> | ||
46 | #include <nvgpu/hw/gv11b/hw_gmmu_gv11b.h> | ||
47 | |||
48 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
49 | struct mmu_fault_info *mmfault); | ||
50 | |||
51 | static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
52 | u32 invalidate_replay_val); | ||
53 | |||
54 | static void gv11b_init_nvlink_soc_credits(struct gk20a *g) | ||
55 | { | ||
56 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
57 | nvgpu_info(g, "nvlink soc credits init done by bpmp"); | ||
58 | } else { | ||
59 | /* MSS_NVLINK_1_BASE */ | ||
60 | void __iomem *soc1 = ioremap(0x01f20010, 4096); | ||
61 | /* MSS_NVLINK_2_BASE */ | ||
62 | void __iomem *soc2 = ioremap(0x01f40010, 4096); | ||
63 | /* MSS_NVLINK_3_BASE */ | ||
64 | void __iomem *soc3 = ioremap(0x01f60010, 4096); | ||
65 | /* MSS_NVLINK_4_BASE */ | ||
66 | void __iomem *soc4 = ioremap(0x01f80010, 4096); | ||
67 | u32 val; | ||
68 | |||
69 | nvgpu_info(g, "init nvlink soc credits"); | ||
70 | |||
71 | val = readl_relaxed(soc1); | ||
72 | writel_relaxed(val, soc1); | ||
73 | val = readl_relaxed(soc1 + 4); | ||
74 | writel_relaxed(val, soc1 + 4); | ||
75 | |||
76 | val = readl_relaxed(soc2); | ||
77 | writel_relaxed(val, soc2); | ||
78 | val = readl_relaxed(soc2 + 4); | ||
79 | writel_relaxed(val, soc2 + 4); | ||
80 | |||
81 | val = readl_relaxed(soc3); | ||
82 | writel_relaxed(val, soc3); | ||
83 | val = readl_relaxed(soc3 + 4); | ||
84 | writel_relaxed(val, soc3 + 4); | ||
85 | |||
86 | val = readl_relaxed(soc4); | ||
87 | writel_relaxed(val, soc4); | ||
88 | val = readl_relaxed(soc4 + 4); | ||
89 | writel_relaxed(val, soc4 + 4); | ||
90 | } | ||
91 | } | ||
92 | |||
93 | void gv11b_fb_init_fs_state(struct gk20a *g) | ||
94 | { | ||
95 | nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb"); | ||
96 | |||
97 | nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x", | ||
98 | gk20a_readl(g, fb_fbhub_num_active_ltcs_r())); | ||
99 | |||
100 | nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u", | ||
101 | fb_mmu_num_active_ltcs_count_v( | ||
102 | gk20a_readl(g, fb_mmu_num_active_ltcs_r()))); | ||
103 | } | ||
104 | |||
105 | void gv11b_fb_init_cbc(struct gk20a *g, struct gr_gk20a *gr) | ||
106 | { | ||
107 | u32 max_size = gr->max_comptag_mem; | ||
108 | /* one tag line covers 64KB */ | ||
109 | u32 max_comptag_lines = max_size << 4; | ||
110 | u32 compbit_base_post_divide; | ||
111 | u64 compbit_base_post_multiply64; | ||
112 | u64 compbit_store_iova; | ||
113 | u64 compbit_base_post_divide64; | ||
114 | |||
115 | if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) | ||
116 | compbit_store_iova = nvgpu_mem_get_phys_addr(g, | ||
117 | &gr->compbit_store.mem); | ||
118 | else | ||
119 | compbit_store_iova = nvgpu_mem_get_addr(g, | ||
120 | &gr->compbit_store.mem); | ||
121 | |||
122 | compbit_base_post_divide64 = compbit_store_iova >> | ||
123 | fb_mmu_cbc_base_address_alignment_shift_v(); | ||
124 | |||
125 | do_div(compbit_base_post_divide64, g->ltc_count); | ||
126 | compbit_base_post_divide = u64_lo32(compbit_base_post_divide64); | ||
127 | |||
128 | compbit_base_post_multiply64 = ((u64)compbit_base_post_divide * | ||
129 | g->ltc_count) << fb_mmu_cbc_base_address_alignment_shift_v(); | ||
130 | |||
131 | if (compbit_base_post_multiply64 < compbit_store_iova) | ||
132 | compbit_base_post_divide++; | ||
133 | |||
134 | if (g->ops.ltc.cbc_fix_config) | ||
135 | compbit_base_post_divide = | ||
136 | g->ops.ltc.cbc_fix_config(g, compbit_base_post_divide); | ||
137 | |||
138 | gk20a_writel(g, fb_mmu_cbc_base_r(), | ||
139 | fb_mmu_cbc_base_address_f(compbit_base_post_divide)); | ||
140 | |||
141 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte, | ||
142 | "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n", | ||
143 | (u32)(compbit_store_iova >> 32), | ||
144 | (u32)(compbit_store_iova & 0xffffffff), | ||
145 | compbit_base_post_divide); | ||
146 | nvgpu_log(g, gpu_dbg_fn, "cbc base %x", | ||
147 | gk20a_readl(g, fb_mmu_cbc_base_r())); | ||
148 | |||
149 | gr->compbit_store.base_hw = compbit_base_post_divide; | ||
150 | |||
151 | g->ops.ltc.cbc_ctrl(g, gk20a_cbc_op_invalidate, | ||
152 | 0, max_comptag_lines - 1); | ||
153 | |||
154 | } | ||
155 | |||
156 | void gv11b_fb_reset(struct gk20a *g) | ||
157 | { | ||
158 | if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) { | ||
159 | nvgpu_log(g, gpu_dbg_info, "mc_elpg_enable set by bpmp"); | ||
160 | } else { | ||
161 | u32 mc_elpg_enable_val; | ||
162 | |||
163 | nvgpu_log(g, gpu_dbg_info, "enable xbar, pfb and hub"); | ||
164 | mc_elpg_enable_val = mc_elpg_enable_xbar_enabled_f() | | ||
165 | mc_elpg_enable_pfb_enabled_f() | | ||
166 | mc_elpg_enable_hub_enabled_f(); | ||
167 | mc_elpg_enable_val |= gk20a_readl(g, mc_elpg_enable_r()); | ||
168 | gk20a_writel(g, mc_elpg_enable_r(), mc_elpg_enable_val); | ||
169 | |||
170 | } | ||
171 | /* fs hub should be out of reset by now */ | ||
172 | gv11b_init_nvlink_soc_credits(g); | ||
173 | } | ||
174 | |||
175 | static const char * const invalid_str = "invalid"; | ||
176 | |||
177 | static const char *const fault_type_descs_gv11b[] = { | ||
178 | "invalid pde", | ||
179 | "invalid pde size", | ||
180 | "invalid pte", | ||
181 | "limit violation", | ||
182 | "unbound inst block", | ||
183 | "priv violation", | ||
184 | "write", | ||
185 | "read", | ||
186 | "pitch mask violation", | ||
187 | "work creation", | ||
188 | "unsupported aperture", | ||
189 | "compression failure", | ||
190 | "unsupported kind", | ||
191 | "region violation", | ||
192 | "poison", | ||
193 | "atomic" | ||
194 | }; | ||
195 | |||
196 | static const char *const fault_client_type_descs_gv11b[] = { | ||
197 | "gpc", | ||
198 | "hub", | ||
199 | }; | ||
200 | |||
201 | static const char *const fault_access_type_descs_gv11b[] = { | ||
202 | "virt read", | ||
203 | "virt write", | ||
204 | "virt atomic strong", | ||
205 | "virt prefetch", | ||
206 | "virt atomic weak", | ||
207 | "xxx", | ||
208 | "xxx", | ||
209 | "xxx", | ||
210 | "phys read", | ||
211 | "phys write", | ||
212 | "phys atomic", | ||
213 | "phys prefetch", | ||
214 | }; | ||
215 | |||
216 | static const char *const hub_client_descs_gv11b[] = { | ||
217 | "vip", "ce0", "ce1", "dniso", "fe", "fecs", "host", "host cpu", | ||
218 | "host cpu nb", "iso", "mmu", "nvdec", "nvenc1", "nvenc2", | ||
219 | "niso", "p2p", "pd", "perf", "pmu", "raster twod", "scc", | ||
220 | "scc nb", "sec", "ssync", "gr copy", "xv", "mmu nb", | ||
221 | "nvenc", "d falcon", "sked", "a falcon", "hsce0", "hsce1", | ||
222 | "hsce2", "hsce3", "hsce4", "hsce5", "hsce6", "hsce7", "hsce8", | ||
223 | "hsce9", "hshub", "ptp x0", "ptp x1", "ptp x2", "ptp x3", | ||
224 | "ptp x4", "ptp x5", "ptp x6", "ptp x7", "vpr scrubber0", | ||
225 | "vpr scrubber1", "dwbif", "fbfalcon", "ce shim", "gsp", | ||
226 | "dont care" | ||
227 | }; | ||
228 | |||
229 | static const char *const gpc_client_descs_gv11b[] = { | ||
230 | "t1 0", "t1 1", "t1 2", "t1 3", | ||
231 | "t1 4", "t1 5", "t1 6", "t1 7", | ||
232 | "pe 0", "pe 1", "pe 2", "pe 3", | ||
233 | "pe 4", "pe 5", "pe 6", "pe 7", | ||
234 | "rast", "gcc", "gpccs", | ||
235 | "prop 0", "prop 1", "prop 2", "prop 3", | ||
236 | "gpm", | ||
237 | "ltp utlb 0", "ltp utlb 1", "ltp utlb 2", "ltp utlb 3", | ||
238 | "ltp utlb 4", "ltp utlb 5", "ltp utlb 6", "ltp utlb 7", | ||
239 | "utlb", | ||
240 | "t1 8", "t1 9", "t1 10", "t1 11", | ||
241 | "t1 12", "t1 13", "t1 14", "t1 15", | ||
242 | "tpccs 0", "tpccs 1", "tpccs 2", "tpccs 3", | ||
243 | "tpccs 4", "tpccs 5", "tpccs 6", "tpccs 7", | ||
244 | "pe 8", "pe 9", "tpccs 8", "tpccs 9", | ||
245 | "t1 16", "t1 17", "t1 18", "t1 19", | ||
246 | "pe 10", "pe 11", "tpccs 10", "tpccs 11", | ||
247 | "t1 20", "t1 21", "t1 22", "t1 23", | ||
248 | "pe 12", "pe 13", "tpccs 12", "tpccs 13", | ||
249 | "t1 24", "t1 25", "t1 26", "t1 27", | ||
250 | "pe 14", "pe 15", "tpccs 14", "tpccs 15", | ||
251 | "t1 28", "t1 29", "t1 30", "t1 31", | ||
252 | "pe 16", "pe 17", "tpccs 16", "tpccs 17", | ||
253 | "t1 32", "t1 33", "t1 34", "t1 35", | ||
254 | "pe 18", "pe 19", "tpccs 18", "tpccs 19", | ||
255 | "t1 36", "t1 37", "t1 38", "t1 39", | ||
256 | }; | ||
257 | |||
258 | u32 gv11b_fb_is_fault_buf_enabled(struct gk20a *g, | ||
259 | unsigned int index) | ||
260 | { | ||
261 | u32 reg_val; | ||
262 | |||
263 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
264 | return fb_mmu_fault_buffer_size_enable_v(reg_val); | ||
265 | } | ||
266 | |||
267 | static void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g, | ||
268 | unsigned int index, u32 next) | ||
269 | { | ||
270 | u32 reg_val; | ||
271 | |||
272 | nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next); | ||
273 | |||
274 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
275 | reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(), | ||
276 | fb_mmu_fault_buffer_get_ptr_f(next)); | ||
277 | |||
278 | /* while the fault is being handled it is possible for overflow | ||
279 | * to happen, | ||
280 | */ | ||
281 | if (reg_val & fb_mmu_fault_buffer_get_overflow_m()) | ||
282 | reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f(); | ||
283 | |||
284 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
285 | |||
286 | /* make sure get ptr update is visible to everyone to avoid | ||
287 | * reading already read entry | ||
288 | */ | ||
289 | nvgpu_mb(); | ||
290 | } | ||
291 | |||
292 | static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, | ||
293 | unsigned int index) | ||
294 | { | ||
295 | u32 reg_val; | ||
296 | |||
297 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
298 | return fb_mmu_fault_buffer_get_ptr_v(reg_val); | ||
299 | } | ||
300 | |||
301 | static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, | ||
302 | unsigned int index) | ||
303 | { | ||
304 | u32 reg_val; | ||
305 | |||
306 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_put_r(index)); | ||
307 | return fb_mmu_fault_buffer_put_ptr_v(reg_val); | ||
308 | } | ||
309 | |||
310 | static u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, | ||
311 | unsigned int index) | ||
312 | { | ||
313 | u32 reg_val; | ||
314 | |||
315 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
316 | return fb_mmu_fault_buffer_size_val_v(reg_val); | ||
317 | } | ||
318 | |||
319 | static bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g, | ||
320 | unsigned int index, u32 *get_idx) | ||
321 | { | ||
322 | u32 put_idx; | ||
323 | |||
324 | *get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
325 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
326 | |||
327 | return *get_idx == put_idx; | ||
328 | } | ||
329 | |||
330 | static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, | ||
331 | unsigned int index) | ||
332 | { | ||
333 | u32 get_idx, put_idx, entries; | ||
334 | |||
335 | |||
336 | get_idx = gv11b_fb_fault_buffer_get_index(g, index); | ||
337 | |||
338 | put_idx = gv11b_fb_fault_buffer_put_index(g, index); | ||
339 | |||
340 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
341 | |||
342 | return get_idx == ((put_idx + 1) % entries); | ||
343 | } | ||
344 | |||
345 | void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g, | ||
346 | unsigned int index, unsigned int state) | ||
347 | { | ||
348 | u32 fault_status; | ||
349 | u32 reg_val; | ||
350 | |||
351 | nvgpu_log_fn(g, " "); | ||
352 | |||
353 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_size_r(index)); | ||
354 | if (state) { | ||
355 | if (gv11b_fb_is_fault_buf_enabled(g, index)) { | ||
356 | nvgpu_log_info(g, "fault buffer is already enabled"); | ||
357 | } else { | ||
358 | reg_val |= fb_mmu_fault_buffer_size_enable_true_f(); | ||
359 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), | ||
360 | reg_val); | ||
361 | } | ||
362 | |||
363 | } else { | ||
364 | struct nvgpu_timeout timeout; | ||
365 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
366 | |||
367 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
368 | NVGPU_TIMER_CPU_TIMER); | ||
369 | |||
370 | reg_val &= (~(fb_mmu_fault_buffer_size_enable_m())); | ||
371 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val); | ||
372 | |||
373 | fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
374 | |||
375 | do { | ||
376 | if (!(fault_status & fb_mmu_fault_status_busy_true_f())) | ||
377 | break; | ||
378 | /* | ||
379 | * Make sure fault buffer is disabled. | ||
380 | * This is to avoid accessing fault buffer by hw | ||
381 | * during the window BAR2 is being unmapped by s/w | ||
382 | */ | ||
383 | nvgpu_log_info(g, "fault status busy set, check again"); | ||
384 | fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
385 | |||
386 | nvgpu_usleep_range(delay, delay * 2); | ||
387 | delay = min_t(u32, delay << 1, GR_IDLE_CHECK_MAX); | ||
388 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
389 | "fault status busy set")); | ||
390 | } | ||
391 | } | ||
392 | |||
393 | void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, unsigned int index) | ||
394 | { | ||
395 | u32 addr_lo; | ||
396 | u32 addr_hi; | ||
397 | |||
398 | nvgpu_log_fn(g, " "); | ||
399 | |||
400 | gv11b_fb_fault_buf_set_state_hw(g, index, | ||
401 | FAULT_BUF_DISABLED); | ||
402 | addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >> | ||
403 | ram_in_base_shift_v()); | ||
404 | addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va); | ||
405 | |||
406 | gk20a_writel(g, fb_mmu_fault_buffer_lo_r(index), | ||
407 | fb_mmu_fault_buffer_lo_addr_f(addr_lo)); | ||
408 | |||
409 | gk20a_writel(g, fb_mmu_fault_buffer_hi_r(index), | ||
410 | fb_mmu_fault_buffer_hi_addr_f(addr_hi)); | ||
411 | |||
412 | gk20a_writel(g, fb_mmu_fault_buffer_size_r(index), | ||
413 | fb_mmu_fault_buffer_size_val_f(g->ops.fifo.get_num_fifos(g)) | | ||
414 | fb_mmu_fault_buffer_size_overflow_intr_enable_f()); | ||
415 | |||
416 | gv11b_fb_fault_buf_set_state_hw(g, index, FAULT_BUF_ENABLED); | ||
417 | } | ||
418 | |||
419 | static void gv11b_fb_intr_en_set(struct gk20a *g, | ||
420 | unsigned int index, u32 mask) | ||
421 | { | ||
422 | u32 reg_val; | ||
423 | |||
424 | reg_val = gk20a_readl(g, fb_niso_intr_en_set_r(index)); | ||
425 | reg_val |= mask; | ||
426 | gk20a_writel(g, fb_niso_intr_en_set_r(index), reg_val); | ||
427 | } | ||
428 | |||
429 | static void gv11b_fb_intr_en_clr(struct gk20a *g, | ||
430 | unsigned int index, u32 mask) | ||
431 | { | ||
432 | u32 reg_val; | ||
433 | |||
434 | reg_val = gk20a_readl(g, fb_niso_intr_en_clr_r(index)); | ||
435 | reg_val |= mask; | ||
436 | gk20a_writel(g, fb_niso_intr_en_clr_r(index), reg_val); | ||
437 | } | ||
438 | |||
439 | static u32 gv11b_fb_get_hub_intr_clr_mask(struct gk20a *g, | ||
440 | unsigned int intr_type) | ||
441 | { | ||
442 | u32 mask = 0; | ||
443 | |||
444 | if (intr_type & HUB_INTR_TYPE_OTHER) { | ||
445 | mask |= | ||
446 | fb_niso_intr_en_clr_mmu_other_fault_notify_m(); | ||
447 | } | ||
448 | |||
449 | if (intr_type & HUB_INTR_TYPE_NONREPLAY) { | ||
450 | mask |= | ||
451 | fb_niso_intr_en_clr_mmu_nonreplayable_fault_notify_m() | | ||
452 | fb_niso_intr_en_clr_mmu_nonreplayable_fault_overflow_m(); | ||
453 | } | ||
454 | |||
455 | if (intr_type & HUB_INTR_TYPE_REPLAY) { | ||
456 | mask |= | ||
457 | fb_niso_intr_en_clr_mmu_replayable_fault_notify_m() | | ||
458 | fb_niso_intr_en_clr_mmu_replayable_fault_overflow_m(); | ||
459 | } | ||
460 | |||
461 | if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) { | ||
462 | mask |= | ||
463 | fb_niso_intr_en_clr_mmu_ecc_uncorrected_error_notify_m(); | ||
464 | } | ||
465 | |||
466 | if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) { | ||
467 | mask |= | ||
468 | fb_niso_intr_en_clr_hub_access_counter_notify_m() | | ||
469 | fb_niso_intr_en_clr_hub_access_counter_error_m(); | ||
470 | } | ||
471 | |||
472 | return mask; | ||
473 | } | ||
474 | |||
475 | static u32 gv11b_fb_get_hub_intr_en_mask(struct gk20a *g, | ||
476 | unsigned int intr_type) | ||
477 | { | ||
478 | u32 mask = 0; | ||
479 | |||
480 | if (intr_type & HUB_INTR_TYPE_OTHER) { | ||
481 | mask |= | ||
482 | fb_niso_intr_en_set_mmu_other_fault_notify_m(); | ||
483 | } | ||
484 | |||
485 | if (intr_type & HUB_INTR_TYPE_NONREPLAY) { | ||
486 | mask |= | ||
487 | fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() | | ||
488 | fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m(); | ||
489 | } | ||
490 | |||
491 | if (intr_type & HUB_INTR_TYPE_REPLAY) { | ||
492 | mask |= | ||
493 | fb_niso_intr_en_set_mmu_replayable_fault_notify_m() | | ||
494 | fb_niso_intr_en_set_mmu_replayable_fault_overflow_m(); | ||
495 | } | ||
496 | |||
497 | if (intr_type & HUB_INTR_TYPE_ECC_UNCORRECTED) { | ||
498 | mask |= | ||
499 | fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); | ||
500 | } | ||
501 | |||
502 | if (intr_type & HUB_INTR_TYPE_ACCESS_COUNTER) { | ||
503 | mask |= | ||
504 | fb_niso_intr_en_set_hub_access_counter_notify_m() | | ||
505 | fb_niso_intr_en_set_hub_access_counter_error_m(); | ||
506 | } | ||
507 | |||
508 | return mask; | ||
509 | } | ||
510 | |||
511 | void gv11b_fb_enable_hub_intr(struct gk20a *g, | ||
512 | unsigned int index, unsigned int intr_type) | ||
513 | { | ||
514 | u32 mask = 0; | ||
515 | |||
516 | mask = gv11b_fb_get_hub_intr_en_mask(g, intr_type); | ||
517 | |||
518 | if (mask) | ||
519 | gv11b_fb_intr_en_set(g, index, mask); | ||
520 | } | ||
521 | |||
522 | void gv11b_fb_disable_hub_intr(struct gk20a *g, | ||
523 | unsigned int index, unsigned int intr_type) | ||
524 | { | ||
525 | u32 mask = 0; | ||
526 | |||
527 | mask = gv11b_fb_get_hub_intr_clr_mask(g, intr_type); | ||
528 | |||
529 | if (mask) | ||
530 | gv11b_fb_intr_en_clr(g, index, mask); | ||
531 | } | ||
532 | |||
533 | static void gv11b_handle_l2tlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
534 | { | ||
535 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
536 | u32 corrected_delta, uncorrected_delta; | ||
537 | u32 corrected_overflow, uncorrected_overflow; | ||
538 | |||
539 | ecc_addr = gk20a_readl(g, fb_mmu_l2tlb_ecc_address_r()); | ||
540 | corrected_cnt = gk20a_readl(g, | ||
541 | fb_mmu_l2tlb_ecc_corrected_err_count_r()); | ||
542 | uncorrected_cnt = gk20a_readl(g, | ||
543 | fb_mmu_l2tlb_ecc_uncorrected_err_count_r()); | ||
544 | |||
545 | corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v( | ||
546 | corrected_cnt); | ||
547 | uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v( | ||
548 | uncorrected_cnt); | ||
549 | corrected_overflow = ecc_status & | ||
550 | fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
551 | |||
552 | uncorrected_overflow = ecc_status & | ||
553 | fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
554 | |||
555 | /* clear the interrupt */ | ||
556 | if ((corrected_delta > 0) || corrected_overflow) | ||
557 | gk20a_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0); | ||
558 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
559 | gk20a_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0); | ||
560 | |||
561 | gk20a_writel(g, fb_mmu_l2tlb_ecc_status_r(), | ||
562 | fb_mmu_l2tlb_ecc_status_reset_clear_f()); | ||
563 | |||
564 | /* Handle overflow */ | ||
565 | if (corrected_overflow) | ||
566 | corrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_corrected_err_count_total_s()); | ||
567 | if (uncorrected_overflow) | ||
568 | uncorrected_delta += (0x1UL << fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s()); | ||
569 | |||
570 | |||
571 | g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0] += | ||
572 | corrected_delta; | ||
573 | g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0] += | ||
574 | uncorrected_delta; | ||
575 | |||
576 | if (ecc_status & fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m()) | ||
577 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
578 | if (ecc_status & fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m()) | ||
579 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
580 | if (corrected_overflow || uncorrected_overflow) | ||
581 | nvgpu_info(g, "mmu l2tlb ecc counter overflow!"); | ||
582 | |||
583 | nvgpu_log(g, gpu_dbg_intr, | ||
584 | "ecc error address: 0x%x", ecc_addr); | ||
585 | nvgpu_log(g, gpu_dbg_intr, | ||
586 | "ecc error count corrected: %d, uncorrected %d", | ||
587 | g->ecc.eng.t19x.mmu_l2tlb_corrected_err_count.counters[0], | ||
588 | g->ecc.eng.t19x.mmu_l2tlb_uncorrected_err_count.counters[0]); | ||
589 | } | ||
590 | |||
591 | static void gv11b_handle_hubtlb_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
592 | { | ||
593 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
594 | u32 corrected_delta, uncorrected_delta; | ||
595 | u32 corrected_overflow, uncorrected_overflow; | ||
596 | |||
597 | ecc_addr = gk20a_readl(g, fb_mmu_hubtlb_ecc_address_r()); | ||
598 | corrected_cnt = gk20a_readl(g, | ||
599 | fb_mmu_hubtlb_ecc_corrected_err_count_r()); | ||
600 | uncorrected_cnt = gk20a_readl(g, | ||
601 | fb_mmu_hubtlb_ecc_uncorrected_err_count_r()); | ||
602 | |||
603 | corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v( | ||
604 | corrected_cnt); | ||
605 | uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v( | ||
606 | uncorrected_cnt); | ||
607 | corrected_overflow = ecc_status & | ||
608 | fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m(); | ||
609 | |||
610 | uncorrected_overflow = ecc_status & | ||
611 | fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
612 | |||
613 | /* clear the interrupt */ | ||
614 | if ((corrected_delta > 0) || corrected_overflow) | ||
615 | gk20a_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0); | ||
616 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
617 | gk20a_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0); | ||
618 | |||
619 | gk20a_writel(g, fb_mmu_hubtlb_ecc_status_r(), | ||
620 | fb_mmu_hubtlb_ecc_status_reset_clear_f()); | ||
621 | |||
622 | /* Handle overflow */ | ||
623 | if (corrected_overflow) | ||
624 | corrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_corrected_err_count_total_s()); | ||
625 | if (uncorrected_overflow) | ||
626 | uncorrected_delta += (0x1UL << fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s()); | ||
627 | |||
628 | |||
629 | g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0] += | ||
630 | corrected_delta; | ||
631 | g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0] += | ||
632 | uncorrected_delta; | ||
633 | |||
634 | if (ecc_status & fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) | ||
635 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error"); | ||
636 | if (ecc_status & fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) | ||
637 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error"); | ||
638 | if (corrected_overflow || uncorrected_overflow) | ||
639 | nvgpu_info(g, "mmu hubtlb ecc counter overflow!"); | ||
640 | |||
641 | nvgpu_log(g, gpu_dbg_intr, | ||
642 | "ecc error address: 0x%x", ecc_addr); | ||
643 | nvgpu_log(g, gpu_dbg_intr, | ||
644 | "ecc error count corrected: %d, uncorrected %d", | ||
645 | g->ecc.eng.t19x.mmu_hubtlb_corrected_err_count.counters[0], | ||
646 | g->ecc.eng.t19x.mmu_hubtlb_uncorrected_err_count.counters[0]); | ||
647 | } | ||
648 | |||
649 | static void gv11b_handle_fillunit_ecc_isr(struct gk20a *g, u32 ecc_status) | ||
650 | { | ||
651 | u32 ecc_addr, corrected_cnt, uncorrected_cnt; | ||
652 | u32 corrected_delta, uncorrected_delta; | ||
653 | u32 corrected_overflow, uncorrected_overflow; | ||
654 | |||
655 | ecc_addr = gk20a_readl(g, fb_mmu_fillunit_ecc_address_r()); | ||
656 | corrected_cnt = gk20a_readl(g, | ||
657 | fb_mmu_fillunit_ecc_corrected_err_count_r()); | ||
658 | uncorrected_cnt = gk20a_readl(g, | ||
659 | fb_mmu_fillunit_ecc_uncorrected_err_count_r()); | ||
660 | |||
661 | corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v( | ||
662 | corrected_cnt); | ||
663 | uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v( | ||
664 | uncorrected_cnt); | ||
665 | corrected_overflow = ecc_status & | ||
666 | fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m(); | ||
667 | |||
668 | uncorrected_overflow = ecc_status & | ||
669 | fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m(); | ||
670 | |||
671 | /* clear the interrupt */ | ||
672 | if ((corrected_delta > 0) || corrected_overflow) | ||
673 | gk20a_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0); | ||
674 | if ((uncorrected_delta > 0) || uncorrected_overflow) | ||
675 | gk20a_writel(g, fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0); | ||
676 | |||
677 | gk20a_writel(g, fb_mmu_fillunit_ecc_status_r(), | ||
678 | fb_mmu_fillunit_ecc_status_reset_clear_f()); | ||
679 | |||
680 | /* Handle overflow */ | ||
681 | if (corrected_overflow) | ||
682 | corrected_delta += (0x1UL << fb_mmu_fillunit_ecc_corrected_err_count_total_s()); | ||
683 | if (uncorrected_overflow) | ||
684 | uncorrected_delta += (0x1UL << fb_mmu_fillunit_ecc_uncorrected_err_count_total_s()); | ||
685 | |||
686 | |||
687 | g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0] += | ||
688 | corrected_delta; | ||
689 | g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0] += | ||
690 | uncorrected_delta; | ||
691 | |||
692 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) | ||
693 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error"); | ||
694 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m()) | ||
695 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error"); | ||
696 | if (ecc_status & fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) | ||
697 | nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error"); | ||
698 | if (ecc_status & fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m()) | ||
699 | nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error"); | ||
700 | |||
701 | if (corrected_overflow || uncorrected_overflow) | ||
702 | nvgpu_info(g, "mmu fillunit ecc counter overflow!"); | ||
703 | |||
704 | nvgpu_log(g, gpu_dbg_intr, | ||
705 | "ecc error address: 0x%x", ecc_addr); | ||
706 | nvgpu_log(g, gpu_dbg_intr, | ||
707 | "ecc error count corrected: %d, uncorrected %d", | ||
708 | g->ecc.eng.t19x.mmu_fillunit_corrected_err_count.counters[0], | ||
709 | g->ecc.eng.t19x.mmu_fillunit_uncorrected_err_count.counters[0]); | ||
710 | } | ||
711 | |||
712 | static void gv11b_fb_parse_mmfault(struct mmu_fault_info *mmfault) | ||
713 | { | ||
714 | if (WARN_ON(mmfault->fault_type >= | ||
715 | ARRAY_SIZE(fault_type_descs_gv11b))) | ||
716 | mmfault->fault_type_desc = invalid_str; | ||
717 | else | ||
718 | mmfault->fault_type_desc = | ||
719 | fault_type_descs_gv11b[mmfault->fault_type]; | ||
720 | |||
721 | if (WARN_ON(mmfault->client_type >= | ||
722 | ARRAY_SIZE(fault_client_type_descs_gv11b))) | ||
723 | mmfault->client_type_desc = invalid_str; | ||
724 | else | ||
725 | mmfault->client_type_desc = | ||
726 | fault_client_type_descs_gv11b[mmfault->client_type]; | ||
727 | |||
728 | mmfault->client_id_desc = invalid_str; | ||
729 | if (mmfault->client_type == | ||
730 | gmmu_fault_client_type_hub_v()) { | ||
731 | |||
732 | if (!(WARN_ON(mmfault->client_id >= | ||
733 | ARRAY_SIZE(hub_client_descs_gv11b)))) | ||
734 | mmfault->client_id_desc = | ||
735 | hub_client_descs_gv11b[mmfault->client_id]; | ||
736 | } else if (mmfault->client_type == | ||
737 | gmmu_fault_client_type_gpc_v()) { | ||
738 | if (!(WARN_ON(mmfault->client_id >= | ||
739 | ARRAY_SIZE(gpc_client_descs_gv11b)))) | ||
740 | mmfault->client_id_desc = | ||
741 | gpc_client_descs_gv11b[mmfault->client_id]; | ||
742 | } | ||
743 | |||
744 | } | ||
745 | |||
746 | static void gv11b_fb_print_fault_info(struct gk20a *g, | ||
747 | struct mmu_fault_info *mmfault) | ||
748 | { | ||
749 | if (mmfault && mmfault->valid) { | ||
750 | nvgpu_err(g, "[MMU FAULT] " | ||
751 | "mmu engine id: %d, " | ||
752 | "ch id: %d, " | ||
753 | "fault addr: 0x%llx, " | ||
754 | "fault addr aperture: %d, " | ||
755 | "fault type: %s, " | ||
756 | "access type: %s, ", | ||
757 | mmfault->mmu_engine_id, | ||
758 | mmfault->chid, | ||
759 | mmfault->fault_addr, | ||
760 | mmfault->fault_addr_aperture, | ||
761 | mmfault->fault_type_desc, | ||
762 | fault_access_type_descs_gv11b[mmfault->access_type]); | ||
763 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
764 | "mmu engine id: %d, " | ||
765 | "faulted act eng id if any: 0x%x, " | ||
766 | "faulted veid if any: 0x%x, " | ||
767 | "faulted pbdma id if any: 0x%x, " | ||
768 | "fault addr: 0x%llx, ", | ||
769 | mmfault->mmu_engine_id, | ||
770 | mmfault->faulted_engine, | ||
771 | mmfault->faulted_subid, | ||
772 | mmfault->faulted_pbdma, | ||
773 | mmfault->fault_addr); | ||
774 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
775 | "fault addr aperture: %d, " | ||
776 | "fault type: %s, " | ||
777 | "access type: %s, " | ||
778 | "inst ptr: 0x%llx, " | ||
779 | "inst ptr aperture: %d, ", | ||
780 | mmfault->fault_addr_aperture, | ||
781 | mmfault->fault_type_desc, | ||
782 | fault_access_type_descs_gv11b[mmfault->access_type], | ||
783 | mmfault->inst_ptr, | ||
784 | mmfault->inst_aperture); | ||
785 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
786 | "ch id: %d, " | ||
787 | "timestamp hi:lo 0x%08x:0x%08x, " | ||
788 | "client type: %s, " | ||
789 | "client id: %s, " | ||
790 | "gpc id if client type is gpc: %d, ", | ||
791 | mmfault->chid, | ||
792 | mmfault->timestamp_hi, mmfault->timestamp_lo, | ||
793 | mmfault->client_type_desc, | ||
794 | mmfault->client_id_desc, | ||
795 | mmfault->gpc_id); | ||
796 | nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] " | ||
797 | "protected mode: %d, " | ||
798 | "replayable fault: %d, " | ||
799 | "replayable fault en: %d ", | ||
800 | mmfault->protected_mode, | ||
801 | mmfault->replayable_fault, | ||
802 | mmfault->replay_fault_en); | ||
803 | } | ||
804 | } | ||
805 | |||
806 | /* | ||
807 | *Fault buffer format | ||
808 | * | ||
809 | * 31 28 24 23 16 15 8 7 4 0 | ||
810 | *.-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-. | ||
811 | *| inst_lo |0 0|apr|0 0 0 0 0 0 0 0| | ||
812 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
813 | *| inst_hi | | ||
814 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
815 | *| addr_31_12 | |AP | | ||
816 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
817 | *| addr_63_32 | | ||
818 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
819 | *| timestamp_lo | | ||
820 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
821 | *| timestamp_hi | | ||
822 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
823 | *| (reserved) | engine_id | | ||
824 | *`-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-' | ||
825 | *|V|R|P| gpc_id |0 0 0|t|0|acctp|0| client |RF0 0|faulttype| | ||
826 | */ | ||
827 | |||
828 | static void gv11b_fb_copy_from_hw_fault_buf(struct gk20a *g, | ||
829 | struct nvgpu_mem *mem, u32 offset, struct mmu_fault_info *mmfault) | ||
830 | { | ||
831 | u32 rd32_val; | ||
832 | u32 addr_lo, addr_hi; | ||
833 | u64 inst_ptr; | ||
834 | u32 chid = FIFO_INVAL_CHANNEL_ID; | ||
835 | struct channel_gk20a *refch; | ||
836 | |||
837 | memset(mmfault, 0, sizeof(*mmfault)); | ||
838 | |||
839 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
840 | gmmu_fault_buf_entry_inst_lo_w()); | ||
841 | addr_lo = gmmu_fault_buf_entry_inst_lo_v(rd32_val); | ||
842 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
843 | |||
844 | addr_hi = nvgpu_mem_rd32(g, mem, offset + | ||
845 | gmmu_fault_buf_entry_inst_hi_w()); | ||
846 | addr_hi = gmmu_fault_buf_entry_inst_hi_v(addr_hi); | ||
847 | |||
848 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
849 | |||
850 | /* refch will be put back after fault is handled */ | ||
851 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
852 | if (refch) | ||
853 | chid = refch->chid; | ||
854 | |||
855 | /* it is ok to continue even if refch is NULL */ | ||
856 | mmfault->refch = refch; | ||
857 | mmfault->chid = chid; | ||
858 | mmfault->inst_ptr = inst_ptr; | ||
859 | mmfault->inst_aperture = gmmu_fault_buf_entry_inst_aperture_v(rd32_val); | ||
860 | |||
861 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
862 | gmmu_fault_buf_entry_addr_lo_w()); | ||
863 | |||
864 | mmfault->fault_addr_aperture = | ||
865 | gmmu_fault_buf_entry_addr_phys_aperture_v(rd32_val); | ||
866 | addr_lo = gmmu_fault_buf_entry_addr_lo_v(rd32_val); | ||
867 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
868 | |||
869 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
870 | gmmu_fault_buf_entry_addr_hi_w()); | ||
871 | addr_hi = gmmu_fault_buf_entry_addr_hi_v(rd32_val); | ||
872 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
873 | |||
874 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
875 | gmmu_fault_buf_entry_timestamp_lo_w()); | ||
876 | mmfault->timestamp_lo = | ||
877 | gmmu_fault_buf_entry_timestamp_lo_v(rd32_val); | ||
878 | |||
879 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
880 | gmmu_fault_buf_entry_timestamp_hi_w()); | ||
881 | mmfault->timestamp_hi = | ||
882 | gmmu_fault_buf_entry_timestamp_hi_v(rd32_val); | ||
883 | |||
884 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
885 | gmmu_fault_buf_entry_engine_id_w()); | ||
886 | |||
887 | mmfault->mmu_engine_id = | ||
888 | gmmu_fault_buf_entry_engine_id_v(rd32_val); | ||
889 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
890 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
891 | &mmfault->faulted_pbdma); | ||
892 | |||
893 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
894 | gmmu_fault_buf_entry_fault_type_w()); | ||
895 | mmfault->client_id = | ||
896 | gmmu_fault_buf_entry_client_v(rd32_val); | ||
897 | mmfault->replayable_fault = | ||
898 | gmmu_fault_buf_entry_replayable_fault_v(rd32_val); | ||
899 | |||
900 | mmfault->fault_type = | ||
901 | gmmu_fault_buf_entry_fault_type_v(rd32_val); | ||
902 | mmfault->access_type = | ||
903 | gmmu_fault_buf_entry_access_type_v(rd32_val); | ||
904 | |||
905 | mmfault->client_type = | ||
906 | gmmu_fault_buf_entry_mmu_client_type_v(rd32_val); | ||
907 | |||
908 | mmfault->gpc_id = | ||
909 | gmmu_fault_buf_entry_gpc_id_v(rd32_val); | ||
910 | mmfault->protected_mode = | ||
911 | gmmu_fault_buf_entry_protected_mode_v(rd32_val); | ||
912 | |||
913 | mmfault->replay_fault_en = | ||
914 | gmmu_fault_buf_entry_replayable_fault_en_v(rd32_val); | ||
915 | |||
916 | mmfault->valid = gmmu_fault_buf_entry_valid_v(rd32_val); | ||
917 | |||
918 | rd32_val = nvgpu_mem_rd32(g, mem, offset + | ||
919 | gmmu_fault_buf_entry_fault_type_w()); | ||
920 | rd32_val &= ~(gmmu_fault_buf_entry_valid_m()); | ||
921 | nvgpu_mem_wr32(g, mem, offset + gmmu_fault_buf_entry_valid_w(), | ||
922 | rd32_val); | ||
923 | |||
924 | gv11b_fb_parse_mmfault(mmfault); | ||
925 | } | ||
926 | |||
927 | static void gv11b_fb_handle_mmu_fault_common(struct gk20a *g, | ||
928 | struct mmu_fault_info *mmfault, u32 *invalidate_replay_val) | ||
929 | { | ||
930 | unsigned int id_type; | ||
931 | u32 num_lce, act_eng_bitmask = 0; | ||
932 | int err = 0; | ||
933 | u32 id = ((u32)~0); | ||
934 | |||
935 | if (!mmfault->valid) | ||
936 | return; | ||
937 | |||
938 | gv11b_fb_print_fault_info(g, mmfault); | ||
939 | |||
940 | num_lce = gv11b_ce_get_num_lce(g); | ||
941 | if ((mmfault->mmu_engine_id >= | ||
942 | gmmu_fault_mmu_eng_id_ce0_v()) && | ||
943 | (mmfault->mmu_engine_id < | ||
944 | gmmu_fault_mmu_eng_id_ce0_v() + num_lce)) { | ||
945 | /* CE page faults are not reported as replayable */ | ||
946 | nvgpu_log(g, gpu_dbg_intr, "CE Faulted"); | ||
947 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
948 | gv11b_fifo_reset_pbdma_and_eng_faulted(g, mmfault->refch, | ||
949 | mmfault->faulted_pbdma, mmfault->faulted_engine); | ||
950 | if (!err) { | ||
951 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Fixed"); | ||
952 | *invalidate_replay_val = 0; | ||
953 | /* refch in mmfault is assigned at the time of copying | ||
954 | * fault info from snap reg or bar2 fault buf | ||
955 | */ | ||
956 | gk20a_channel_put(mmfault->refch); | ||
957 | return; | ||
958 | } | ||
959 | /* Do recovery. Channel recovery needs refch */ | ||
960 | nvgpu_log(g, gpu_dbg_intr, "CE Page Fault Not Fixed"); | ||
961 | } | ||
962 | |||
963 | if (!mmfault->replayable_fault) { | ||
964 | if (mmfault->fault_type == | ||
965 | gmmu_fault_type_unbound_inst_block_v()) { | ||
966 | /* | ||
967 | * Bug 1847172: When an engine faults due to an unbound | ||
968 | * instance block, the fault cannot be isolated to a | ||
969 | * single context so we need to reset the entire runlist | ||
970 | */ | ||
971 | id_type = ID_TYPE_UNKNOWN; | ||
972 | nvgpu_log(g, gpu_dbg_intr, "UNBOUND INST BLOCK MMU FAULT"); | ||
973 | |||
974 | } else if (mmfault->refch) { | ||
975 | if (gk20a_is_channel_marked_as_tsg(mmfault->refch)) { | ||
976 | id = mmfault->refch->tsgid; | ||
977 | id_type = ID_TYPE_TSG; | ||
978 | } else { | ||
979 | id = mmfault->chid; | ||
980 | id_type = ID_TYPE_CHANNEL; | ||
981 | } | ||
982 | } else { | ||
983 | id_type = ID_TYPE_UNKNOWN; | ||
984 | } | ||
985 | if (mmfault->faulted_engine != FIFO_INVAL_ENGINE_ID) | ||
986 | act_eng_bitmask = BIT(mmfault->faulted_engine); | ||
987 | |||
988 | g->ops.fifo.teardown_ch_tsg(g, act_eng_bitmask, | ||
989 | id, id_type, RC_TYPE_MMU_FAULT, mmfault); | ||
990 | } else { | ||
991 | if (mmfault->fault_type == gmmu_fault_type_pte_v()) { | ||
992 | nvgpu_log(g, gpu_dbg_intr, "invalid pte! try to fix"); | ||
993 | err = gv11b_fb_fix_page_fault(g, mmfault); | ||
994 | if (err) | ||
995 | *invalidate_replay_val |= | ||
996 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
997 | else | ||
998 | *invalidate_replay_val |= | ||
999 | fb_mmu_invalidate_replay_start_ack_all_f(); | ||
1000 | } else { | ||
1001 | /* cancel faults other than invalid pte */ | ||
1002 | *invalidate_replay_val |= | ||
1003 | fb_mmu_invalidate_replay_cancel_global_f(); | ||
1004 | } | ||
1005 | /* refch in mmfault is assigned at the time of copying | ||
1006 | * fault info from snap reg or bar2 fault buf | ||
1007 | */ | ||
1008 | gk20a_channel_put(mmfault->refch); | ||
1009 | } | ||
1010 | } | ||
1011 | |||
1012 | static void gv11b_fb_replay_or_cancel_faults(struct gk20a *g, | ||
1013 | u32 invalidate_replay_val) | ||
1014 | { | ||
1015 | int err = 0; | ||
1016 | |||
1017 | nvgpu_log_fn(g, " "); | ||
1018 | |||
1019 | if (invalidate_replay_val & | ||
1020 | fb_mmu_invalidate_replay_cancel_global_f()) { | ||
1021 | /* | ||
1022 | * cancel faults so that next time it faults as | ||
1023 | * replayable faults and channel recovery can be done | ||
1024 | */ | ||
1025 | err = gv11b_fb_mmu_invalidate_replay(g, | ||
1026 | fb_mmu_invalidate_replay_cancel_global_f()); | ||
1027 | } else if (invalidate_replay_val & | ||
1028 | fb_mmu_invalidate_replay_start_ack_all_f()) { | ||
1029 | /* pte valid is fixed. replay faulting request */ | ||
1030 | err = gv11b_fb_mmu_invalidate_replay(g, | ||
1031 | fb_mmu_invalidate_replay_start_ack_all_f()); | ||
1032 | } | ||
1033 | } | ||
1034 | |||
1035 | static void gv11b_fb_handle_mmu_nonreplay_replay_fault(struct gk20a *g, | ||
1036 | u32 fault_status, unsigned int index) | ||
1037 | { | ||
1038 | u32 get_indx, offset, rd32_val, entries; | ||
1039 | struct nvgpu_mem *mem; | ||
1040 | struct mmu_fault_info *mmfault; | ||
1041 | u32 invalidate_replay_val = 0; | ||
1042 | u64 prev_fault_addr = 0ULL; | ||
1043 | u64 next_fault_addr = 0ULL; | ||
1044 | |||
1045 | if (gv11b_fb_is_fault_buffer_empty(g, index, &get_indx)) { | ||
1046 | nvgpu_log(g, gpu_dbg_intr, | ||
1047 | "SPURIOUS mmu fault: reg index:%d", index); | ||
1048 | return; | ||
1049 | } | ||
1050 | nvgpu_info(g, "%s MMU FAULT" , | ||
1051 | index == REPLAY_REG_INDEX ? "REPLAY" : "NON-REPLAY"); | ||
1052 | |||
1053 | nvgpu_log(g, gpu_dbg_intr, "get ptr = %d", get_indx); | ||
1054 | |||
1055 | mem = &g->mm.hw_fault_buf[index]; | ||
1056 | mmfault = g->mm.fault_info[index]; | ||
1057 | |||
1058 | entries = gv11b_fb_fault_buffer_size_val(g, index); | ||
1059 | nvgpu_log(g, gpu_dbg_intr, "buffer num entries = %d", entries); | ||
1060 | |||
1061 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
1062 | nvgpu_log(g, gpu_dbg_intr, "starting word offset = 0x%x", offset); | ||
1063 | |||
1064 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
1065 | offset + gmmu_fault_buf_entry_valid_w()); | ||
1066 | nvgpu_log(g, gpu_dbg_intr, "entry valid offset val = 0x%x", rd32_val); | ||
1067 | |||
1068 | while ((rd32_val & gmmu_fault_buf_entry_valid_m())) { | ||
1069 | |||
1070 | nvgpu_log(g, gpu_dbg_intr, "entry valid = 0x%x", rd32_val); | ||
1071 | |||
1072 | gv11b_fb_copy_from_hw_fault_buf(g, mem, offset, mmfault); | ||
1073 | |||
1074 | get_indx = (get_indx + 1) % entries; | ||
1075 | nvgpu_log(g, gpu_dbg_intr, "new get index = %d", get_indx); | ||
1076 | |||
1077 | gv11b_fb_fault_buffer_get_ptr_update(g, index, get_indx); | ||
1078 | |||
1079 | offset = (get_indx * gmmu_fault_buf_size_v()) / sizeof(u32); | ||
1080 | nvgpu_log(g, gpu_dbg_intr, "next word offset = 0x%x", offset); | ||
1081 | |||
1082 | rd32_val = nvgpu_mem_rd32(g, mem, | ||
1083 | offset + gmmu_fault_buf_entry_valid_w()); | ||
1084 | |||
1085 | if (index == REPLAY_REG_INDEX && mmfault->fault_addr != 0ULL) { | ||
1086 | /* fault_addr "0" is not supposed to be fixed ever. | ||
1087 | * For the first time when prev = 0, next = 0 and | ||
1088 | * fault addr is also 0 then handle_mmu_fault_common will | ||
1089 | * not be called. Fix by checking fault_addr not equal to 0 | ||
1090 | */ | ||
1091 | prev_fault_addr = next_fault_addr; | ||
1092 | next_fault_addr = mmfault->fault_addr; | ||
1093 | if (prev_fault_addr == next_fault_addr) { | ||
1094 | nvgpu_log(g, gpu_dbg_intr, "pte is fixed"); | ||
1095 | if (mmfault->refch) | ||
1096 | gk20a_channel_put(mmfault->refch); | ||
1097 | /* pte already fixed for this addr */ | ||
1098 | continue; | ||
1099 | } | ||
1100 | } | ||
1101 | |||
1102 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1103 | &invalidate_replay_val); | ||
1104 | |||
1105 | } | ||
1106 | if (index == REPLAY_REG_INDEX && invalidate_replay_val) | ||
1107 | gv11b_fb_replay_or_cancel_faults(g, invalidate_replay_val); | ||
1108 | } | ||
1109 | |||
1110 | static void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g, | ||
1111 | u32 fault_status, struct mmu_fault_info *mmfault) | ||
1112 | { | ||
1113 | u32 reg_val; | ||
1114 | u32 addr_lo, addr_hi; | ||
1115 | u64 inst_ptr; | ||
1116 | int chid = FIFO_INVAL_CHANNEL_ID; | ||
1117 | struct channel_gk20a *refch; | ||
1118 | |||
1119 | memset(mmfault, 0, sizeof(*mmfault)); | ||
1120 | |||
1121 | if (!(fault_status & fb_mmu_fault_status_valid_set_f())) { | ||
1122 | |||
1123 | nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set"); | ||
1124 | return; | ||
1125 | } | ||
1126 | |||
1127 | reg_val = gk20a_readl(g, fb_mmu_fault_inst_lo_r()); | ||
1128 | addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val); | ||
1129 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1130 | |||
1131 | addr_hi = gk20a_readl(g, fb_mmu_fault_inst_hi_r()); | ||
1132 | addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi); | ||
1133 | inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1134 | |||
1135 | /* refch will be put back after fault is handled */ | ||
1136 | refch = gk20a_refch_from_inst_ptr(g, inst_ptr); | ||
1137 | if (refch) | ||
1138 | chid = refch->chid; | ||
1139 | |||
1140 | /* It is still ok to continue if refch is NULL */ | ||
1141 | mmfault->refch = refch; | ||
1142 | mmfault->chid = chid; | ||
1143 | mmfault->inst_ptr = inst_ptr; | ||
1144 | mmfault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val); | ||
1145 | mmfault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val); | ||
1146 | |||
1147 | gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(g, mmfault->mmu_engine_id, | ||
1148 | &mmfault->faulted_engine, &mmfault->faulted_subid, | ||
1149 | &mmfault->faulted_pbdma); | ||
1150 | |||
1151 | reg_val = gk20a_readl(g, fb_mmu_fault_addr_lo_r()); | ||
1152 | addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val); | ||
1153 | addr_lo = addr_lo << ram_in_base_shift_v(); | ||
1154 | |||
1155 | mmfault->fault_addr_aperture = | ||
1156 | fb_mmu_fault_addr_lo_phys_aperture_v(reg_val); | ||
1157 | |||
1158 | addr_hi = gk20a_readl(g, fb_mmu_fault_addr_hi_r()); | ||
1159 | addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi); | ||
1160 | mmfault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo); | ||
1161 | |||
1162 | reg_val = gk20a_readl(g, fb_mmu_fault_info_r()); | ||
1163 | mmfault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val); | ||
1164 | mmfault->replayable_fault = | ||
1165 | fb_mmu_fault_info_replayable_fault_v(reg_val); | ||
1166 | mmfault->client_id = fb_mmu_fault_info_client_v(reg_val); | ||
1167 | mmfault->access_type = fb_mmu_fault_info_access_type_v(reg_val); | ||
1168 | mmfault->client_type = fb_mmu_fault_info_client_type_v(reg_val); | ||
1169 | mmfault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val); | ||
1170 | mmfault->protected_mode = | ||
1171 | fb_mmu_fault_info_protected_mode_v(reg_val); | ||
1172 | mmfault->replay_fault_en = | ||
1173 | fb_mmu_fault_info_replayable_fault_en_v(reg_val); | ||
1174 | |||
1175 | mmfault->valid = fb_mmu_fault_info_valid_v(reg_val); | ||
1176 | |||
1177 | fault_status &= ~(fb_mmu_fault_status_valid_m()); | ||
1178 | gk20a_writel(g, fb_mmu_fault_status_r(), fault_status); | ||
1179 | |||
1180 | gv11b_fb_parse_mmfault(mmfault); | ||
1181 | |||
1182 | } | ||
1183 | |||
1184 | static void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g, | ||
1185 | u32 fault_status) | ||
1186 | { | ||
1187 | u32 reg_val; | ||
1188 | unsigned int index = REPLAY_REG_INDEX; | ||
1189 | |||
1190 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1191 | |||
1192 | if (fault_status & | ||
1193 | fb_mmu_fault_status_replayable_getptr_corrupted_m()) { | ||
1194 | |||
1195 | nvgpu_err(g, "replayable getptr corrupted set"); | ||
1196 | |||
1197 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1198 | |||
1199 | reg_val = set_field(reg_val, | ||
1200 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1201 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1202 | } | ||
1203 | |||
1204 | if (fault_status & | ||
1205 | fb_mmu_fault_status_replayable_overflow_m()) { | ||
1206 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1207 | |||
1208 | nvgpu_err(g, "replayable overflow: buffer full:%s", | ||
1209 | buffer_full?"true":"false"); | ||
1210 | |||
1211 | reg_val = set_field(reg_val, | ||
1212 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1213 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1214 | } | ||
1215 | |||
1216 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1217 | } | ||
1218 | |||
1219 | static void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g, | ||
1220 | u32 fault_status) | ||
1221 | { | ||
1222 | u32 reg_val; | ||
1223 | unsigned int index = NONREPLAY_REG_INDEX; | ||
1224 | |||
1225 | reg_val = gk20a_readl(g, fb_mmu_fault_buffer_get_r(index)); | ||
1226 | |||
1227 | if (fault_status & | ||
1228 | fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) { | ||
1229 | |||
1230 | nvgpu_err(g, "non replayable getptr corrupted set"); | ||
1231 | |||
1232 | gv11b_fb_fault_buf_configure_hw(g, index); | ||
1233 | |||
1234 | reg_val = set_field(reg_val, | ||
1235 | fb_mmu_fault_buffer_get_getptr_corrupted_m(), | ||
1236 | fb_mmu_fault_buffer_get_getptr_corrupted_clear_f()); | ||
1237 | } | ||
1238 | |||
1239 | if (fault_status & | ||
1240 | fb_mmu_fault_status_non_replayable_overflow_m()) { | ||
1241 | |||
1242 | bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index); | ||
1243 | |||
1244 | nvgpu_err(g, "non replayable overflow: buffer full:%s", | ||
1245 | buffer_full?"true":"false"); | ||
1246 | |||
1247 | reg_val = set_field(reg_val, | ||
1248 | fb_mmu_fault_buffer_get_overflow_m(), | ||
1249 | fb_mmu_fault_buffer_get_overflow_clear_f()); | ||
1250 | } | ||
1251 | |||
1252 | gk20a_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val); | ||
1253 | } | ||
1254 | |||
1255 | static void gv11b_fb_handle_bar2_fault(struct gk20a *g, | ||
1256 | struct mmu_fault_info *mmfault, u32 fault_status) | ||
1257 | { | ||
1258 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, | ||
1259 | HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY); | ||
1260 | |||
1261 | |||
1262 | if (fault_status & fb_mmu_fault_status_non_replayable_error_m()) { | ||
1263 | if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) | ||
1264 | gv11b_fb_fault_buf_configure_hw(g, NONREPLAY_REG_INDEX); | ||
1265 | } | ||
1266 | |||
1267 | if (fault_status & fb_mmu_fault_status_replayable_error_m()) { | ||
1268 | if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) | ||
1269 | gv11b_fb_fault_buf_configure_hw(g, REPLAY_REG_INDEX); | ||
1270 | } | ||
1271 | gv11b_ce_mthd_buffer_fault_in_bar2_fault(g); | ||
1272 | |||
1273 | g->ops.mm.init_bar2_mm_hw_setup(g); | ||
1274 | |||
1275 | if (mmfault->refch) { | ||
1276 | gk20a_channel_put(mmfault->refch); | ||
1277 | mmfault->refch = NULL; | ||
1278 | } | ||
1279 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, | ||
1280 | HUB_INTR_TYPE_NONREPLAY | HUB_INTR_TYPE_REPLAY); | ||
1281 | } | ||
1282 | |||
1283 | static void gv11b_fb_handle_other_fault_notify(struct gk20a *g, | ||
1284 | u32 fault_status) | ||
1285 | { | ||
1286 | struct mmu_fault_info *mmfault; | ||
1287 | u32 invalidate_replay_val = 0; | ||
1288 | |||
1289 | mmfault = g->mm.fault_info[FAULT_TYPE_OTHER_AND_NONREPLAY]; | ||
1290 | |||
1291 | gv11b_mm_copy_from_fault_snap_reg(g, fault_status, mmfault); | ||
1292 | |||
1293 | /* BAR2/Physical faults will not be snapped in hw fault buf */ | ||
1294 | if (mmfault->mmu_engine_id == gmmu_fault_mmu_eng_id_bar2_v()) { | ||
1295 | nvgpu_err(g, "BAR2 MMU FAULT"); | ||
1296 | gv11b_fb_handle_bar2_fault(g, mmfault, fault_status); | ||
1297 | |||
1298 | } else if (mmfault->mmu_engine_id == | ||
1299 | gmmu_fault_mmu_eng_id_physical_v()) { | ||
1300 | /* usually means VPR or out of bounds physical accesses */ | ||
1301 | nvgpu_err(g, "PHYSICAL MMU FAULT"); | ||
1302 | |||
1303 | } else { | ||
1304 | gv11b_fb_handle_mmu_fault_common(g, mmfault, | ||
1305 | &invalidate_replay_val); | ||
1306 | |||
1307 | if (invalidate_replay_val) | ||
1308 | gv11b_fb_replay_or_cancel_faults(g, | ||
1309 | invalidate_replay_val); | ||
1310 | } | ||
1311 | } | ||
1312 | |||
1313 | static void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status) | ||
1314 | { | ||
1315 | u32 dropped_faults = 0; | ||
1316 | |||
1317 | dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() | | ||
1318 | fb_mmu_fault_status_dropped_bar1_virt_set_f() | | ||
1319 | fb_mmu_fault_status_dropped_bar2_phys_set_f() | | ||
1320 | fb_mmu_fault_status_dropped_bar2_virt_set_f() | | ||
1321 | fb_mmu_fault_status_dropped_ifb_phys_set_f() | | ||
1322 | fb_mmu_fault_status_dropped_ifb_virt_set_f() | | ||
1323 | fb_mmu_fault_status_dropped_other_phys_set_f()| | ||
1324 | fb_mmu_fault_status_dropped_other_virt_set_f(); | ||
1325 | |||
1326 | if (fault_status & dropped_faults) { | ||
1327 | nvgpu_err(g, "dropped mmu fault (0x%08x)", | ||
1328 | fault_status & dropped_faults); | ||
1329 | gk20a_writel(g, fb_mmu_fault_status_r(), dropped_faults); | ||
1330 | } | ||
1331 | } | ||
1332 | |||
1333 | |||
1334 | static void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr) | ||
1335 | { | ||
1336 | u32 fault_status = gk20a_readl(g, fb_mmu_fault_status_r()); | ||
1337 | |||
1338 | nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status); | ||
1339 | |||
1340 | if (niso_intr & | ||
1341 | fb_niso_intr_mmu_other_fault_notify_m()) { | ||
1342 | |||
1343 | gv11b_fb_handle_dropped_mmu_fault(g, fault_status); | ||
1344 | |||
1345 | gv11b_fb_handle_other_fault_notify(g, fault_status); | ||
1346 | } | ||
1347 | |||
1348 | if (gv11b_fb_is_fault_buf_enabled(g, NONREPLAY_REG_INDEX)) { | ||
1349 | |||
1350 | if (niso_intr & | ||
1351 | fb_niso_intr_mmu_nonreplayable_fault_notify_m()) { | ||
1352 | |||
1353 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1354 | fault_status, NONREPLAY_REG_INDEX); | ||
1355 | |||
1356 | /* | ||
1357 | * When all the faults are processed, | ||
1358 | * GET and PUT will have same value and mmu fault status | ||
1359 | * bit will be reset by HW | ||
1360 | */ | ||
1361 | } | ||
1362 | if (niso_intr & | ||
1363 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) { | ||
1364 | |||
1365 | gv11b_fb_handle_nonreplay_fault_overflow(g, | ||
1366 | fault_status); | ||
1367 | } | ||
1368 | |||
1369 | } | ||
1370 | |||
1371 | if (gv11b_fb_is_fault_buf_enabled(g, REPLAY_REG_INDEX)) { | ||
1372 | |||
1373 | if (niso_intr & | ||
1374 | fb_niso_intr_mmu_replayable_fault_notify_m()) { | ||
1375 | |||
1376 | gv11b_fb_handle_mmu_nonreplay_replay_fault(g, | ||
1377 | fault_status, REPLAY_REG_INDEX); | ||
1378 | } | ||
1379 | if (niso_intr & | ||
1380 | fb_niso_intr_mmu_replayable_fault_overflow_m()) { | ||
1381 | |||
1382 | gv11b_fb_handle_replay_fault_overflow(g, | ||
1383 | fault_status); | ||
1384 | } | ||
1385 | |||
1386 | } | ||
1387 | |||
1388 | nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status"); | ||
1389 | gk20a_writel(g, fb_mmu_fault_status_r(), | ||
1390 | fb_mmu_fault_status_valid_clear_f()); | ||
1391 | } | ||
1392 | |||
1393 | void gv11b_fb_hub_isr(struct gk20a *g) | ||
1394 | { | ||
1395 | u32 status, niso_intr; | ||
1396 | |||
1397 | nvgpu_mutex_acquire(&g->mm.hub_isr_mutex); | ||
1398 | |||
1399 | niso_intr = gk20a_readl(g, fb_niso_intr_r()); | ||
1400 | |||
1401 | nvgpu_info(g, "enter hub isr, niso_intr = 0x%08x", niso_intr); | ||
1402 | |||
1403 | if (niso_intr & | ||
1404 | (fb_niso_intr_hub_access_counter_notify_m() | | ||
1405 | fb_niso_intr_hub_access_counter_error_m())) { | ||
1406 | |||
1407 | nvgpu_info(g, "hub access counter notify/error"); | ||
1408 | } | ||
1409 | if (niso_intr & | ||
1410 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) { | ||
1411 | |||
1412 | nvgpu_info(g, "ecc uncorrected error notify"); | ||
1413 | |||
1414 | /* disable interrupts during handling */ | ||
1415 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, | ||
1416 | HUB_INTR_TYPE_ECC_UNCORRECTED); | ||
1417 | |||
1418 | status = gk20a_readl(g, fb_mmu_l2tlb_ecc_status_r()); | ||
1419 | if (status) | ||
1420 | gv11b_handle_l2tlb_ecc_isr(g, status); | ||
1421 | |||
1422 | status = gk20a_readl(g, fb_mmu_hubtlb_ecc_status_r()); | ||
1423 | if (status) | ||
1424 | gv11b_handle_hubtlb_ecc_isr(g, status); | ||
1425 | |||
1426 | status = gk20a_readl(g, fb_mmu_fillunit_ecc_status_r()); | ||
1427 | if (status) | ||
1428 | gv11b_handle_fillunit_ecc_isr(g, status); | ||
1429 | |||
1430 | /* re-enable interrupts after handling */ | ||
1431 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, | ||
1432 | HUB_INTR_TYPE_ECC_UNCORRECTED); | ||
1433 | |||
1434 | } | ||
1435 | if (niso_intr & | ||
1436 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1437 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1438 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1439 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1440 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) { | ||
1441 | |||
1442 | nvgpu_info(g, "MMU Fault"); | ||
1443 | gv11b_fb_handle_mmu_fault(g, niso_intr); | ||
1444 | } | ||
1445 | |||
1446 | nvgpu_mutex_release(&g->mm.hub_isr_mutex); | ||
1447 | } | ||
1448 | |||
1449 | bool gv11b_fb_mmu_fault_pending(struct gk20a *g) | ||
1450 | { | ||
1451 | if (gk20a_readl(g, fb_niso_intr_r()) & | ||
1452 | (fb_niso_intr_mmu_other_fault_notify_m() | | ||
1453 | fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() | | ||
1454 | fb_niso_intr_mmu_replayable_fault_notify_m() | | ||
1455 | fb_niso_intr_mmu_replayable_fault_overflow_m() | | ||
1456 | fb_niso_intr_mmu_nonreplayable_fault_notify_m() | | ||
1457 | fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) | ||
1458 | return true; | ||
1459 | |||
1460 | return false; | ||
1461 | } | ||
1462 | |||
1463 | static int gv11b_fb_mmu_invalidate_replay(struct gk20a *g, | ||
1464 | u32 invalidate_replay_val) | ||
1465 | { | ||
1466 | int err = -ETIMEDOUT; | ||
1467 | u32 reg_val; | ||
1468 | struct nvgpu_timeout timeout; | ||
1469 | |||
1470 | gk20a_dbg_fn(""); | ||
1471 | |||
1472 | nvgpu_mutex_acquire(&g->mm.tlb_lock); | ||
1473 | |||
1474 | reg_val = gk20a_readl(g, fb_mmu_invalidate_r()); | ||
1475 | |||
1476 | reg_val |= fb_mmu_invalidate_all_va_true_f() | | ||
1477 | fb_mmu_invalidate_all_pdb_true_f() | | ||
1478 | invalidate_replay_val | | ||
1479 | fb_mmu_invalidate_trigger_true_f(); | ||
1480 | |||
1481 | gk20a_writel(g, fb_mmu_invalidate_r(), reg_val); | ||
1482 | |||
1483 | /* retry 200 times */ | ||
1484 | nvgpu_timeout_init(g, &timeout, 200, NVGPU_TIMER_RETRY_TIMER); | ||
1485 | do { | ||
1486 | reg_val = gk20a_readl(g, fb_mmu_ctrl_r()); | ||
1487 | if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) != | ||
1488 | fb_mmu_ctrl_pri_fifo_empty_false_f()) { | ||
1489 | err = 0; | ||
1490 | break; | ||
1491 | } | ||
1492 | nvgpu_udelay(5); | ||
1493 | } while (!nvgpu_timeout_expired_msg(&timeout, | ||
1494 | "invalidate replay failed on 0x%llx")); | ||
1495 | if (err) | ||
1496 | nvgpu_err(g, "invalidate replay timedout"); | ||
1497 | |||
1498 | nvgpu_mutex_release(&g->mm.tlb_lock); | ||
1499 | |||
1500 | return err; | ||
1501 | } | ||
1502 | |||
1503 | static int gv11b_fb_fix_page_fault(struct gk20a *g, | ||
1504 | struct mmu_fault_info *mmfault) | ||
1505 | { | ||
1506 | int err = 0; | ||
1507 | u32 pte[2]; | ||
1508 | |||
1509 | if (mmfault->refch == NULL) { | ||
1510 | nvgpu_log(g, gpu_dbg_intr, "refch from mmu_fault_info is NULL"); | ||
1511 | return -EINVAL; | ||
1512 | } | ||
1513 | |||
1514 | err = __nvgpu_get_pte(g, | ||
1515 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1516 | if (err) { | ||
1517 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not found"); | ||
1518 | return err; | ||
1519 | } | ||
1520 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1521 | "pte: %#08x %#08x", pte[1], pte[0]); | ||
1522 | |||
1523 | if (pte[0] == 0x0 && pte[1] == 0x0) { | ||
1524 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1525 | "pte all zeros, do not set valid"); | ||
1526 | return -1; | ||
1527 | } | ||
1528 | if (pte[0] & gmmu_new_pte_valid_true_f()) { | ||
1529 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1530 | "pte valid already set"); | ||
1531 | return -1; | ||
1532 | } | ||
1533 | |||
1534 | pte[0] |= gmmu_new_pte_valid_true_f(); | ||
1535 | if (pte[0] & gmmu_new_pte_read_only_true_f()) | ||
1536 | pte[0] &= ~(gmmu_new_pte_read_only_true_f()); | ||
1537 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1538 | "new pte: %#08x %#08x", pte[1], pte[0]); | ||
1539 | |||
1540 | err = __nvgpu_set_pte(g, | ||
1541 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1542 | if (err) { | ||
1543 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, "pte not fixed"); | ||
1544 | return err; | ||
1545 | } | ||
1546 | /* invalidate tlb so that GMMU does not use old cached translation */ | ||
1547 | g->ops.fb.tlb_invalidate(g, mmfault->refch->vm->pdb.mem); | ||
1548 | |||
1549 | err = __nvgpu_get_pte(g, | ||
1550 | mmfault->refch->vm, mmfault->fault_addr, &pte[0]); | ||
1551 | nvgpu_log(g, gpu_dbg_intr | gpu_dbg_pte, | ||
1552 | "pte after tlb invalidate: %#08x %#08x", | ||
1553 | pte[1], pte[0]); | ||
1554 | return err; | ||
1555 | } | ||